收藏一个碉堡了的robots.txt 我喜欢屏蔽了若干蜘蛛

然而屏蔽百度是必须的  但是人家不知道 360 和 搜狗这类无名小辈
  1. User-agent: *
  2. Disallow: *query=
  3. Disallow: *gtm.start
  4. Disallow: *tbgl=
  5. Disallow: *index.php
  6. Disallow: */date/
  7. Disallow: *link_key=
  8. Disallow: *usg=
  9. Disallow: *cad=
  10. Disallow: *slider
  11. Disallow: *post=
  12. Disallow: *amp;
  13. Disallow: *nbsp;
  14. Disallow: *page_
  15. Disallow: *destination=
  16. Disallow: *page%
  17. Disallow: *subscribe=
  18. Disallow: *height=
  19. Disallow: *width=
  20. Disallow: *0=
  21. # Directories
  22. Disallow: *tp=
  23. Disallow: /tag/
  24. Disallow: /tag/*page=
  25. Disallow: /app/
  26. Disallow: /vendor/
  27. Disallow: /bootstrap/
  28. Disallow: *comments
  29. # Paths (clean URLs)
  30. Disallow: /moderator/
  31. Disallow: /system/
  32. Disallow: /admin/
  33. Disallow: /acount/article/create
  34. Disallow: /search/
  35. Disallow: /register/
  36. Disallow: /forgot/
  37. Disallow: /login/
  38. Disallow: /logout/
  39. Disallow: /events
  40. Disallow: /followers
  41. Disallow: /reviews
  42. Disallow: /imageboard/
  43. Disallow: /thematic-ads
  44. # Paths (no clean URLs)
  45. Disallow: /?q=moderator/
  46. Disallow: /?q=system/
  47. Disallow: /?q=acount/article/create/
  48. Disallow: /?q=search/
  49. Disallow: /?q=forgot/
  50. Disallow: /?q=register/
  51. Disallow: /?q=login/
  52. Disallow: /?q=logout/
  53. Disallow: *q=
  54. Disallow: *openstat
  55. Disallow: *from
  56. Disallow: *gclid
  57. Disallow: *ggl
  58. Disallow: *utm
  59. Disallow: *node
  60. Disallow: /rubric/
  61. Disallow: /tag/
  62. Disallow: */feed
  63. Disallow: *id=
  64. Disallow: *fb_xd_fragment
  65. Disallow: /tw/
  66. Disallow: *p=
  67. Disallow: *cat=
  68. Disallow: /account/
  69. Disallow: *wmode=transparent
  70. Disallow: */dialogs/
  71. Disallow: */vopros/
  72. Disallow: *dialog/create/
  73. Disallow: */followers
  74. Disallow: */events
  75. Sitemap: http://www.kakprosto.ru/sitemap.xml
  76. User-agent: Yandex
  77. Disallow: */amp/
  78. Disallow: *query=
  79. Disallow: *gtm.start
  80. Disallow: *tbgl=
  81. Disallow: *index.php
  82. Disallow: */date/
  83. Disallow: *link_key=
  84. Disallow: *usg=
  85. Disallow: *cad=
  86. Disallow: *Width=
  87. Disallow: *slider
  88. Disallow: *post=
  89. Disallow: */kak-*?
  90. Disallow: *amp;
  91. Disallow: *nbsp;
  92. Disallow: *page_
  93. Disallow: *width
  94. Disallow: *height
  95. Disallow: *destination=
  96. Disallow: *page%
  97. Disallow: *subscribe=
  98. Disallow: *height=
  99. Disallow: *width=
  100. Disallow: *0=
  101. # Directories
  102. Disallow: *tp=
  103. Disallow: /tag/
  104. Disallow: /tag/*page=
  105. Disallow: /app/
  106. Disallow: /vendor/
  107. Disallow: /bootstrap/
  108. Disallow: *comments
  109. # Paths (clean URLs)
  110. Disallow: /moderator/
  111. Disallow: /system/
  112. Disallow: /account/article/create/
  113. Disallow: /search/
  114. Disallow: /register/
  115. Disallow: /forgot/
  116. Disallow: /login/
  117. Disallow: /logout/
  118. Disallow: /events
  119. Disallow: /followers
  120. Disallow: /reviews
  121. Disallow: /profile/
  122. Disallow: /imageboard/
  123. Disallow: /thematic-ads
  124. # Paths (no clean URLs)
  125. Disallow: /?q=moderator/
  126. Disallow: /?q=system/
  127. Disallow: /?q=account/article/create/
  128. Disallow: /?q=search/
  129. Disallow: /?q=forgot/
  130. Disallow: /?q=register/
  131. Disallow: /?q=login/
  132. Disallow: /?q=logout/
  133. Disallow: *q=
  134. Disallow: *openstat
  135. Disallow: *from
  136. Disallow: *gclid
  137. Disallow: *ggl
  138. Disallow: *utm
  139. Disallow: *node
  140. Disallow: /rubric/
  141. Disallow: /tag/
  142. Disallow: */feed
  143. Disallow: *id=
  144. Disallow: *fb_xd_fragment
  145. Disallow: /tw/
  146. Disallow: /print
  147. Disallow: *p=
  148. Disallow: *cat=
  149. Disallow: /account/
  150. Disallow: *wmode=transparent
  151. Disallow: */dialogs/
  152. Disallow: */vopros/
  153. Disallow: *dialog/create/
  154. Disallow: */followers
  155. Disallow: */events
  156. Host: www.kakprosto.ru
  157. User-agent: Googlebot
  158. Disallow: *query=
  159. Disallow: *tbgl=
  160. Disallow: *index.php
  161. Disallow: */date/
  162. Disallow: *link_key=
  163. Disallow: *usg=
  164. Disallow: *cad=
  165. Disallow: *slider
  166. Disallow: *post=
  167. Disallow: /profile/
  168. Disallow: /tag/
  169. Disallow: *amp;
  170. Disallow: *nbsp;
  171. Disallow: *page_
  172. Disallow: *width
  173. Disallow: *height
  174. Disallow: *destination=
  175. Disallow: *page%
  176. Disallow: *subscribe=
  177. Disallow: *height=
  178. Disallow: *width=
  179. Disallow: *0=
  180. # Directories
  181. Disallow: *tp=
  182. Disallow: /tag/*page=
  183. Disallow: /app/
  184. Disallow: /vendor/
  185. Disallow: /bootstrap/
  186. Disallow: *comments
  187. # Paths (clean URLs)
  188. Disallow: /moderator/
  189. Disallow: /system/
  190. Disallow: /account/article/add/
  191. Disallow: /search/
  192. Disallow: /register/
  193. Disallow: /password/
  194. Disallow: /login/
  195. Disallow: /logout/
  196. Disallow: /events
  197. Disallow: /followers
  198. Disallow: /reviews
  199. Disallow: /imageboard/
  200. Disallow: /thematic-ads
  201. # Paths (no clean URLs)
  202. Disallow: /?q=moderator/
  203. Disallow: /?q=system/
  204. Disallow: /?q=account/article/create/
  205. Disallow: /?q=search/
  206. Disallow: /?q=forgot/
  207. Disallow: /?q=register/
  208. Disallow: /?q=login/
  209. Disallow: /?q=logout/
  210. Disallow: *q=
  211. Disallow: *openstat
  212. Disallow: *from
  213. Disallow: *gclid
  214. Disallow: *ggl
  215. Disallow: *utm
  216. Disallow: *node
  217. Disallow: /rubric/
  218. Disallow: /tag/
  219. Disallow: */feed
  220. Disallow: *id=
  221. Disallow: *fb_xd_fragment
  222. Disallow: /tw/
  223. Disallow: /print
  224. Disallow: *p=
  225. Disallow: *cat=
  226. Disallow: /account/
  227. Disallow: *wmode=transparent
  228. Disallow: */dialogs/
  229. Disallow: */vopros/
  230. Disallow: *dialog/create/
  231. Disallow: */followers
  232. Disallow: */events
  233. User-Agent: DISCo Pump
  234. Disallow: /
  235. User-Agent: Wget
  236. Disallow: /
  237. User-Agent: WebZIP
  238. Disallow: /
  239. User-Agent: Teleport Pro
  240. Disallow: /
  241. User-Agent: WebSnake
  242. Disallow: /
  243. User-Agent: Offline Explorer
  244. Disallow: /
  245. User-Agent: Web-By-Mail
  246. Disallow: /
  247. User-agent: psbot
  248. Disallow: /
  249. User-agent: gigabot
  250. Disallow: /
  251. User-agent: Twiceler
  252. Disallow: /
  253. User-agent: NetinfoBot
  254. Disallow: /
  255. User-agent: Googlebot-Image
  256. Allow: /images/
  257. User-agent: YandexImages
  258. Allow: /images/
  259. User-agent: Gigabot/2.0
  260. Disallow: /
  261. User-agent: ia_archiver
  262. Disallow: /
  263. User-agent: libwww-perl
  264. Disallow: /
  265. User-agent: NetStat.Ru Agent
  266. Disallow: /
  267. User-agent: WebAlta Crawler/1.3.25
  268. Disallow: /
  269. User-agent: Yahoo!-MMCrawler/3.x
  270. Disallow: /
  271. User-agent: MMCrawler/3.x
  272. Disallow: /
  273. User-agent: NG/2.0
  274. Disallow: /
  275. User-agent: slurp
  276. Disallow: /
  277. User-agent: aipbot
  278. Disallow: /
  279. User-agent: Alexibot
  280. Disallow: /
  281. User-agent: GameSpyHTTP/1.0
  282. Disallow: /
  283. User-agent: Aqua_Products
  284. Disallow: /
  285. User-agent: asterias
  286. Disallow: /
  287. User-agent: b2w/0.1
  288. Disallow: /
  289. User-agent: BackDoorBot/1.0
  290. Disallow: /
  291. User-agent: becomebot
  292. Disallow: /
  293. User-agent: BlowFish/1.0
  294. Disallow: /
  295. User-agent: Bookmark search tool
  296. Disallow: /
  297. User-agent: BotALot
  298. Disallow: /
  299. User-agent: BotRightHere
  300. Disallow: /
  301. User-agent: BuiltBotTough
  302. Disallow: /
  303. User-agent: Bullseye/1.0
  304. Disallow: /
  305. User-agent: BunnySlippers
  306. Disallow: /
  307. User-agent: CheeseBot
  308. Disallow: /
  309. User-agent: CherryPicker
  310. Disallow: /
  311. User-agent: CherryPickerElite/1.0
  312. Disallow: /
  313. User-agent: CherryPickerSE/1.0
  314. Disallow: /
  315. User-agent: Copernic
  316. Disallow: /
  317. User-agent: CopyRightCheck
  318. Disallow: /
  319. User-agent: cosmos
  320. Disallow: /
  321. User-agent: Crescent
  322. Disallow: /
  323. User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
  324. Disallow: /
  325. User-agent: DittoSpyder
  326. Disallow: /
  327. User-agent: EmailCollector
  328. Disallow: /
  329. User-agent: EmailSiphon
  330. Disallow: /
  331. User-agent: EmailWolf
  332. Disallow: /
  333. User-agent: EroCrawler
  334. Disallow: /
  335. User-agent: ExtractorPro
  336. Disallow: /
  337. User-agent: FairAd Client
  338. Disallow: /
  339. User-agent: Fasterfox
  340. Disallow: /
  341. User-agent: Flaming AttackBot
  342. Disallow: /
  343. User-agent: Foobot
  344. Disallow: /
  345. User-agent: Gaisbot
  346. Disallow: /
  347. User-agent: GetRight/4.2
  348. Disallow: /
  349. User-agent: Harvest/1.5
  350. Disallow: /
  351. User-agent: hloader
  352. Disallow: /
  353. User-agent: httplib
  354. Disallow: /
  355. User-agent: HTTrack 3.0
  356. Disallow: /
  357. User-agent: humanlinks
  358. Disallow: /
  359. User-agent: IconSurf
  360. Disallow: /
  361. User-agent: InfoNaviRobot
  362. Disallow: /
  363. User-agent: Iron33/1.0.2
  364. Disallow: /
  365. User-agent: JennyBot
  366. Disallow: /
  367. User-agent: Kenjin Spider
  368. Disallow: /
  369. User-agent: Keyword Density/0.9
  370. Disallow: /
  371. User-agent: larbin
  372. Disallow: /
  373. User-agent: LexiBot
  374. Disallow: /
  375. User-agent: libWeb/clsHTTP
  376. Disallow: /
  377. User-agent: LinkextractorPro
  378. Disallow: /
  379. User-agent: LinkScan/8.1a Unix
  380. Disallow: /
  381. User-agent: LinkWalker
  382. Disallow: /
  383. User-agent: LNSpiderguy
  384. Disallow: /
  385. User-agent: lwp-trivial
  386. Disallow: /
  387. User-agent: lwp-trivial/1.34
  388. Disallow: /
  389. User-agent: Mata Hari
  390. Disallow: /
  391. User-agent: Microsoft URL Control
  392. Disallow: /
  393. User-agent: Microsoft URL Control - 5.01.4511
  394. Disallow: /
  395. User-agent: Microsoft URL Control - 6.00.8169
  396. Disallow: /
  397. User-agent: MIIxpc
  398. Disallow: /
  399. User-agent: MIIxpc/4.2
  400. Disallow: /
  401. User-agent: Mister PiX
  402. Disallow: /
  403. User-agent: moget
  404. Disallow: /
  405. User-agent: moget/2.1
  406. Disallow: /
  407. User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95)
  408. Disallow: /
  409. User-agent: MSIECrawler
  410. Disallow: /
  411. User-agent: NetAnts
  412. Disallow: /
  413. User-agent: NICErsPRO
  414. Disallow: /
  415. User-agent: Offline Explorer
  416. Disallow: /
  417. User-agent: Openbot
  418. Disallow: /
  419. User-agent: Openfind
  420. Disallow: /
  421. User-agent: Openfind data gatherer
  422. Disallow: /
  423. User-agent: Oracle Ultra Search
  424. Disallow: /
  425. User-agent: PerMan
  426. Disallow: /
  427. User-agent: ProPowerBot/2.14
  428. Disallow: /
  429. User-agent: ProWebWalker
  430. Disallow: /
  431. User-agent: Python-urllib
  432. Disallow: /
  433. User-agent: QueryN Metasearch
  434. Disallow: /
  435. User-agent: Radiation Retriever 1.1
  436. Disallow: /
  437. User-agent: RepoMonkey
  438. Disallow: /
  439. User-agent: RepoMonkey Bait & Tackle/v1.01
  440. Disallow: /
  441. User-agent: RMA
  442. Disallow: /
  443. User-agent: searchpreview
  444. Disallow: /
  445. User-agent: SiteSnagger
  446. Disallow: /
  447. User-agent: SpankBot
  448. Disallow: /
  449. User-agent: spanner
  450. Disallow: /
  451. User-agent: SurveyBot
  452. Disallow: /
  453. User-agent: suzuran
  454. Disallow: /
  455. User-agent: Szukacz/1.4
  456. Disallow: /
  457. User-agent: Teleport
  458. Disallow: /
  459. User-agent: TeleportPro
  460. Disallow: /
  461. User-agent: Telesoft
  462. Disallow: /
  463. User-agent: The Intraformant
  464. Disallow: /
  465. User-agent: TheNomad
  466. Disallow: /
  467. User-agent: TightTwatBot
  468. Disallow: /
  469. User-agent: toCrawl/UrlDispatcher
  470. Disallow: /
  471. User-agent: True_Robot
  472. Disallow: /
  473. User-agent: True_Robot/1.0
  474. Disallow: /
  475. User-agent: turingos
  476. Disallow: /
  477. User-agent: TurnitinBot
  478. Disallow: /
  479. User-agent: TurnitinBot/1.5
  480. Disallow: /
  481. User-agent: URL Control
  482. Disallow: /
  483. User-agent: URL_Spider_Pro
  484. Disallow: /
  485. User-agent: URLy Warning
  486. Disallow: /
  487. User-agent: VCI
  488. Disallow: /
  489. User-agent: VCI WebViewer VCI WebViewer Win32
  490. Disallow: /
  491. User-agent: Web Image Collector
  492. Disallow: /
  493. User-agent: WebAuto
  494. Disallow: /
  495. User-agent: WebBandit
  496. Disallow: /
  497. User-agent: WebBandit/3.50
  498. Disallow: /
  499. User-agent: WebCapture 2.0
  500. Disallow: /
  501. User-agent: WebCopier
  502. Disallow: /
  503. User-agent: WebCopier v.2.2
  504. Disallow: /
  505. User-agent: WebCopier v3.2a
  506. Disallow: /
  507. User-agent: WebEnhancer
  508. Disallow: /
  509. User-agent: WebSauger
  510. Disallow: /
  511. User-agent: Website Quester
  512. Disallow: /
  513. User-agent: Webster Pro
  514. Disallow: /
  515. User-agent: WebStripper
  516. Disallow: /
  517. User-agent: WebZip
  518. Disallow: /
  519. User-agent: WebZip/4.0
  520. Disallow: /
  521. User-agent: WebZIP/4.21
  522. Disallow: /
  523. User-agent: WebZIP/5.0
  524. Disallow: /
  525. User-agent: wget
  526. Disallow: /
  527. User-agent: Wget/1.5.3
  528. Disallow: /
  529. User-agent: Wget/1.6
  530. Disallow: /
  531. User-agent: WWW-Collector-E
  532. Disallow: /
  533. User-agent: Xenu's
  534. Disallow: /
  535. User-agent: Xenu's Link Sleuth 1.1c
  536. Disallow: /
  537. User-agent: Zeus
  538. Disallow: /
  539. User-agent: Zeus 32297 Webster Pro V2.9 Win32
  540. Disallow: /
  541. User-agent: Zeus Link Scout
  542. Disallow: /
  543. User-agent: Teoma
  544. Disallow: /
  545. User-agent: MLBot
  546. Disallow: /
  547. User-agent: yacybot
  548. Disallow: /
  549. User-agent: YodaoBot
  550. Disallow: /
  551. User-agent: daumoa
  552. Disallow: /
  553. User-agent: OOZBOT
  554. Disallow: /
  555. User-agent: Charlotte
  556. Disallow: /
  557. User-agent: daumoa
  558. Disallow: /
  559. User-agent: oBot
  560. Disallow: /
  561. User-agent: discobot
  562. Disallow: /
  563. User-agent: EvilRobot
  564. Disallow: /
  565. User-agent: NaverBot
  566. Disallow: /
  567. User-agent: Baiduspider
  568. Disallow: /

非凡网 博客

我只是帮你筛选价值。

你也许喜欢

发表评论