User-Agent: AbachoBOT # www.abacho.com... srv-ze-robot1.tricus.com Disallow: / User-Agent: abcdatos_botlink # www.abcdatos.com, 217.126.39.167 Disallow: / User-Agent: Acoon Robot # www.acoon.de, 194.231.42.178 Disallow: / User-Agent: BaiDuSpider # ... Chinese spider search bot... 61.135.145.205... it does NOT read robots.txt... Disallow: / User-Agent: Kototoi/0.1 # www.s.u-tokyo.ac.jp, crawler-red3.is.s.u-tokyo.ac.jp Disallow: / User-Agent: moget/2.0 # www.goo.ne.jp, 202.229.31.13 Disallow: / User-Agent: Mole2/1.0 # www.intags.de, 217.160.75.10 Disallow: / User-Agent: MP3Bot # http://mp3bot.de/, <..> Disallow: / User-Agent: MSNBOT/0.1 # http://search.msn.com/, 131.107.163.47 Disallow: /ads/ Disallow: /buttons/ Disallow: /forms/ Disallow: /images/ Disallow: /inc/ Disallow: /newsite/ Disallow: /oldsite/ Disallow: /Reports/ Disallow: /toolbox/ Disallow: /VRL-TestArea/ Disallow: /app-atmcard.asp Disallow: /app-deposit.asp Disallow: /app-estatements.asp Disallow: /app-hsa.asp Disallow: /app-loan-mortgage.asp Disallow: /app-loan-personal.asp Disallow: /app-onlineACHODT.asp Disallow: /apps.js Disallow: /app-VRLtest.asp Disallow: /basic.css Disallow: /contactus.asp Disallow: /dlt-this-calcs-old.asp Disallow: /template.asp Disallow: /template2.asp Disallow: /thanks.asp User-Agent: myweb # robot-name: Internet Shinchakubin, robot-cover-url: http://naragw.sharp.co.jp/myweb/home/ Disallow: / User-Agent: nttdirectory_robot # http://navi.ocn.ne.jp, lilis00.navi.ocn.ne.jp, lilis04.navi.ocn.ne.jp Disallow: / User-Agent: psbot # http://www.picsearch.com/bot.html User-Agent: QweeryBot # www.qweery.nl, 84.82.133.41 Disallow: / User-Agent: Stackambler/ # www.rambler.ru, 81.222.64.10 Disallow: / User-Agent: SeznamBot # www.seznam.cz, 212.80.76.87 Disallow: / User-Agent: Slurp # Yahoo (?) Crawl-delay: 1.2 # ...see http://help.yahoo.com/l/us/yahoo/search/webcrawler/slurp-03.html User-Agent: Slurp.so/1.0 # (inktomi) Disallow: / User-Agent: Slurp/2.0j # (inktomi) Disallow: / User-Agent: Slurp/2.0-KiteHourly # (inktomi) Disallow: / User-Agent: Slurp/2.0-OwlWeekly # (inktomi) Disallow: / User-Agent: Slurp/3.0-AU # (inktomi) Disallow: / User-Agent: Speedy Spider # http://www.entireweb.com/speedy.html Disallow: / User-Agent: speedfind ramBot xtreme # www.speedfind.de, BWEB.highway.telekom.at Disallow: / User-Agent: UK Searcher Spider # www.uksearcher.co.uk Disallow: / User-Agent: VWbot # MetaQuerier crawler... part of the MetaQuerier project @ the Univ. of Illinois in Urbana-Champaign Disallow: / # see http://vwbot.cs.uiuc.edu/ User-Agent: www.WebWombat.com.au # www.webwombat.com.au, 202.139.99.131 Disallow: / # #-------------------------------------------------------------------------------------------------------- # User-Agent: * # matches all bots... remainder lists dirs & pgs not allowed to all bots... Disallow: /ads/ Disallow: /buttons/ Disallow: /forms/ Disallow: /images/ Disallow: /inc/ Disallow: /newsite/ Disallow: /oldsite/ Disallow: /Reports/ Disallow: /toolbox/ Disallow: /VRL-TestArea/ Disallow: /app-atmcard.asp Disallow: /app-deposit.asp Disallow: /app-estatements.asp Disallow: /app-hsa.asp Disallow: /app-loan-mortgage.asp Disallow: /app-loan-personal.asp Disallow: /app-onlineACHODT.asp Disallow: /apps.js Disallow: /app-VRLtest.asp Disallow: /basic.css Disallow: /contactus.asp Disallow: /dlt-this-calcs-old.asp Disallow: /template.asp Disallow: /template2.asp Disallow: /thanks.asp