User-agent: *
Disallow: /admin/
Disallow: /product_reviews_write.php
Disallow: /product_reviews_write.html
Disallow: /tell_a_friend.php
Disallow: /create_account.php
Disallow: /images/
Disallow: /login.php
Disallow: /login.html
Disallow: /account.php
Disallow: /popup_image.php
Disallow: /shopping_cart.php
Disallow: /catalog/download/
Disallow: /catalog/account.php
Disallow: /catalog/shopping_cart.php
Disallow: /catalog/checkout_shipping.php
Disallow: /catalog/advanced_search.php
Disallow: /catalog/product_review.php
Disallow: /catalog/product_reviews_write.php
Disallow: /catalog/cookie_usage.php
Disallow: /catalog/password_forgotten.php
Disallow: /catalog/popup_image.php


# User-agent: Miva (AlgoFeedback@miva.com)
# Disallow: /
# User-agent: r7.law.dsi.unimi.it
# Disallow: /
User-agent: UbiCrawler
Disallow: /

crawl
slurp
spider
ebot
obot
abot
dbot
hbot
kbot
mbot
nbot
pbot
rbot
sbot
tbot
ybot
bot.
accoona
appie
architext
aspseek
asterias
atlocal
atomz
augurfind
bannana_bot
boitho
booch
cfetch
dmoz
docomo
falcon
findlinks
gazz
goforit
grub
gulliver
harvest
helix
holmes
homer
ia_archiver
ichiro
iconsurf
iltrovatore
indexer
infoseek
ingrid
ivia
java/
jetbot
kit_fireball
knowledge
lachesis
larbin
libwww
linkwalker
lwp
mantraagent
mediapartners
mercator
miva
mj12
mnogo
moget/
multitext
muscatferret
myweb
nameprotect
ncsa beta
netmechanic
netresearchserver
ng/
npbot
nutch
objectssearch
omni
osis-project
pear.
pompos
poppelsdorf
rambler
sbider
scooter
scrubby
seeker
shopwiki
sidewinder
smartwit
sna-
sohu
spyder
steeler/
sygol
szukacz
t-h-u-n-d-e-r-s-t-o-n-e
/teoma
tutorgig
ultraseek
vagabondo
volcano
voyager/
w3c_validator
websitepulse
wget
worldlight
worm
zao/
xenu
zippp
zyborg
....
! 




spiders.txt Contribution version 2005-09-16