# Robots.txt for TWiki servers

User-agent: Googlebot
Disallow: /
Allow: /twiki/bin/view
#Allow: /twiki/bin/search
User-agent: gsa-crawler-bnl
Disallow: /
Allow: /twiki/bin/view
User-agent: FAST Enterprise Crawler 6 used by CERN
Disallow: /twiki/bin/attach
Disallow: /twiki/bin/changes
Disallow: /twiki/bin/configure
Disallow: /twiki/bin/edit
Disallow: /twiki/bin/genpdf
Disallow: /twiki/bin/login
Disallow: /twiki/bin/logon
Disallow: /twiki/bin/manage
Disallow: /twiki/bin/natlogon
Disallow: /twiki/bin/natsearch
Disallow: /twiki/bin/oops
Disallow: /twiki/bin/passwd
Disallow: /twiki/bin/preview
Disallow: /twiki/bin/rdiff
Disallow: /twiki/bin/rdiffauth
Disallow: /twiki/bin/register
Disallow: /twiki/bin/rename
Disallow: /twiki/bin/resetpasswd
Disallow: /twiki/bin/rest
Disallow: /twiki/bin/save
Disallow: /twiki/bin/search
Disallow: /twiki/bin/speedyview
Disallow: /twiki/bin/spellerpages
Disallow: /twiki/bin/statistics
Disallow: /twiki/bin/twiki
Disallow: /twiki/bin/upload
Disallow: /twiki/bin/viewauth
Disallow: /twiki/bin/viewfile
Disallow: /twiki/bin/vote

Allow: /twiki/bin/view
#Allow: /twiki/bin/search
#Disallow: /

User-agent: * 
Disallow: /

User-agent: *
Crawl-delay: 60