# Following page has details on how to setup a robots.txt file and 
# related security issues:
#   http://www.chami.com/tips/internet/010198I.html
#


# 1. Uncomment following two lines by removing "#" at the beginning to 
# indicate that all robots should stay away from the /cgi-bin/ folder

User-agent: *
Disallow: /Test/
Disallow: /Amazon/
Disallow: /cgi-bin/
Disallow: /cgi-win/
Crawl-delay: 60
#  delay was 20


# 2. Following two lines indicate that robots named "WWW_Robot_Name"
# should stay away from this site.

#User-agent: WWW_Robot_Name
#Disallow: /

# OOZBOT is looking for non-English sites
User-agent: OOZBOT
Disallow: / 

# BuzzTracker is looking for RSS for news, but does not obey ttl (time to live).  
# On On June 22, 2009 BuzzTracker was the most frequent visitor user agent, 
# but total bytes was low.  Always looking for /RSS/Daily-Jigsaw.xml, 
# not robots.txt so putting it here may be fruitless.
User-agent: BuzzTracker
Disallow: / 




# 3. Following two lines indicate that robots named "WWW_Wanderer_Name"
# should not index files in the /private/ folder.

#User-agent: WWW_Wanderer_Name
#Disallow: /private/

# Block panscient robot
User-agent: panscient
Disallow: /