# All robots will spider the domain except naughty bots

# Disallow BadBots
User-agent: ia_archiver
Disallow: /
#
User-agent: Teleport Pro
Disallow: /

# Disallow directories
User-agent: *
Disallow: /bot-trap/
Disallow: /cgi-bin/
Disallow: /email-addresses/
Disallow: /fullimages/

# Disallow All robots from certain things...
User-agent: *
Disallow: *.cgi
Disallow: *.jpg
Disallow: *.gif
Disallow: *.swf