# # /robots.txt # # Webcrawlers (search engines) # should NOT index these folders: # User-agent: * Disallow: /cgi-bin/ # Internet Archive User-agent: ia_archiver Disallow: /