#Teoma User-agent: Teoma Crawl-delay: 10 #Yahoo User-agent: Slurp Crawl-delay: 10 # AllTheWeb User-agent: fast-webcrawler Crawl-Delay: 10 Disallow: /inc/ Disallow: /images/ Disallow: /cgi-bin/ # AltaVista User-agent: scooter Disallow: /inc/ Disallow: /images/ Disallow: /cgi-bin/ # Googlebot User-agent: googlebot Crawl-Delay: 10 Disallow: /inc/ Disallow: /images/ Disallow: /cgi-bin/ # Looksmart User-agent: wisenutbot Crawl-Delay: 10 Disallow: /inc/ Disallow: /images/ Disallow: /cgi-bin/ User-agent: zyborg Crawl-Delay: 10 Disallow: /inc/ Disallow: /images/ Disallow: /cgi-bin/ # MSN User-agent: msnbot Crawl-Delay: 10 Disallow: /inc/ Disallow: /images/ Disallow: /cgi-bin/ # Yahoo Vertical Crawler User-agent: yahoo-verticalcrawler Crawl-Delay: 10 Disallow: /inc/ Disallow: /images/ Disallow: /cgi-bin/ # Exclude every other bot (hopefully) User-agent: * Disallow: /