# exclude help system from robots User-agent: * Crawl-delay: 3 Disallow: /manual/ Disallow: /doc/ Disallow: /gif/ Sitemap: http://www.shinchiku-athome.jp/sitemap.xml # but allow htdig to index our doc-tree User-agent: susedig Disallow: # disallow stress test user-agent: stress-agent Disallow: /