# exclude help system from robots User-agent: * #Disallow: /manual/ #Disallow: /doc/ #Disallow: /shop.html Disallow: /cgi-bin/ Disallow: /gastbuch.htm Disallow: /guest/ Disallow: *.cgi # but allow htdig to index our doc-tree #User-agent: susedig #Disallow: