#Default for missing robots.txt User-agent: * Disallow: /updates/ Disallow: /cgi-bin/ Crawl-delay: 20