# robots.txt # # Configuration file for controlling robots on a web server # # What robot is this rule for? User-agent: * # All robots Crawl-delay: 10 # What files to restrict access? Disallow: / # All files