#robots.txt file for webcrawlers User-agent: * Disallow: /etc/ Disallow: /img/ Disallow: /Music/ Disallow: /pics/ Disallow: /stats/