# This says to apply these settings to ALL search engine spiders/crawlers
User-agent: *
# Stop spiders from indexing unwanted pages
Disallow: /cache/
Disallow: /cgi-bin/
Disallow: /docs/
Disallow: /download/
Disallow: /editors/
Disallow: /email/
Disallow: /extras/
Disallow: /feed/
Disallow: /googlecheckout/
Disallow: /includes/
Disallow: /media/
Disallow: /pub/
Sitemap: http://thebigrugstore.co.uk/sitemapindex.xml