Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
39 lines (27 sloc) 0.597 kb
# This is a demonstration staging server and should not be indexed by search engines.
# The canonical site to crawl is https://data.gov
User-agent: * # match all bots
Disallow: / # keep them out
User-Agent: Googlebot
Disallow: /
User-Agent: Googlebot-Mobile
Disallow: /
User-Agent: msnbot
Disallow: /
User-Agent: bingbot
Disallow: /
# Adsense
User-Agent: Mediapartners-Google
Disallow: /
# Blekko
User-Agent: ScoutJet
Disallow: /
User-Agent: Yandex
Disallow: /
User-agent: baiduspider
Disallow: /
User-agent: DuckDuckBot
Disallow: /
# CommonCrawl
User-agent: ccbot
Disallow: /
Jump to Line
Something went wrong with that request. Please try again.