Merge pull request blockscout#6 from crypto-org-chain/feature/robots-txt

Backport robots.txt changes
planq-network · Jul 27, 2021 · 604acf1 · 604acf1
2 parents 86733e3 + 43612d3
commit 604acf1
Showing 1 changed file with 15 additions and 0 deletions.
diff --git a/apps/block_scout_web/assets/static/robots.txt b/apps/block_scout_web/assets/static/robots.txt
@@ -3,3 +3,18 @@
 # To ban all spiders from the entire site uncomment the next two lines:
 # User-agent: *
 # Disallow: /
+
+User-agent : *
+Disallow: /blocks
+Disallow: /uncles
+Disallow: /reorgs
+Disallow: /txs
+Disallow: /pending-transactions
+Disallow: /accounts
+Disallow: /address
+Disallow: /tokens
+Disallow: /tx
+Disallow: /graphiql
+Disallow: /api-docs
+Disallow: /eth-rpc-api-docs
+Disallow: /search