From 3759a0ae967d94698fc00f886d84f2176e58f690 Mon Sep 17 00:00:00 2001 From: Nolan Lawson Date: Sat, 8 Apr 2017 16:20:24 -0700 Subject: [PATCH] block all robots in robots.txt (#1) --- public/robots.txt | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/public/robots.txt b/public/robots.txt index 6672eeba1f22d..1f53798bb4fe3 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,8 +1,2 @@ -# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file - -User-agent: GPTBot -Disallow: / - User-agent: * -Disallow: /media_proxy/ -Disallow: /interact/ +Disallow: /