Disallow auth callbacks in robots.txt

discourse · Jan 14, 2014 · 88d9f3a · 88d9f3a
1 parent 2a89f51
commit 88d9f3a
Showing 1 changed file with 9 additions and 0 deletions.
diff --git a/app/views/robots_txt/index.erb b/app/views/robots_txt/index.erb
@@ -1,2 +1,11 @@
 # See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
 #
+User-agent: *
+Disallow: /auth/cas
+Disallow: /auth/facebook/callback
+Disallow: /auth/twitter/callback
+Disallow: /auth/google/callback
+Disallow: /auth/yahoo/callback
+Disallow: /auth/github/callback
+Disallow: /auth/persona/callback
+Disallow: /auth/cas/callback