Add custom robots.txt to docs

wagtail · Nov 12, 2020 · 4702b6f · 4702b6f
1 parent be45cee
commit 4702b6f
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 1 deletion.
diff --git a/docs/conf.py b/docs/conf.py
@@ -163,7 +163,7 @@
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.
-# html_extra_path = []
+html_extra_path = ['robots.txt']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.

diff --git a/docs/robots.txt b/docs/robots.txt
@@ -0,0 +1,8 @@
+User-agent: *
+
+# Only index `/en/stable/`
+Disallow: /en/v*/
+Disallow: /en/latest/
+Disallow: /*/pdf/
+
+Sitemap: https://docs.wagtail.io/sitemap.xml