Add custom robots.txt to docs

docs/conf.py

@@ -163,7 +163,7 @@
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.
-# html_extra_path = []
+html_extra_path = ['robots.txt']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.

docs/robots.txt

@@ -0,0 +1,8 @@
+User-agent: *
+
+# Only index `/en/stable/`
+Disallow: /en/v*/
+Disallow: /en/latest/
+Disallow: /*/pdf/
+
+Sitemap: https://docs.wagtail.io/sitemap.xml