Add robots.txt

flake.nix

@@ -37,6 +37,12 @@
             }
           );
 
+          sphinx-sitemap = super.sphinx-sitemap.overridePythonAttrs (
+            old: {
+              buildInputs = (old.buildInputs or [ ]) ++ [ super.setuptools ];
+            }
+          );
+
         };
       in
       rec {

pyproject.toml

@@ -14,6 +14,7 @@ sphinx = "*"
 sphinx-book-theme = "^1.0.0"
 sphinx-copybutton = "*"
 sphinx-design = "^0.3.0"
+sphinx-sitemap = "^2.5.0"
 
 [tool.poetry.dev-dependencies]
 black = "==22.8.0"

source/conf.py

@@ -42,6 +42,7 @@
     "sphinx_copybutton",
     "sphinx_design",
     "extractable_code_block",
+    "sphinx_sitemap",
 ]
 
 # Add myst-specific extensions, see what is available on
@@ -135,6 +136,8 @@
 
 # -- Options for HTML output ----------------------------------------------
 
+html_baseurl = "https://nix.dev"
+
 html_theme = "sphinx_book_theme"
 
 # https://alabaster.readthedocs.io/en/latest/customization.html
@@ -174,7 +177,7 @@
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.
-# html_extra_path = []
+html_extra_path = ["robots.txt"]
 
 # If not None, a 'Last updated on:' timestamp is inserted at every page
 # bottom, using the given strftime format.

source/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Disallow: /_static/
+Disallow: /_sources/
+Sitemap: https://nix.dev/sitemap.xml