ci: Only allow index, follow for production

.github/workflows/deploy.yaml

@@ -31,6 +31,7 @@ jobs:
         if: github.ref == 'refs/heads/master'
         run: |
           echo "DOCS_BASE_URL=https://docs.hypernode.com/" >> $GITHUB_ENV
+          echo "DOCS_INDEX_FOLLOW=1" >> $GITHUB_ENV
       - run: hypernode-deploy build -vvv
       - name: archive production artifacts
         uses: actions/upload-artifact@v3

docs/_templates/layout.html

@@ -20,6 +20,9 @@
   {{- metatags }}
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
   <meta name="theme-color" content="#063b67" />
+  {%- if meta_robots %}
+  <meta name="robots" content="{{ meta_robots }}">
+  {%- endif %}
   {%- block htmltitle %}
   <title>{{ title|striptags|e }}{{ titlesuffix }}</title>
   {%- endblock -%}

docs/conf.py

@@ -44,6 +44,7 @@
     "sphinx_copybutton",
     "notfound.extension",
     "hypernode.sphinx.extensions.updated_at",
+    "hypernode.sphinx.extensions.meta_robots",
     "sphinxcontrib.mermaid",
 ]
 
@@ -80,6 +81,9 @@
 
 sitemap_url_scheme = "{link}"
 
+if os.getenv("DOCS_INDEX_FOLLOW", 0):
+    html_meta_robots = "index, follow"
+
 if os.getenv("DOCS_BASE_URL"):
     html_baseurl = os.getenv("DOCS_BASE_URL")
     extensions.append("sphinx_sitemap")  # Only generate sitemap when we have a base url

hypernode/sphinx/extensions/meta_robots.py

@@ -0,0 +1,20 @@
+from sphinx.application import Sphinx
+
+
+def page_context_handler(app: Sphinx, pagename: str, templatename, context, doctree):
+    context["meta_robots"] = app.config.html_meta_robots
+
+
+def setup_sphinx(app: Sphinx):
+    app.add_config_value("html_meta_robots", "noindex, nofollow", "html")
+    app.connect("html-page-context", page_context_handler)
+
+    return {
+        "version": "0.1",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }
+
+
+def setup(app: Sphinx):
+    return setup_sphinx(app)

requirements/development.txt

@@ -5,7 +5,7 @@ pre-commit==2.18.1
 black==22.10.0
 pytest==7.1.2
 pytest-xdist==2.5.0
-mypy==0.950
+mypy==1.0.0
 flake8==3.9.2
 tox==3.25.0
 inotify==0.2.10

tests/sphinx/extensions/test_meta_robots.py

@@ -0,0 +1,44 @@
+from unittest.mock import Mock
+
+from sphinx.application import Sphinx
+from sphinx.config import Config
+
+from hypernode.sphinx.extensions.meta_robots import page_context_handler, setup_sphinx
+from tests.testcase import HypernodeTestCase
+
+
+class TestPageContextHandler(HypernodeTestCase):
+    def setUp(self) -> None:
+        self.app = Mock(spec=Sphinx)
+        self.app.config = Config({})
+        self.app.config["html_meta_robots"] = "noindex, nofollow"
+        self.context: dict = {}
+
+    def test_page_context_handler_sets_noindex_nofollow(self):
+        page_context_handler(self.app, "", "", self.context, None)
+
+        self.assertIn("meta_robots", self.context)
+        self.assertEqual("noindex, nofollow", self.context["meta_robots"])
+
+    def test_page_context_handler_sets_index_follow(self):
+        self.app.config.html_meta_robots = "index, follow"
+
+        page_context_handler(self.app, "", "", self.context, None)
+
+        self.assertIn("meta_robots", self.context)
+        self.assertEqual("index, follow", self.context["meta_robots"])
+
+
+class TestSetupSphinx(HypernodeTestCase):
+    def setUp(self) -> None:
+        self.app = Mock(spec=Sphinx)
+
+    def test_setup_adds_config_value(self):
+        setup_sphinx(self.app)
+
+        self.app.add_config_value.assert_called_once_with(
+            "html_meta_robots", "noindex, nofollow", "html"
+        )
+        self.app.connect.assert_called_once_with(
+            "html-page-context", page_context_handler
+        )