Build: disable Sphinx manipulation (#11441)

* Build: disable Sphinx manipulation

Add a feature flag called `DISABLE_SPHINX_MANIPULATION` that:

- don't install `readthedocs-sphinx-ext` Python package
- don't append anything to the Sphinx's `conf.py` file
- enable Read the Docs Addons on these versions automatically

The idea behind this is start defaulting new projects to Read the Docs Addons
without breaking old projects.

This is a potential first step in favor of the full deprecation/removal of the
Sphinx manipulation (as we already did for MkDocs). Once this
happens, **building on Read the Docs will produce the exact same result than
building locally**.

Related readthedocs/addons#72

* Build: add `READTHEDOCS_REPOSITORY_PATH` environment variable

This is a useful variable that we will require during the deprecation of the
Sphinx context manipulation.

Besides, it seems a useful variable that we should expose to users anyways so
they can use it as relative to calculate other useful paths.

* Build: add `READTHEDOCS_PRODUCTION_DOMAIN` as environment variable

* Document `READTHEDOCS_PRODUCTION_DOMAIN` env var

* Update tests

docs/user/reference/environment-variables.rst

@@ -9,6 +9,17 @@ All :doc:`build processes </builds>` have the following environment variables au
 
     :Default: ``True``
 
+.. envvar:: READTHEDOCS_PRODUCTION_DOMAIN
+
+    Domain where Read the Docs application/dashboard and API are running.
+
+    :Example: ``readthedocs.org``
+    :Example: ``readthedocs.com``
+    :Example: ``app.readthedocs.org``
+    :Example: ``app.readthedocs.com``
+    :Example: ``devthedocs.org``
+    :Example: ``devthedocs.com``
+
 .. envvar:: READTHEDOCS_PROJECT
 
     The :term:`slug` of the project being built. For example, ``my-example-project``.
@@ -81,6 +92,12 @@ All :doc:`build processes </builds>` have the following environment variables au
     :Example: ``https://docs.readthedocs.io/ja/stable/``
     :Example: ``https://example--17.org.readthedocs.build/fr/17/``
 
+.. envvar:: READTHEDOCS_REPOSITORY_PATH
+
+    Path where the repository was cloned.
+
+    :Example: ``/home/docs/checkouts/readthedocs.org/user_builds/test-builds/checkouts/latest``
+
 .. envvar:: READTHEDOCS_GIT_CLONE_URL
 
     URL for the remote source repository, from which the documentation is cloned.

readthedocs/doc_builder/backends/sphinx.py

@@ -264,20 +264,21 @@ def append_conf(self):
                 },
             )
 
-        # Allow symlinks, but only the ones that resolve inside the base directory.
-        # NOTE: if something goes wrong,
-        # `safe_open` raises an exception that's clearly communicated to the user.
-        outfile = safe_open(
-            self.config_file, "a", allow_symlinks=True, base_path=self.project_path
-        )
+        if not self.project.has_feature(Feature.DISABLE_SPHINX_MANIPULATION):
+            # Allow symlinks, but only the ones that resolve inside the base directory.
+            # NOTE: if something goes wrong,
+            # `safe_open` raises an exception that's clearly communicated to the user.
+            outfile = safe_open(
+                self.config_file, "a", allow_symlinks=True, base_path=self.project_path
+            )
 
-        # Append config to project conf file
-        tmpl = template_loader.get_template("doc_builder/conf.py.tmpl")
-        rendered = tmpl.render(self.get_config_params())
+            # Append config to project conf file
+            tmpl = template_loader.get_template("doc_builder/conf.py.tmpl")
+            rendered = tmpl.render(self.get_config_params())
 
-        with outfile:
-            outfile.write("\n")
-            outfile.write(rendered)
+            with outfile:
+                outfile.write("\n")
+                outfile.write(rendered)
 
         # Print the contents of conf.py in order to make the rendered
         # configfile visible in the build logs

readthedocs/doc_builder/director.py

@@ -25,6 +25,7 @@
 from readthedocs.doc_builder.python_environments import Conda, Virtualenv
 from readthedocs.projects.constants import BUILD_COMMANDS_OUTPUT_PATH_HTML
 from readthedocs.projects.exceptions import RepositoryError
+from readthedocs.projects.models import Feature
 from readthedocs.projects.signals import after_build, before_build, before_vcs
 from readthedocs.storage import build_tools_storage
 
@@ -200,6 +201,10 @@ def build(self):
         self.run_build_job("post_build")
         self.store_readthedocs_build_yaml()
 
+        if self.data.project.has_feature(Feature.DISABLE_SPHINX_MANIPULATION):
+            # Mark this version to inject the new js client when serving it via El Proxito
+            self.data.version.addons = True
+
         after_build.send(
             sender=self.data.version,
         )
@@ -645,6 +650,9 @@ def get_rtd_env_vars(self):
             "READTHEDOCS_VERSION_NAME": self.data.version.verbose_name,
             "READTHEDOCS_PROJECT": self.data.project.slug,
             "READTHEDOCS_LANGUAGE": self.data.project.language,
+            "READTHEDOCS_REPOSITORY_PATH": self.data.project.checkout_path(
+                self.data.version.slug
+            ),
             "READTHEDOCS_OUTPUT": os.path.join(
                 self.data.project.checkout_path(self.data.version.slug), "_readthedocs/"
             ),
@@ -654,6 +662,7 @@ def get_rtd_env_vars(self):
             # "READTHEDOCS_GIT_HTML_URL": self.data.project.remote_repository.html_url,
             "READTHEDOCS_GIT_IDENTIFIER": self.data.version.identifier,
             "READTHEDOCS_GIT_COMMIT_HASH": self.data.build["commit"],
+            "READTHEDOCS_PRODUCTION_DOMAIN": settings.PRODUCTION_DOMAIN,
         }
         return env
 

readthedocs/doc_builder/python_environments.py

@@ -174,12 +174,11 @@ def _install_latest_requirements(self, pip_install_cmd):
         if self.config.doctype == "mkdocs":
             requirements.append("mkdocs")
         else:
-            requirements.extend(
-                [
-                    "sphinx",
-                    "readthedocs-sphinx-ext",
-                ]
-            )
+            requirements.append("sphinx")
+
+            # Install ``readthedocs-sphinx-ext`` only on old projects
+            if not self.project.has_feature(Feature.DISABLE_SPHINX_MANIPULATION):
+                requirements.append("readthedocs-sphinx-ext")
 
         cmd = copy.copy(pip_install_cmd)
         cmd.extend(requirements)

readthedocs/projects/models.py

@@ -1896,6 +1896,7 @@ def add_features(sender, **kwargs):
     DONT_INSTALL_LATEST_PIP = "dont_install_latest_pip"
     USE_SPHINX_RTD_EXT_LATEST = "rtd_sphinx_ext_latest"
     INSTALL_LATEST_CORE_REQUIREMENTS = "install_latest_core_requirements"
+    DISABLE_SPHINX_MANIPULATION = "disable_sphinx_manipulation"
 
     # Search related features
     DISABLE_SERVER_SIDE_SEARCH = "disable_server_side_search"
@@ -1973,6 +1974,12 @@ def add_features(sender, **kwargs):
                 "Build: Install all the latest versions of Read the Docs core requirements"
             ),
         ),
+        (
+            DISABLE_SPHINX_MANIPULATION,
+            _(
+                "Sphinx: Don't append `conf.py` and don't install ``readthedocs-sphinx-ext``"
+            ),
+        ),
         # Search related features.
         (
             DISABLE_SERVER_SIDE_SEARCH,
@@ -2033,7 +2040,7 @@ def get_feature_display(self):
         Because the field is not a ChoiceField here, we need to manually
         implement this behavior.
         """
-        return dict(self.FEATURES).get(self.feature_id, self.feature_id)
+        return str(dict(self.FEATURES).get(self.feature_id, self.feature_id))
 
 
 class EnvironmentVariable(TimeStampedModel, models.Model):

readthedocs/projects/tests/test_build_tasks.py

@@ -361,12 +361,16 @@ def test_get_env_vars(self, load_yaml_config, build_environment, config, externa
             "READTHEDOCS_VERSION_NAME": self.version.verbose_name,
             "READTHEDOCS_PROJECT": self.project.slug,
             "READTHEDOCS_LANGUAGE": self.project.language,
+            "READTHEDOCS_REPOSITORY_PATH": os.path.join(
+                self.project.checkout_path(self.version.slug),
+            ),
             "READTHEDOCS_OUTPUT": os.path.join(
                 self.project.checkout_path(self.version.slug), "_readthedocs/"
             ),
             "READTHEDOCS_GIT_CLONE_URL": self.project.repo,
             "READTHEDOCS_GIT_IDENTIFIER": self.version.identifier,
             "READTHEDOCS_GIT_COMMIT_HASH": self.build.commit,
+            "READTHEDOCS_PRODUCTION_DOMAIN": settings.PRODUCTION_DOMAIN,
         }
 
         self._trigger_update_docs_task()