Use a specific venv for specific sphinx versions.

README.md

@@ -1,12 +1,16 @@
 This repository contains scripts for automatically building the Python
 documentation on [docs.python.org](https://docs.python.org).
 
+
 # How to test it?
 
-    $ mkdir -p www logs build_root
-    $ python3 -m venv build_root/venv/
-    $ build_root/venv/bin/python -m pip install -r requirements.txt
-    $ python3 ./build_docs.py --quick --build-root build_root --www-root www --log-directory logs --group $(id -g) --skip-cache-invalidation
+The following command should build all maintained versions and
+translations in ``./www``, beware it can take a few hours:
+
+    $ python3 ./build_docs.py --quick --build-root ./build_root --www-root ./www --log-directory ./logs --group $(id -g) --skip-cache-invalidation
+
+If you don't need to build all translations of all branches, add
+``--language en --branch master``.
 
 
 # Check current version

build_docs.py

@@ -58,7 +58,7 @@
     sentry_sdk.init()
 
 VERSION = "19.0"
-
+DEFAULT_SPHINX_VERSION = "2.3.1"
 
 if not hasattr(shlex, "join"):
     # Add shlex.join if missing (pre 3.8)
@@ -70,14 +70,15 @@
 class Version:
     STATUSES = {"EOL", "security-fixes", "stable", "pre-release", "in development"}
 
-    def __init__(self, name, branch, status):
+    def __init__(self, name, branch, status, sphinx_version=DEFAULT_SPHINX_VERSION):
         if status not in self.STATUSES:
             raise ValueError(
                 "Version status expected to be in {}".format(", ".join(self.STATUSES))
             )
         self.name = name
         self.branch = branch
         self.status = status
+        self.sphinx_version = sphinx_version
 
     @property
     def url(self):
@@ -93,11 +94,13 @@ def title(self):
 )
 
 # EOL and security-fixes are not automatically built, no need to remove them
-# from the list.
+# from the list, this way we can still rebuild them manually as needed.
+# Please pin the sphinx_versions of EOL and security-fixes, as we're not maintaining
+# their doc, they don't follow Sphinx deprecations.
 VERSIONS = [
-    Version("2.7", "2.7", "EOL"),
-    Version("3.5", "3.5", "security-fixes"),
-    Version("3.6", "3.6", "security-fixes"),
+    Version("2.7", "2.7", "EOL", sphinx_version="2.3.1"),
+    Version("3.5", "3.5", "security-fixes", sphinx_version="1.8.4"),
+    Version("3.6", "3.6", "security-fixes", sphinx_version="2.3.1"),
     Version("3.7", "3.7", "stable"),
     Version("3.8", "3.8", "stable"),
     Version("3.9", "3.9", "pre-release"),
@@ -455,6 +458,25 @@ def build_one(
     logging.info("Build done for version: %s, language: %s", version.name, language.tag)
 
 
+def build_venv(build_root, version):
+    """Build a venv for the specific version.
+    This is used to pin old Sphinx versions to old cpython branches.
+    """
+    requirements = [
+        "blurb",
+        "jieba",
+        "python-docs-theme",
+        "sphinx=={}".format(version.sphinx_version),
+    ]
+    venv_path = os.path.join(build_root, "venv-with-sphinx-" + version.sphinx_version)
+    shell_out(["python3", "-m", "venv", venv_path])
+    shell_out(
+        [os.path.join(venv_path, "bin", "python"), "-m", "pip", "install"]
+        + requirements
+    )
+    return venv_path
+
+
 def copy_build_to_webroot(
     build_root,
     version,
@@ -469,6 +491,7 @@ def copy_build_to_webroot(
     logging.info(
         "Publishing start for version: %s, language: %s", version.name, language.tag
     )
+    Path(www_root).mkdir(parents=True, exist_ok=True)
     checkout = os.path.join(
         build_root, version.name, "cpython-{lang}".format(lang=language.tag)
     )
@@ -670,6 +693,7 @@ def setup_logging(log_directory):
     if sys.stderr.isatty():
         logging.basicConfig(format="%(levelname)s:%(message)s", stream=sys.stderr)
     else:
+        Path(log_directory).mkdir(parents=True, exist_ok=True)
         handler = logging.handlers.WatchedFileHandler(
             os.path.join(log_directory, "docsbuild.log")
         )
@@ -691,7 +715,6 @@ def main():
     if args.www_root:
         args.www_root = os.path.abspath(args.www_root)
     setup_logging(args.log_directory)
-    venv = os.path.join(args.build_root, "venv")
     if args.branch:
         versions_to_build = [
             version
@@ -720,6 +743,7 @@ def main():
                     scope.set_tag("version", version.name)
                     scope.set_tag("language", language.tag)
             try:
+                venv = build_venv(args.build_root, version)
                 build_one(
                     version,
                     args.quick,

requirements.in

@@ -1,6 +1 @@
-blurb
-jieba
-python-docs-theme
-requests
 sentry-sdk
-sphinx==2.3.1

requirements.txt

@@ -4,34 +4,6 @@
 #
 #    pip-compile requirements.in
 #
-alabaster==0.7.12         # via sphinx
-babel==2.7.0              # via sphinx
-blurb==1.0.7              # via -r requirements.in (line 1)
-certifi==2019.11.28       # via requests, sentry-sdk
-chardet==3.0.4            # via requests
-docutils==0.15.2          # via sphinx
-idna==2.8                 # via requests
-imagesize==1.1.0          # via sphinx
-jieba==0.42.1             # via -r requirements.in (line 2)
-jinja2==2.10.3            # via sphinx
-markupsafe==1.1.1         # via jinja2
-packaging==19.2           # via sphinx
-pygments==2.5.2           # via sphinx
-pyparsing==2.4.5          # via packaging
-python-docs-theme==2020.1  # via -r requirements.in (line 3)
-pytz==2019.3              # via babel
-requests==2.22.0          # via -r requirements.in (line 4), sphinx
-sentry-sdk==0.13.5        # via -r requirements.in (line 5)
-six==1.13.0               # via packaging
-snowballstemmer==2.0.0    # via sphinx
-sphinx==2.3.1             # via -r requirements.in (line 6)
-sphinxcontrib-applehelp==1.0.1  # via sphinx
-sphinxcontrib-devhelp==1.0.1  # via sphinx
-sphinxcontrib-htmlhelp==1.0.2  # via sphinx
-sphinxcontrib-jsmath==1.0.1  # via sphinx
-sphinxcontrib-qthelp==1.0.2  # via sphinx
-sphinxcontrib-serializinghtml==1.1.3  # via sphinx
-urllib3==1.25.7           # via requests, sentry-sdk
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
+certifi==2020.6.20        # via sentry-sdk
+sentry-sdk==0.15.1        # via -r requirements.in
+urllib3==1.25.9           # via sentry-sdk