From b51916a87ebb2521b115182b9cb312d9e8cd19b9 Mon Sep 17 00:00:00 2001
From: Julien Palard <julien@palard.fr>
Date: Fri, 19 Jun 2020 16:03:04 +0200
Subject: [PATCH 1/4] Use a specific venv for specific sphinx versions.

---
 build_docs.py | 34 +++++++++++++++++++++++++++++++---
 1 file changed, 31 insertions(+), 3 deletions(-)

diff --git a/build_docs.py b/build_docs.py
index e56b89f..5774ee5 100755
--- a/build_docs.py
+++ b/build_docs.py
@@ -70,7 +70,7 @@
 class Version:
     STATUSES = {"EOL", "security-fixes", "stable", "pre-release", "in development"}
 
-    def __init__(self, name, branch, status):
+    def __init__(self, name, branch, status, sphinx_version=None):
         if status not in self.STATUSES:
             raise ValueError(
                 "Version status expected to be in {}".format(", ".join(self.STATUSES))
@@ -78,6 +78,7 @@ def __init__(self, name, branch, status):
         self.name = name
         self.branch = branch
         self.status = status
+        self.sphinx_version = sphinx_version
 
     @property
     def url(self):
@@ -96,7 +97,7 @@ def title(self):
 # from the list.
 VERSIONS = [
     Version("2.7", "2.7", "EOL"),
-    Version("3.5", "3.5", "security-fixes"),
+    Version("3.5", "3.5", "security-fixes", sphinx_version="1.8.4"),
     Version("3.6", "3.6", "security-fixes"),
     Version("3.7", "3.7", "stable"),
     Version("3.8", "3.8", "stable"),
@@ -455,6 +456,29 @@ def build_one(
     logging.info("Build done for version: %s, language: %s", version.name, language.tag)
 
 
+def build_venv(build_root, version):
+    """Build a venv for the specific version.
+    This is used to pin old Sphinx versions to old cpython branches.
+    """
+    requirements = ["blurb", "jieba", "python-docs-theme"]
+    venv_path = os.path.join(build_root, "venv-" + version.sphinx_version)
+    shell_out(["python3", "-m", "venv", venv_path])
+    shell_out(
+        [os.path.join(venv_path, "bin", "python"), "-m", "pip", "install"]
+        + requirements
+    )
+    shell_out(
+        [
+            os.path.join(venv_path, "bin", "python"),
+            "-m",
+            "pip",
+            "install",
+            "sphinx=={}".format(version.sphinx_version),
+        ]
+    )
+    return venv_path
+
+
 def copy_build_to_webroot(
     build_root,
     version,
@@ -691,7 +715,7 @@ def main():
     if args.www_root:
         args.www_root = os.path.abspath(args.www_root)
     setup_logging(args.log_directory)
-    venv = os.path.join(args.build_root, "venv")
+    default_venv = os.path.join(args.build_root, "venv")
     if args.branch:
         versions_to_build = [
             version
@@ -720,6 +744,10 @@ def main():
                     scope.set_tag("version", version.name)
                     scope.set_tag("language", language.tag)
             try:
+                if version.sphinx_version:
+                    venv = build_venv(args.build_root, version)
+                else:
+                    venv = default_venv
                 build_one(
                     version,
                     args.quick,

From bbd03b0bf4284c13d688fe73a4a881a5af789333 Mon Sep 17 00:00:00 2001
From: Julien Palard <julien@palard.fr>
Date: Tue, 23 Jun 2020 00:08:45 +0200
Subject: [PATCH 2/4] Split docsbuild requirements and build requirements.

---
 README.md        | 12 ++++++++----
 build_docs.py    | 21 +++++++++++----------
 requirements.in  |  5 -----
 requirements.txt | 34 +++-------------------------------
 4 files changed, 22 insertions(+), 50 deletions(-)

diff --git a/README.md b/README.md
index 1fcbb15..c08882e 100644
--- a/README.md
+++ b/README.md
@@ -1,12 +1,16 @@
 This repository contains scripts for automatically building the Python
 documentation on [docs.python.org](https://docs.python.org).
 
+
 # How to test it?
 
-    $ mkdir -p www logs build_root
-    $ python3 -m venv build_root/venv/
-    $ build_root/venv/bin/python -m pip install -r requirements.txt
-    $ python3 ./build_docs.py --quick --build-root build_root --www-root www --log-directory logs --group $(id -g) --skip-cache-invalidation
+The following command should build all maintained versions and
+translations in ``./www``, beware it can take a few hours:
+
+    $ python3 ./build_docs.py --quick --build-root ./build_root --www-root ./www --log-directory ./logs --group $(id -g) --skip-cache-invalidation
+
+If you don't need to build all translations of all branches, add
+``--language en --branch master``.
 
 
 # Check current version
diff --git a/build_docs.py b/build_docs.py
index 5774ee5..46ec326 100755
--- a/build_docs.py
+++ b/build_docs.py
@@ -58,7 +58,7 @@
     sentry_sdk.init()
 
 VERSION = "19.0"
-
+DEFAULT_SPHINX_VERSION = "2.3.1"
 
 if not hasattr(shlex, "join"):
     # Add shlex.join if missing (pre 3.8)
@@ -70,7 +70,7 @@
 class Version:
     STATUSES = {"EOL", "security-fixes", "stable", "pre-release", "in development"}
 
-    def __init__(self, name, branch, status, sphinx_version=None):
+    def __init__(self, name, branch, status, sphinx_version=DEFAULT_SPHINX_VERSION):
         if status not in self.STATUSES:
             raise ValueError(
                 "Version status expected to be in {}".format(", ".join(self.STATUSES))
@@ -94,11 +94,13 @@ def title(self):
 )
 
 # EOL and security-fixes are not automatically built, no need to remove them
-# from the list.
+# from the list, this way we can still rebuild them manually as needed.
+# Please pin the sphinx_versions of EOL and security-fixes, as we're not maintaining
+# their doc, they don't follow Sphinx deprecations.
 VERSIONS = [
-    Version("2.7", "2.7", "EOL"),
+    Version("2.7", "2.7", "EOL", sphinx_version="2.3.1"),
     Version("3.5", "3.5", "security-fixes", sphinx_version="1.8.4"),
-    Version("3.6", "3.6", "security-fixes"),
+    Version("3.6", "3.6", "security-fixes", sphinx_version="2.3.1"),
     Version("3.7", "3.7", "stable"),
     Version("3.8", "3.8", "stable"),
     Version("3.9", "3.9", "pre-release"),
@@ -461,7 +463,7 @@ def build_venv(build_root, version):
     This is used to pin old Sphinx versions to old cpython branches.
     """
     requirements = ["blurb", "jieba", "python-docs-theme"]
-    venv_path = os.path.join(build_root, "venv-" + version.sphinx_version)
+    venv_path = os.path.join(build_root, "venv-with-sphinx-" + version.sphinx_version)
     shell_out(["python3", "-m", "venv", venv_path])
     shell_out(
         [os.path.join(venv_path, "bin", "python"), "-m", "pip", "install"]
@@ -493,6 +495,7 @@ def copy_build_to_webroot(
     logging.info(
         "Publishing start for version: %s, language: %s", version.name, language.tag
     )
+    Path(www_root).mkdir(parents=True, exist_ok=True)
     checkout = os.path.join(
         build_root, version.name, "cpython-{lang}".format(lang=language.tag)
     )
@@ -694,6 +697,7 @@ def setup_logging(log_directory):
     if sys.stderr.isatty():
         logging.basicConfig(format="%(levelname)s:%(message)s", stream=sys.stderr)
     else:
+        Path(log_directory).mkdir(parents=True, exist_ok=True)
         handler = logging.handlers.WatchedFileHandler(
             os.path.join(log_directory, "docsbuild.log")
         )
@@ -744,10 +748,7 @@ def main():
                     scope.set_tag("version", version.name)
                     scope.set_tag("language", language.tag)
             try:
-                if version.sphinx_version:
-                    venv = build_venv(args.build_root, version)
-                else:
-                    venv = default_venv
+                venv = build_venv(args.build_root, version)
                 build_one(
                     version,
                     args.quick,
diff --git a/requirements.in b/requirements.in
index 8cd884d..9898221 100644
--- a/requirements.in
+++ b/requirements.in
@@ -1,6 +1 @@
-blurb
-jieba
-python-docs-theme
-requests
 sentry-sdk
-sphinx==2.3.1
diff --git a/requirements.txt b/requirements.txt
index 5c35b03..dbe8732 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,34 +4,6 @@
 #
 #    pip-compile requirements.in
 #
-alabaster==0.7.12         # via sphinx
-babel==2.7.0              # via sphinx
-blurb==1.0.7              # via -r requirements.in (line 1)
-certifi==2019.11.28       # via requests, sentry-sdk
-chardet==3.0.4            # via requests
-docutils==0.15.2          # via sphinx
-idna==2.8                 # via requests
-imagesize==1.1.0          # via sphinx
-jieba==0.42.1             # via -r requirements.in (line 2)
-jinja2==2.10.3            # via sphinx
-markupsafe==1.1.1         # via jinja2
-packaging==19.2           # via sphinx
-pygments==2.5.2           # via sphinx
-pyparsing==2.4.5          # via packaging
-python-docs-theme==2020.1  # via -r requirements.in (line 3)
-pytz==2019.3              # via babel
-requests==2.22.0          # via -r requirements.in (line 4), sphinx
-sentry-sdk==0.13.5        # via -r requirements.in (line 5)
-six==1.13.0               # via packaging
-snowballstemmer==2.0.0    # via sphinx
-sphinx==2.3.1             # via -r requirements.in (line 6)
-sphinxcontrib-applehelp==1.0.1  # via sphinx
-sphinxcontrib-devhelp==1.0.1  # via sphinx
-sphinxcontrib-htmlhelp==1.0.2  # via sphinx
-sphinxcontrib-jsmath==1.0.1  # via sphinx
-sphinxcontrib-qthelp==1.0.2  # via sphinx
-sphinxcontrib-serializinghtml==1.1.3  # via sphinx
-urllib3==1.25.7           # via requests, sentry-sdk
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
+certifi==2020.6.20        # via sentry-sdk
+sentry-sdk==0.15.1        # via -r requirements.in
+urllib3==1.25.9           # via sentry-sdk

From cf3fd0925bcee943b3a4754a75aa099dcc3ec731 Mon Sep 17 00:00:00 2001
From: Julien Palard <julien@palard.fr>
Date: Tue, 23 Jun 2020 00:53:48 +0200
Subject: [PATCH 3/4] Does not needs two pip install.

---
 build_docs.py | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/build_docs.py b/build_docs.py
index 46ec326..9d6abd1 100755
--- a/build_docs.py
+++ b/build_docs.py
@@ -462,22 +462,18 @@ def build_venv(build_root, version):
     """Build a venv for the specific version.
     This is used to pin old Sphinx versions to old cpython branches.
     """
-    requirements = ["blurb", "jieba", "python-docs-theme"]
+    requirements = [
+        "blurb",
+        "jieba",
+        "python-docs-theme",
+        "sphinx=={}".format(version.sphinx_version),
+    ]
     venv_path = os.path.join(build_root, "venv-with-sphinx-" + version.sphinx_version)
     shell_out(["python3", "-m", "venv", venv_path])
     shell_out(
         [os.path.join(venv_path, "bin", "python"), "-m", "pip", "install"]
         + requirements
     )
-    shell_out(
-        [
-            os.path.join(venv_path, "bin", "python"),
-            "-m",
-            "pip",
-            "install",
-            "sphinx=={}".format(version.sphinx_version),
-        ]
-    )
     return venv_path
 
 

From 17624eab42d917d688b2a3d2df2bfcc86b98a3e7 Mon Sep 17 00:00:00 2001
From: Julien Palard <julien@palard.fr>
Date: Tue, 23 Jun 2020 00:56:21 +0200
Subject: [PATCH 4/4] unused

---
 build_docs.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/build_docs.py b/build_docs.py
index 9d6abd1..3096537 100755
--- a/build_docs.py
+++ b/build_docs.py
@@ -715,7 +715,6 @@ def main():
     if args.www_root:
         args.www_root = os.path.abspath(args.www_root)
     setup_logging(args.log_directory)
-    default_venv = os.path.join(args.build_root, "venv")
     if args.branch:
         versions_to_build = [
             version