diff --git a/README.md b/README.md index 1fcbb15..c08882e 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,16 @@ This repository contains scripts for automatically building the Python documentation on [docs.python.org](https://docs.python.org). + # How to test it? - $ mkdir -p www logs build_root - $ python3 -m venv build_root/venv/ - $ build_root/venv/bin/python -m pip install -r requirements.txt - $ python3 ./build_docs.py --quick --build-root build_root --www-root www --log-directory logs --group $(id -g) --skip-cache-invalidation +The following command should build all maintained versions and +translations in ``./www``, beware it can take a few hours: + + $ python3 ./build_docs.py --quick --build-root ./build_root --www-root ./www --log-directory ./logs --group $(id -g) --skip-cache-invalidation + +If you don't need to build all translations of all branches, add +``--language en --branch master``. # Check current version diff --git a/build_docs.py b/build_docs.py index e56b89f..3096537 100755 --- a/build_docs.py +++ b/build_docs.py @@ -58,7 +58,7 @@ sentry_sdk.init() VERSION = "19.0" - +DEFAULT_SPHINX_VERSION = "2.3.1" if not hasattr(shlex, "join"): # Add shlex.join if missing (pre 3.8) @@ -70,7 +70,7 @@ class Version: STATUSES = {"EOL", "security-fixes", "stable", "pre-release", "in development"} - def __init__(self, name, branch, status): + def __init__(self, name, branch, status, sphinx_version=DEFAULT_SPHINX_VERSION): if status not in self.STATUSES: raise ValueError( "Version status expected to be in {}".format(", ".join(self.STATUSES)) @@ -78,6 +78,7 @@ def __init__(self, name, branch, status): self.name = name self.branch = branch self.status = status + self.sphinx_version = sphinx_version @property def url(self): @@ -93,11 +94,13 @@ def title(self): ) # EOL and security-fixes are not automatically built, no need to remove them -# from the list. +# from the list, this way we can still rebuild them manually as needed. +# Please pin the sphinx_versions of EOL and security-fixes, as we're not maintaining +# their doc, they don't follow Sphinx deprecations. VERSIONS = [ - Version("2.7", "2.7", "EOL"), - Version("3.5", "3.5", "security-fixes"), - Version("3.6", "3.6", "security-fixes"), + Version("2.7", "2.7", "EOL", sphinx_version="2.3.1"), + Version("3.5", "3.5", "security-fixes", sphinx_version="1.8.4"), + Version("3.6", "3.6", "security-fixes", sphinx_version="2.3.1"), Version("3.7", "3.7", "stable"), Version("3.8", "3.8", "stable"), Version("3.9", "3.9", "pre-release"), @@ -455,6 +458,25 @@ def build_one( logging.info("Build done for version: %s, language: %s", version.name, language.tag) +def build_venv(build_root, version): + """Build a venv for the specific version. + This is used to pin old Sphinx versions to old cpython branches. + """ + requirements = [ + "blurb", + "jieba", + "python-docs-theme", + "sphinx=={}".format(version.sphinx_version), + ] + venv_path = os.path.join(build_root, "venv-with-sphinx-" + version.sphinx_version) + shell_out(["python3", "-m", "venv", venv_path]) + shell_out( + [os.path.join(venv_path, "bin", "python"), "-m", "pip", "install"] + + requirements + ) + return venv_path + + def copy_build_to_webroot( build_root, version, @@ -469,6 +491,7 @@ def copy_build_to_webroot( logging.info( "Publishing start for version: %s, language: %s", version.name, language.tag ) + Path(www_root).mkdir(parents=True, exist_ok=True) checkout = os.path.join( build_root, version.name, "cpython-{lang}".format(lang=language.tag) ) @@ -670,6 +693,7 @@ def setup_logging(log_directory): if sys.stderr.isatty(): logging.basicConfig(format="%(levelname)s:%(message)s", stream=sys.stderr) else: + Path(log_directory).mkdir(parents=True, exist_ok=True) handler = logging.handlers.WatchedFileHandler( os.path.join(log_directory, "docsbuild.log") ) @@ -691,7 +715,6 @@ def main(): if args.www_root: args.www_root = os.path.abspath(args.www_root) setup_logging(args.log_directory) - venv = os.path.join(args.build_root, "venv") if args.branch: versions_to_build = [ version @@ -720,6 +743,7 @@ def main(): scope.set_tag("version", version.name) scope.set_tag("language", language.tag) try: + venv = build_venv(args.build_root, version) build_one( version, args.quick, diff --git a/requirements.in b/requirements.in index 8cd884d..9898221 100644 --- a/requirements.in +++ b/requirements.in @@ -1,6 +1 @@ -blurb -jieba -python-docs-theme -requests sentry-sdk -sphinx==2.3.1 diff --git a/requirements.txt b/requirements.txt index 5c35b03..dbe8732 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,34 +4,6 @@ # # pip-compile requirements.in # -alabaster==0.7.12 # via sphinx -babel==2.7.0 # via sphinx -blurb==1.0.7 # via -r requirements.in (line 1) -certifi==2019.11.28 # via requests, sentry-sdk -chardet==3.0.4 # via requests -docutils==0.15.2 # via sphinx -idna==2.8 # via requests -imagesize==1.1.0 # via sphinx -jieba==0.42.1 # via -r requirements.in (line 2) -jinja2==2.10.3 # via sphinx -markupsafe==1.1.1 # via jinja2 -packaging==19.2 # via sphinx -pygments==2.5.2 # via sphinx -pyparsing==2.4.5 # via packaging -python-docs-theme==2020.1 # via -r requirements.in (line 3) -pytz==2019.3 # via babel -requests==2.22.0 # via -r requirements.in (line 4), sphinx -sentry-sdk==0.13.5 # via -r requirements.in (line 5) -six==1.13.0 # via packaging -snowballstemmer==2.0.0 # via sphinx -sphinx==2.3.1 # via -r requirements.in (line 6) -sphinxcontrib-applehelp==1.0.1 # via sphinx -sphinxcontrib-devhelp==1.0.1 # via sphinx -sphinxcontrib-htmlhelp==1.0.2 # via sphinx -sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.2 # via sphinx -sphinxcontrib-serializinghtml==1.1.3 # via sphinx -urllib3==1.25.7 # via requests, sentry-sdk - -# The following packages are considered to be unsafe in a requirements file: -# setuptools +certifi==2020.6.20 # via sentry-sdk +sentry-sdk==0.15.1 # via -r requirements.in +urllib3==1.25.9 # via sentry-sdk