CI Build doc on CircleCI

.circleci/config.yml

@@ -1,38 +1,78 @@
 version: 2.1
 
-# Parameters required to trigger the execution
-# of the "doc-min-dependencies" and "doc" jobs
-parameters:
-  GITHUB_RUN_URL:
-    type: string
-    default: "none"
-
 jobs:
   doc-min-dependencies:
     docker:
       - image: cimg/python:3.8.12
     environment:
-      - GITHUB_ARTIFACT_URL: << pipeline.parameters.GITHUB_RUN_URL >>/doc-min-dependencies.zip
+      - MKL_NUM_THREADS: 2
+      - OPENBLAS_NUM_THREADS: 2
+      - CONDA_ENV_NAME: testenv
+      - LOCK_FILE: build_tools/circle/doc_min_dependencies_linux-64_conda.lock
+      # Sphinx race condition in doc-min-dependencies is causing job to stall
+      # Here we run the job serially
+      - SPHINX_NUMJOBS: 1
     steps:
       - checkout
-      - run: bash build_tools/circle/download_documentation.sh
+      - run: ./build_tools/circle/checkout_merge_commit.sh
+      - restore_cache:
+          key: v1-doc-min-deps-datasets-{{ .Branch }}
+      - restore_cache:
+          keys:
+            - doc-min-deps-ccache-{{ .Branch }}
+            - doc-min-deps-ccache
+      - run: ./build_tools/circle/build_doc.sh
+      - save_cache:
+          key: doc-min-deps-ccache-{{ .Branch }}-{{ .BuildNum }}
+          paths:
+            - ~/.ccache
+            - ~/.cache/pip
+      - save_cache:
+          key: v1-doc-min-deps-datasets-{{ .Branch }}
+          paths:
+            - ~/scikit_learn_data
       - store_artifacts:
           path: doc/_build/html/stable
           destination: doc
+      - store_artifacts:
+          path: ~/log.txt
+          destination: log.txt
 
   doc:
     docker:
       - image: cimg/python:3.8.12
     environment:
-      - GITHUB_ARTIFACT_URL: << pipeline.parameters.GITHUB_RUN_URL >>/doc.zip
+      - MKL_NUM_THREADS: 2
+      - OPENBLAS_NUM_THREADS: 2
+      - CONDA_ENV_NAME: testenv
+      - LOCK_FILE: build_tools/circle/doc_linux-64_conda.lock
     steps:
       - checkout
-      - run: bash build_tools/circle/download_documentation.sh
+      - run: ./build_tools/circle/checkout_merge_commit.sh
+      - restore_cache:
+          key: v1-doc-datasets-{{ .Branch }}
+      - restore_cache:
+          keys:
+            - doc-ccache-{{ .Branch }}
+            - doc-ccache
+      - run: ./build_tools/circle/build_doc.sh
+      - save_cache:
+          key: doc-ccache-{{ .Branch }}-{{ .BuildNum }}
+          paths:
+            - ~/.ccache
+            - ~/.cache/pip
+      - save_cache:
+          key: v1-doc-datasets-{{ .Branch }}
+          paths:
+            - ~/scikit_learn_data
       - store_artifacts:
           path: doc/_build/html/stable
           destination: doc
-      # Persists the generated documentation, so that it
-      # can be attached and deployed in the "deploy" job
+      - store_artifacts:
+          path: ~/log.txt
+          destination: log.txt
+      # Persists generated documentation so that it can be attached and deployed
+      # in the 'deploy' step.
       - persist_to_workspace:
           root: doc/_build/html
           paths: .
@@ -54,36 +94,12 @@ jobs:
               bash build_tools/circle/push_doc.sh doc/_build/html/stable
             fi
 
-  # This noop job is required for the pipeline to exist, so that the
-  # documentation related jobs can be triggered.
-  noop:
-    docker:
-      - image: cimg/python:3.8.12
-    steps:
-    - run: |
-        echo "This is no-op job for the pipeline to exist, so that it triggers "
-        echo "Circle CI jobs pushing the artifacts of the documentation built "
-        echo "via GitHub actions."
-
 workflows:
   version: 2
-
   build-doc-and-deploy:
-    when:
-      not:
-        equal: [ "none", << pipeline.parameters.GITHUB_RUN_URL >> ]
-    # The jobs should run only when triggered by the workflow
     jobs:
-      - doc-min-dependencies
       - doc
+      - doc-min-dependencies
       - deploy:
           requires:
             - doc
-
-  noop:
-    when:
-      equal: [ "none", << pipeline.parameters.GITHUB_RUN_URL >> ]
-    # Prevent double execution of this job: on push
-    # by default and when triggered by the workflow
-    jobs:
-      - noop

build_tools/github/build_doc.sh → build_tools/circle/build_doc.sh

@@ -16,9 +16,12 @@ set -e
 # If the inspection of the current commit fails for any reason, the default
 # behavior is to quick build the documentation.
 
+# defines the get_dep and show_installed_libraries functions
+source build_tools/shared.sh
+
 if [ -n "$GITHUB_ACTION" ]
 then
-    # Map the variables for the new documentation builder to the old one
+    # Map the variables from Github Action to CircleCI
     CIRCLE_SHA1=$(git log -1 --pretty=format:%H)
 
     CIRCLE_JOB=$GITHUB_JOB
@@ -169,11 +172,12 @@ ccache -M 512M
 export CCACHE_COMPRESS=1
 
 # pin conda-lock to latest released version (needs manual update from time to time)
-mamba install conda-lock==1.0.5 -y
+mamba install "$(get_dep conda-lock min)" -y
+
 conda-lock install --log-level WARNING --name $CONDA_ENV_NAME $LOCK_FILE
 source activate $CONDA_ENV_NAME
 
-mamba list
+show_installed_libraries
 
 # Set parallelism to 3 to overlap IO bound tasks with CPU bound tasks on CI
 # workers with 2 cores when building the compiled extensions of scikit-learn.
@@ -185,12 +189,18 @@ ccache -s
 
 export OMP_NUM_THREADS=1
 
+# Avoid CI job getting killed because it uses too much memory
+if [[ -z $SPHINX_NUMJOBS ]]; then
+    export SPHINX_NUMJOBS=2
+fi
+
 if [[ "$CIRCLE_BRANCH" =~ ^main$ && -z "$CI_PULL_REQUEST" ]]
 then
     # List available documentation versions if on main
     python build_tools/circle/list_versions.py > doc/versions.rst
 fi
 
+
 # The pipefail is requested to propagate exit code
 set -o pipefail && cd doc && make $make_args 2>&1 | tee ~/log.txt
 

build_tools/update_environments_and_lock_files.py

@@ -251,7 +251,7 @@ def remove_from(alist, to_remove):
     },
     {
         "build_name": "doc_min_dependencies",
-        "folder": "build_tools/github",
+        "folder": "build_tools/circle",
         "platform": "linux-64",
         "channel": "conda-forge",
         "conda_dependencies": common_dependencies_without_coverage
@@ -286,7 +286,7 @@ def remove_from(alist, to_remove):
     },
     {
         "build_name": "doc",
-        "folder": "build_tools/github",
+        "folder": "build_tools/circle",
         "platform": "linux-64",
         "channel": "conda-forge",
         "conda_dependencies": common_dependencies_without_coverage