Merge branch 'master' of github.com:nilearn/nilearn into int-niftis

* 'master' of github.com:nilearn/nilearn: Fixed the redundant & missing test case in merged PR nilearn#2035 (nilearn#2205) Modify fetch_development_fmri to fetch adults or children (nilearn#2035) Verbose doc building to ease tracking of progress & diagnose stalls (nilearn#2203) New conda env is created once conda path has been activated Conda environment is created for full-builds Refactor CircleCI config for reduced redundancy (nilearn#2204)
kchawla-pi · Nov 10, 2019 · 3ada5ef · 3ada5ef
2 parents 392a1a9 + 4e1ccac
commit 3ada5ef
Show file tree

Hide file tree

Showing 10 changed files with 258 additions and 119 deletions.
diff --git a/.circleci/auto-cache-timestamp b/.circleci/auto-cache-timestamp
diff --git a/.circleci/clean-cache.py b/.circleci/clean-cache.py
@@ -7,7 +7,7 @@
 
 
 def update_cache_timestamp(timestamp_filename):
-    """ Updates the contents of the manual-cache-timestamp file
+    """ Updates the contents of the docs-cache-timestamp file
     with current timestamp.
 
     Returns
@@ -22,5 +22,5 @@ def update_cache_timestamp(timestamp_filename):
 
 
 if __name__ == '__main__':
-    update_cache_timestamp('manual-cache-timestamp')
-    update_cache_timestamp('auto-cache-timestamp')
+    update_cache_timestamp('docs-cache-timestamp')
+    update_cache_timestamp('packages-cache-timestamp')
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -3,42 +3,46 @@
 # It doesn't operate on master branch. New branches are always built from scratch.
 # full-build always rebuilds from scratch, without any cache. Only for changes in master branch.
 
-version: 2
+version: 2.1
 
-jobs:
-  quick-build:
-    docker:
-      - image: circleci/python:3.6
-    environment:
-      DISTRIB: "conda"
-      PYTHON_VERSION: "3.6"
-      NUMPY_VERSION: "*"
-      SCIPY_VERSION: "*"
-      SCIKIT_LEARN_VERSION: "*"
-      JOBLIB_VERSION: "*"
-      MATPLOTLIB_VERSION: "*"
+commands:
 
+  preinstall:
+    description: "Cleans up unused packages; Updates system packages"
+    steps:
+      - run:
+          name: Remove conflicting packages
+          command: |
+            # Get rid of existing virtualenvs on circle ci as they conflict with conda.
+            # Trick found here:
+            # https://discuss.circleci.com/t/disable-autodetection-of-project-or-application-of-python-venv/235/10
+            cd && rm -rf ~/.pyenv && rm -rf ~/virtualenvs
+
+            # We need to remove conflicting texlive packages.
+            sudo -E apt-get -yq remove texlive-binaries --purge
+      - run:
+          name: Install packages for make -C doc check
+          command: |
+            # Installing required packages for `make -C doc check command` to work.
+            sudo -E apt-get -yq update
+            sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install dvipng texlive-latex-base texlive-latex-extra
+
+  restore_from_cache:
+    description: "Restores the cache of previously built docs & python packages if present"
     steps:
-      - checkout
-        # Get rid of existing virtualenvs on circle ci as they conflict with conda.
-        # Trick found here:
-        # https://discuss.circleci.com/t/disable-autodetection-of-project-or-application-of-python-venv/235/10
-      - run: cd && rm -rf ~/.pyenv && rm -rf ~/virtualenvs
-        # We need to remove conflicting texlive packages.
-      - run: sudo -E apt-get -yq remove texlive-binaries --purge
-        # Installing required packages for `make -C doc check command` to work.
-      - run: sudo -E apt-get -yq update
-      - run: sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install dvipng texlive-latex-base texlive-latex-extra
       - run:
-          name: Today & Week  # Saving today's date and current week of the year in files to generate daily & weekly new cache key respectively.
+          name: Generate cache keys from today's date for built docs & week number for python packages
           command: |
             date +%F > today
             date +%U > week_num
       - restore_cache:
-          key: v1-packages+datasets-{{ checksum "week_num" }}
+          key: v1-packages+datasets-{{ checksum "week_num" }}-{{ checksum ".circleci/packages-cache-timestamp" }}
       - restore_cache:
-          key: v1-docs-{{ .Branch }}-{{ checksum "today" }}-{{ checksum ".circleci/manual-cache-timestamp" }}
+          key: v1-docs-{{ .Branch }}-{{ checksum "today" }}-{{ checksum ".circleci/docs-cache-timestamp" }}
 
+  cache_aware_conda_setup:
+    description: "Downloads & installs conda if not restord by cache"
+    steps:
       - run:
           name: Download & install conda if absent
           command: |
@@ -64,91 +68,109 @@ jobs:
             else
               conda create -n testenv -yq
             fi
+
+  cache_ignorant_conda_setup:
+    description: "Downloads & installs only the fresh copy of conda."
+    steps:
+      - run:
+          name: setup conda afresh
+          command: |
+            wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
+            chmod +x ~/miniconda.sh && ~/miniconda.sh -b
+            echo 'export PATH="$HOME/miniconda3/bin:$PATH"'  >> $BASH_ENV
+      - run: conda create -n testenv
+
+  install_dependencies:
+    description: "Installs the necessary Python packages"
+    steps:
       - run:
           name: Install packages in conda env
           command: |
-            conda install -n testenv python=3.6 numpy scipy scikit-learn matplotlib pandas \
+            conda install -n testenv python=3.7 numpy scipy scikit-learn matplotlib pandas \
             lxml mkl sphinx numpydoc pillow pandas -yq
             conda install -n testenv nibabel sphinx-gallery junit-xml -c conda-forge -yq
+
+  build_docs:
+    description: "Installs Nilearn & builds documentation using Sphinx's make html-strict"
+    steps:
       - run:
-          name: Running CircleCI test (make html)
+          name: Building documentation
           command: |
             source activate testenv
             pip install -e .
             set -o pipefail && cd doc && make html-strict 2>&1 | tee log.txt
           no_output_timeout: 7h
+
+  store_results:
+    description: "Stores build times and artifacts"
+    steps:
       - store_test_results:
           path: doc/_build/test-results
       - store_artifacts:
           path: doc/_build/test-results
+      - store_artifacts:
+          path: doc/_build/html
+      - store_artifacts:
+          path: coverage
+      - store_artifacts:
+          path: doc/log.txt
 
+  save_to_cache:
+    description: "Caches the downloaded packages & buit docs."
+    steps:
       - save_cache:
           key: v1-packages+datasets-{{ checksum "week_num" }}
           paths:
             - ../nilearn_data
             - ../miniconda3
       - save_cache:
-          key: v1-docs-{{ .Branch }}-{{ checksum "today" }}-{{ checksum ".circleci/manual-cache-timestamp" }}
+          key: v1-docs-{{ .Branch }}-{{ checksum "today" }}-{{ checksum ".circleci/docs-cache-timestamp" }}
           paths:
             - doc
 
-      - store_artifacts:
-          path: doc/_build/html
-      - store_artifacts:
-          path: coverage
-      - store_artifacts:
-          path: doc/log.txt
 
+jobs:
 
-  full-build:
+  quick-build:
     docker:
-      - image: circleci/python:3.6
+      - image: circleci/python:3.7
     environment:
       DISTRIB: "conda"
-      PYTHON_VERSION: "3.6"
+      PYTHON_VERSION: "3.7"
       NUMPY_VERSION: "*"
       SCIPY_VERSION: "*"
       SCIKIT_LEARN_VERSION: "*"
+      JOBLIB_VERSION: "*"
       MATPLOTLIB_VERSION: "*"
 
     steps:
       - checkout
-        # Get rid of existing virtualenvs on circle ci as they conflict with conda.
-        # Trick found here:
-        # https://discuss.circleci.com/t/disable-autodetection-of-project-or-application-of-python-venv/235/10
-      - run: cd && rm -rf ~/.pyenv && rm -rf ~/virtualenvs
-        # We need to remove conflicting texlive packages.
-      - run: sudo -E apt-get -yq remove texlive-binaries --purge
-        # Installing required packages for `make -C doc check command` to work.
-      - run: sudo -E apt-get -yq update
-      - run: sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install dvipng texlive-latex-base texlive-latex-extra
-      - run: wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
-      - run: chmod +x ~/miniconda.sh && ~/miniconda.sh -b
-      - run: echo 'export PATH="$HOME/miniconda3/bin:$PATH"'  >> $BASH_ENV
-      - run:
-          name: Install packages in conda env
-          command: |
-            conda create -n testenv python=3.6 numpy scipy scikit-learn matplotlib pandas \
-            lxml mkl sphinx numpydoc pillow pandas -yq
-            conda install -n testenv nibabel sphinx-gallery junit-xml -c conda-forge -yq
-      - run:
-          name: Running CircleCI test (make html)
-          command: |
-            source activate testenv
-            pip install -e .
-            set -o pipefail && cd doc && make html-strict 2>&1 | tee log.txt
-          no_output_timeout: 7h
-      - store_test_results:
-          path: doc/_build/test-results
-      - store_artifacts:
-          path: doc/_build/test-results
+      - preinstall
+      - restore_from_cache
+      - cache_aware_conda_setup
+      - install_dependencies
+      - build_docs
+      - store_results
+      - save_to_cache
 
-      - store_artifacts:
-          path: doc/_build/html
-      - store_artifacts:
-          path: coverage
-      - store_artifacts:
-          path: doc/log.txt
+  full-build:
+    docker:
+      - image: circleci/python:3.7
+    environment:
+      DISTRIB: "conda"
+      PYTHON_VERSION: "3.7"
+      NUMPY_VERSION: "*"
+      SCIPY_VERSION: "*"
+      SCIKIT_LEARN_VERSION: "*"
+      MATPLOTLIB_VERSION: "*"
+
+    steps:
+      - checkout
+      - preinstall
+      - cache_ignorant_conda_setup
+      - install_dependencies
+      - build_docs
+      - store_results
 
 
 workflows:

diff --git a/.circleci/docs-cache-timestamp b/.circleci/docs-cache-timestamp
@@ -0,0 +1 @@
+2019-11-07 13:17:41.900352
diff --git a/.circleci/manual-cache-timestamp b/.circleci/manual-cache-timestamp
diff --git a/.circleci/packages-cache-timestamp b/.circleci/packages-cache-timestamp
@@ -0,0 +1 @@
+2019-11-07 13:17:41.900617
diff --git a/doc/Makefile b/doc/Makefile
@@ -2,7 +2,7 @@
 #
 
 # You can set these variables from the command line.
-SPHINXOPTS    =
+SPHINXOPTS    = -v
 SPHINXBUILD   = sphinx-build
 PAPER         =
 BUILDDIR      = _build