docker: prevent manual pin of dependencies and improve build speed

Jenkinsfile

@@ -10,7 +10,7 @@ pipeline {
     // https://jenkins.io/doc/book/pipeline/syntax/
     agent {
         docker {
-            image "pavics/workflow-tests:211123-update211216"
+            image "pavics/workflow-tests:211221"
             label 'linux && docker'
         }
     }

binder/Dockerfile

@@ -1,4 +1,4 @@
-FROM pavics/workflow-tests:211123-update211216
+FROM pavics/workflow-tests:211221
 
 USER root
 

docker/Dockerfile

@@ -1,6 +1,10 @@
 FROM continuumio/miniconda3
 
-RUN conda update conda
+# Use mamba for much improved performance over conda.
+# The 'channel_priority strict' did help conda but it was not enough.
+RUN conda update conda -n base && \
+    conda install mamba -n base -c defaults -c conda-forge && \
+    conda config --set channel_priority strict
 
 # to checkout other notebooks and to run pip install
 RUN apt-get update && \
@@ -9,18 +13,40 @@ RUN apt-get update && \
     firefox-esr x11-utils && \
     apt-get clean
 
-COPY environment.yml /environment.yml
-
-# needed for our specific jenkins
+# Create user jenkins for our Jenkins e2e notebooks test suite.
+# Change /opt/conda folder permissions for jupyter-conda extension.
 RUN groupadd --gid 1000 jenkins \
-    && useradd --uid 1000 --gid jenkins --create-home jenkins
+    && useradd --uid 1000 --gid jenkins --create-home jenkins && \
+    chmod -R a+rwx /opt/conda
 
-# Change these folders' permissions for jupyter-conda extension
-RUN chmod -R a+rwx /opt/conda
+COPY environment.yml /environment.yml
 
 # create env "birdy"
 # use umask 0000 so that the files for the new environment are usable by user 'jenkins' for the jupyter-conda-extension
-RUN umask 0000 && conda env create -f /environment.yml
+#
+# Perform 2 stages install because one single 'conda env create -f
+# /environment.yml' was taking forever to complete, same with mamba.
+# Had to do this 2 stages install.  2 stages install was also taking forever
+# with conda so had to switch to mamba.
+#
+# One single 'conda env create -f /environment.yml' takes forever because we
+# removed all direct dependencies of xclim and ravenpy in /environment.yml for
+# dependencies pinning by xclim and ravenpy to take effect.  This results in
+# conda having a lot more packages to "solve" and it seems the solver
+# performance dropped exponentially with the number of packages to solve.
+#
+# Conda was stuck at this step:
+# DEBUG conda.common._logic:_run_sat(607): Invoking SAT with clause count: 2500273
+#
+# Pin python=3.8 for vtk-cdat needed by vcs.  Otherwise, 3.9 would have been selected.
+#   package vcs-8.1-py_0 requires vtk-cdat >8.1, but none of the providers can be installed
+#   package vtk-cdat-8.2.0.8.2.1-py38hbc81915_0 requires python >=3.8,<3.9.0a0 *_cpython
+# Pin python=3.7 for libgdal (note libgdal was working with python 3.9 but blew up when we pin 3.8 for vcs)
+#   package gdal-3.1.4-py38h25844d8_1 requires libgdal 3.1.4 h50e41a3_1, but none of the providers can be installed
+# TODO remove python pin once a vtk-cdat for py39 exist, libgdal already worked for 3.9.
+RUN umask 0000 && \
+    mamba create --name birdy --channel conda-forge --channel defaults xclim ravenpy python=3.7 && \
+    mamba env update --name birdy --file /environment.yml
 
 # alternate way to 'source activate birdy'
 ENV PATH="/opt/conda/envs/birdy/bin:$PATH"
@@ -33,35 +59,41 @@ RUN python -m ipykernel install --name birdy
 # anything accidentally
 # this is for debug only, all dependencies should be specified in
 # environment.yml above
-# RUN conda install -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy nbdime
+# RUN mamba install -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy nbdime
 
 # build jupyterlab extensions installed by conda, see `jupyter labextension list`
+# Supposedly not needed with jupyterlab v3 anymore but see
+# https://github.com/jupyterlab/jupyterlab/issues/11726#issuecomment-998901247
+# TODO: remove 'jupyter lab build' step once all extensions move to prebuilt extensions,
+# see comment https://github.com/jupyterlab/jupyterlab/issues/11726#issuecomment-998917305
+# Currently jupyter-dash is holding back this step, see
+# https://github.com/plotly/jupyter-dash/issues/49
 RUN jupyter lab build
 
-# for ipywidgets to work with jupyter lab (notebooks works out of the box)
-RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager \
-    && jupyter serverextension enable voila --sys-prefix \
-    && jupyter labextension install @jupyter-widgets/jupyterlab-manager jupyter-leaflet \
-    && jupyter labextension install jupyterlab-topbar-text \
-                                    jupyterlab-theme-toggle
+RUN jupyter serverextension enable voila --sys-prefix
 #    && jupyter labextension install jupyterlab-clipboard
 
-ADD https://raw.githubusercontent.com/jupyter/docker-stacks/master/base-notebook/start.sh /usr/local/bin/
-ADD https://raw.githubusercontent.com/jupyter/docker-stacks/master/base-notebook/start-singleuser.sh /usr/local/bin/
-ADD https://raw.githubusercontent.com/jupyter/docker-stacks/master/base-notebook/start-notebook.sh /usr/local/bin/
-ADD https://raw.githubusercontent.com/jupyter/docker-stacks/master/base-notebook/fix-permissions /usr/local/bin/
-ADD https://raw.githubusercontent.com/jupyter/docker-stacks/master/base-notebook/jupyter_notebook_config.py /etc/jupyter/
-RUN chmod a+rx /usr/local/bin/start.sh /usr/local/bin/start-singleuser.sh /usr/local/bin/start-notebook.sh /usr/local/bin/fix-permissions; \
-    chmod a+r /etc/jupyter/jupyter_notebook_config.py
+# This should be "master" but commit
+# https://github.com/jupyter/docker-stacks/commit/c772e98ac794173d6ed83a08ec249038b27ca3be
+# is breaking with us since we do not have user jovyan.
+ENV DOCKER_STACKS_COMMIT=709206ac8788475728cc9c992c25fb5f1501bc29
 
-# For Pavics-landing notebooks to re-create Jupyter env layout:
+# /notebook_dir for Pavics-landing notebooks to re-create Jupyter env layout:
 # /notebook_dir/writable-workspace, /notebook_dir/pavics-homepage.
 #
 # Path to the /notebook_dir/pavics-homepage/tutorial_data/*.geojson files are
 # hardcoded so users can copy the nb to writable-workspace/ dir and still be able
 # to run them seemlessly from the Jupyter env (without having to also copy
 # those *.geojson files with the notebooks).
-RUN mkdir /notebook_dir && chown jenkins /notebook_dir
+RUN wget https://raw.githubusercontent.com/jupyter/docker-stacks/$DOCKER_STACKS_COMMIT/base-notebook/start.sh --output-document /usr/local/bin/start.sh && \
+    wget https://raw.githubusercontent.com/jupyter/docker-stacks/$DOCKER_STACKS_COMMIT/base-notebook/start-singleuser.sh --output-document /usr/local/bin/start-singleuser.sh && \
+    wget https://raw.githubusercontent.com/jupyter/docker-stacks/$DOCKER_STACKS_COMMIT/base-notebook/start-notebook.sh --output-document /usr/local/bin/start-notebook.sh && \
+    wget https://raw.githubusercontent.com/jupyter/docker-stacks/$DOCKER_STACKS_COMMIT/base-notebook/fix-permissions --output-document /usr/local/bin/fix-permissions && \
+    mkdir /etc/jupyter && \
+    wget https://raw.githubusercontent.com/jupyter/docker-stacks/$DOCKER_STACKS_COMMIT/base-notebook/jupyter_notebook_config.py --output-document /etc/jupyter/jupyter_notebook_config.py && \
+    chmod a+rx /usr/local/bin/start.sh /usr/local/bin/start-singleuser.sh /usr/local/bin/start-notebook.sh /usr/local/bin/fix-permissions && \
+    chmod a+r /etc/jupyter/jupyter_notebook_config.py && \
+    mkdir /notebook_dir && chown jenkins /notebook_dir
 
 # problem running start-notebook.sh when being root
 # the jupyter/base-notebook image also do not default to root user so we do the same here

docker/environment.yml

@@ -6,13 +6,27 @@ channels:
   - bokeh
   - plotly  # for jupyter-dash
   - defaults
+
 dependencies:
+
+  # Do not put xclim and ravenpy direct dependencies here to let xclim and ravenpy
+  # manage their own dependencies pinning.
+  #
+  # xclim direct dependencies: https://github.com/conda-forge/xclim-feedstock/blob/master/recipe/meta.yaml
+  # ravenpy direct dependencies: https://github.com/conda-forge/ravenpy-feedstock/blob/master/recipe/meta.yaml
+
+  # Pin latest xclim and ravenpy to avoid downgrading during the second install
+  # phase.  Mamba is quicker to solve dependencies than conda but it is less
+  # precise so accidental downgrade happends.
+  - xclim >= 0.32.1
+  - ravenpy >= 0.7.7
+
   - matplotlib
-  - xarray
-  - numpy
+    # - xarray  # from xclim and ravenpy
+    # - numpy  # from xclim and ravenpy
   - birdy
-  - owslib>=0.23.0
-  - netcdf4
+    # - owslib>=0.23.0  # from ravenpy
+    # - netcdf4  # from ravenpy
   # https://github.com/ecmwf/cfgrib
   # Python interface to map GRIB files to the Unidata's Common Data Model v4
   # following the CF Conventions.
@@ -22,11 +36,11 @@ dependencies:
   - descartes
   # Pin rasterio for ravenpy, remove on next build.
   # See https://github.com/CSHS-CWRA/RavenPy/commit/eae66e9afc30e2381e9119644a0695d1d248c739
-  - rasterio <= 1.2.6
-  - gdal  # for osgeo
-  - geopandas
-  - pandas
-  - rioxarray
+    # - rasterio <= 1.2.6  # from ravenpy
+    # - gdal  # for osgeo, from ravenpy
+    # - geopandas  # from ravenpy
+    # - pandas  # from xclim and ravenpy
+    # - rioxarray  # from ravenpy
   - scikit-image
   - ipyleaflet
   - threddsclient
@@ -48,12 +62,12 @@ dependencies:
   # pinning hvplot did not solve the problem with violin plot.
   - hvplot
   - nc-time-axis
-  - cftime
-  - statsmodels  # for ravenpy
+    # - cftime  # from xclim and ravenpy
+    # - statsmodels  # for ravenpy
   # for error 'ImportError: HTTPFileSystem requires "requests" and "aiohttp" to
   # be installed' with call 'fsspec.filesystem('https')'
   - aiohttp
-  - pydantic
+    # - pydantic  # from ravenpy
   # Intake is a lightweight set of tools for loading and sharing data in data science projects
   - intake
   # https://intake.readthedocs.io/en/latest/plugin-directory.html
@@ -74,11 +88,9 @@ dependencies:
   - zarr
   # https://github.com/dask/s3fs/
   - s3fs
-  - xclim
   # Pinning shapely for ravenpy.  Remove on next rebuild.
   # https://github.com/CSHS-CWRA/RavenPy/blob/f63e1e5b967c0d7c17e679c8f9d6d309a94096e6/environment.yml#L35
-  - shapely <=1.7.1
-  - ravenpy
+  # - shapely <=1.7.1  # from ravenpy
   # https://github.com/roocs/clisops
   - clisops
   # Universal Regridder for Geospatial Data
@@ -105,6 +117,10 @@ dependencies:
   - notebook
   - jupyterlab
   - jupyterhub
+  # https://ipywidgets.readthedocs.io/en/latest/user_install.html
+  - ipywidgets
+  # https://github.com/jupyter-widgets/ipyleaflet
+  - ipyleaflet
   # https://github.com/mamba-org/gator (was jupyter_conda)
   - mamba_gator
   # to diff .ipynb files
@@ -125,6 +141,11 @@ dependencies:
   # xeus-python: back-end kernel implementing the Jupyter Debug Protocol
   - xeus-python
   - jupyter-dash
+  # Force newer nodejs for 'jupyter lab build' issue
+  # https://github.com/jupyterlab/jupyterlab/issues/11726#issuecomment-998901247
+  # TODO: remove nodejs once all extensions move to prebuilt extensions, see comment
+  # https://github.com/jupyterlab/jupyterlab/issues/11726#issuecomment-998917305
+  - nodejs >= 16.0
   # utilities
   - curl
   - wget

launchcontainer

@@ -1,7 +1,7 @@
 #!/bin/sh -x
 
 if [ -z "$DOCKER_IMAGE" ]; then
-    DOCKER_IMAGE="pavics/workflow-tests:211123-update211216"
+    DOCKER_IMAGE="pavics/workflow-tests:211221"
 fi
 
 if [ -z "$CONTAINER_NAME" ]; then

launchnotebook

@@ -7,7 +7,7 @@ if [ -z "$PORT" ]; then
 fi
 
 if [ -z "$DOCKER_IMAGE" ]; then
-    DOCKER_IMAGE="pavics/workflow-tests:211123-update211216"
+    DOCKER_IMAGE="pavics/workflow-tests:211221"
 fi
 
 if [ -z "$CONTAINER_NAME" ]; then