From 74cee6e402baab0a6884b0be2a8c2f83b095e895 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Mon, 3 Apr 2023 16:46:40 -0400 Subject: [PATCH 1/3] docker: incremental build to get new packages (esgf-pyclient and xncml) to production faster MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Full build pulls newer RavenPy which requires more testing and a new RavenWPS to match. This requires more time for testing. esgf-pyclient for pavics-sdi esgf-dap.ipynb, PR https://github.com/Ouranosinc/pavics-sdi/pull/269 xncml for gen_catalog refactoring, PR https://github.com/Ouranosinc/pavics-vdb/pull/46 ``` Step 4/5 : RUN umask 0000 && mamba install -c conda-forge -c cdat -c bokeh -c plotly -c pyston -c pyviz/label/dev -c defaults -n birdy esgf-pyclient && pip install xncml ---> Running in 11f0bd71b9ff Transaction Prefix: /opt/conda/envs/birdy Updating specs: - esgf-pyclient - ca-certificates - certifi - openssl Package Version Build Channel Size ─────────────────────────────────────────────────────────────────────────────────────────── Install: ─────────────────────────────────────────────────────────────────────────────────────────── + astor 0.8.1 pyh9f0ad1d_0 conda-forge/noarch 26kB + aws-c-auth 0.6.26 hf365957_1 conda-forge/linux-64 96kB + aws-c-compression 0.2.16 h03acc5a_5 conda-forge/linux-64 19kB + aws-c-http 0.7.6 hf342b9f_0 conda-forge/linux-64 192kB + aws-c-mqtt 0.8.6 hc4349f7_12 conda-forge/linux-64 144kB + aws-c-s3 0.2.7 h909e904_1 conda-forge/linux-64 76kB + aws-c-sdkutils 0.1.8 h03acc5a_0 conda-forge/linux-64 54kB + aws-crt-cpp 0.19.8 hf7fbfca_12 conda-forge/linux-64 318kB + base58 2.1.1 pyhd8ed1ab_0 conda-forge/noarch 9kB + boto3 1.24.59 pyhd8ed1ab_0 conda-forge/noarch 74kB + cattrs 22.2.0 pyhd8ed1ab_0 conda-forge/noarch 31kB + esgf-pyclient 0.3.1 pyh1a96a4e_2 conda-forge/noarch 23kB + libarrow 11.0.0 h33598ff_9_cpu conda-forge/linux-64 27MB + libgrpc 1.51.1 hcf146ea_3 conda-forge/linux-64 5MB + myproxyclient 2.1.0 pyhd8ed1ab_2 conda-forge/noarch 27kB + qhull 2020.2 h4bd325d_2 conda-forge/linux-64 2MB + requests-cache 1.0.1 pyhd8ed1ab_0 conda-forge/noarch 49kB + s3transfer 0.6.0 pyhd8ed1ab_0 conda-forge/noarch 57kB + ujson 5.7.0 py38h8dc9893_0 conda-forge/linux-64 51kB + url-normalize 1.4.3 pyhd8ed1ab_0 conda-forge/noarch 10kB + xeus-zmq 1.0.2 h0541b36_2 conda-forge/linux-64 347kB Change: ─────────────────────────────────────────────────────────────────────────────────────────── - fiona 1.8.22 py38hc72d8cd_2 conda-forge + fiona 1.8.22 py38h85a5e0f_5 conda-forge/linux-64 801kB - hdf5 1.12.2 mpi_mpich_h08b82f9_0 conda-forge + hdf5 1.12.2 mpi_mpich_h5d83325_1 conda-forge/linux-64 3MB - libarchive 3.5.2 hb890918_3 conda-forge + libarchive 3.5.2 hada088e_3 conda-forge/linux-64 2MB - libcups 2.3.3 h3e49a29_2 conda-forge + libcups 2.3.3 h36d4200_3 conda-forge/linux-64 5MB - libevent 2.1.10 h9b69904_4 conda-forge + libevent 2.1.10 h28343ad_4 conda-forge/linux-64 1MB - libssh2 1.10.0 haa6b8db_3 conda-forge + libssh2 1.10.0 hf14f497_3 conda-forge/linux-64 Cached - libzip 1.9.2 hc869a4a_1 conda-forge + libzip 1.9.2 hc929e4a_1 conda-forge/linux-64 Cached - pulseaudio 16.1 h4a94279_0 conda-forge + pulseaudio 16.1 h126f2b6_0 conda-forge/linux-64 2MB - pycurl 7.45.1 py38h61f0cdf_3 conda-forge + pycurl 7.45.1 py38he9e1d38_3 conda-forge/linux-64 77kB - qt-main 5.15.6 h7acdfc8_2 conda-forge + qt-main 5.15.6 hf6cd601_5 conda-forge/linux-64 55MB - wget 1.20.3 ha56f1ee_1 conda-forge + wget 1.20.3 ha35d2d1_1 conda-forge/linux-64 835kB Upgrade: ─────────────────────────────────────────────────────────────────────────────────────────── - arrow-cpp 9.0.0 py38he270906_2_cpu conda-forge + arrow-cpp 11.0.0 ha770c72_9_cpu conda-forge/linux-64 31kB - aws-c-cal 0.5.11 h95a6274_0 conda-forge + aws-c-cal 0.5.21 h48707d8_2 conda-forge/linux-64 44kB - aws-c-common 0.6.2 h7f98852_0 conda-forge + aws-c-common 0.8.14 h0b41bf4_0 conda-forge/linux-64 200kB - aws-c-event-stream 0.2.7 h3541f99_13 conda-forge + aws-c-event-stream 0.2.20 h00877a2_4 conda-forge/linux-64 54kB - aws-c-io 0.10.5 hfb6a706_0 conda-forge + aws-c-io 0.13.19 h5b20300_3 conda-forge/linux-64 144kB - aws-checksums 0.1.11 ha31a3da_7 conda-forge + aws-checksums 0.1.14 h03acc5a_5 conda-forge/linux-64 50kB - aws-sdk-cpp 1.8.186 hecaee15_4 conda-forge + aws-sdk-cpp 1.10.57 h17c43bd_8 conda-forge/linux-64 4MB - blosc 1.21.1 h83bc5f7_3 conda-forge + blosc 1.21.3 hafa529b_0 conda-forge/linux-64 40kB - ca-certificates 2022.9.24 ha878542_0 conda-forge + ca-certificates 2022.12.7 ha878542_0 conda-forge/linux-64 146kB - certifi 2022.9.24 pyhd8ed1ab_0 conda-forge + certifi 2022.12.7 pyhd8ed1ab_0 conda-forge/noarch 151kB - cryptography 38.0.4 py38h2b5fc30_0 conda-forge + cryptography 40.0.1 py38h3d167d9_0 conda-forge/linux-64 1MB - curl 7.86.0 h7bff187_1 conda-forge + curl 7.88.1 hdc1c0ab_1 conda-forge/linux-64 88kB - gdal 3.5.3 py38h1f15b03_4 conda-forge + gdal 3.6.0 py38h58634bd_13 conda-forge/linux-64 1MB - grpc-cpp 1.47.1 hbad87ad_6 conda-forge + grpc-cpp 1.51.1 h27aab58_3 conda-forge/linux-64 22kB - grpcio 1.47.1 py38h5b6373e_6 conda-forge + grpcio 1.51.1 py38h8dc9893_3 conda-forge/linux-64 769kB - kealib 1.4.15 ha7026e8_1 conda-forge + kealib 1.5.0 ha7026e8_0 conda-forge/linux-64 Cached - krb5 1.19.3 h3790be6_0 conda-forge + krb5 1.20.1 h81ceb04_0 conda-forge/linux-64 1MB - libabseil 20220623.0 cxx17_h48a1fff_5 conda-forge + libabseil 20230125.0 cxx17_hcb278e6_1 conda-forge/linux-64 1MB - libcurl 7.86.0 h7bff187_1 conda-forge + libcurl 7.88.1 hdc1c0ab_1 conda-forge/linux-64 359kB - libgdal 3.5.3 hcb70f40_4 conda-forge + libgdal 3.6.0 h2f87c3e_13 conda-forge/linux-64 10MB - libgoogle-cloud 2.1.0 h9ebe8e8_2 conda-forge + libgoogle-cloud 2.8.0 h3c06191_0 conda-forge/linux-64 38MB - libnghttp2 1.47.0 hdcd2b5c_1 conda-forge + libnghttp2 1.52.0 h61bc06f_0 conda-forge/linux-64 622kB - libpq 14.5 hd77ab85_1 conda-forge + libpq 15.2 hb675445_0 conda-forge/linux-64 2MB - libprotobuf 3.20.2 h6239696_0 conda-forge + libprotobuf 3.21.12 h3eb15da_0 conda-forge/linux-64 2MB - libthrift 0.16.0 h491838f_2 conda-forge + libthrift 0.18.0 h5e4af38_0 conda-forge/linux-64 4MB - mysql-common 8.0.31 haf5c9bc_0 conda-forge + mysql-common 8.0.32 ha901b37_1 conda-forge/linux-64 760kB - mysql-libs 8.0.31 h28c427c_0 conda-forge + mysql-libs 8.0.32 hd7da12d_1 conda-forge/linux-64 2MB - nodejs 18.12.1 h96d913c_0 conda-forge + nodejs 18.15.0 h8d033a5_0 conda-forge/linux-64 15MB - openssl 1.1.1s h166bdaf_0 conda-forge + openssl 3.1.0 h0b41bf4_0 conda-forge/linux-64 3MB - orc 1.7.6 h6c59b99_0 conda-forge + orc 1.8.2 hfdbbad2_2 conda-forge/linux-64 907kB - postgresql 14.5 hdeef612_1 conda-forge + postgresql 15.2 h3248436_0 conda-forge/linux-64 5MB - protobuf 3.20.2 py38hfa26641_1 conda-forge + protobuf 4.21.12 py38h8dc9893_0 conda-forge/linux-64 323kB - pyarrow 9.0.0 py38h097c49a_2_cpu conda-forge + pyarrow 11.0.0 py38hf05218d_9_cpu conda-forge/linux-64 4MB - pyopenssl 22.1.0 pyhd8ed1ab_0 conda-forge + pyopenssl 23.1.1 pyhd8ed1ab_0 conda-forge/noarch 128kB - python 3.8.15 h257c98d_0_cpython conda-forge + python 3.8.16 he550d4f_1_cpython conda-forge/linux-64 23MB - rasterio 1.3.3 py38hb7ee17c_3 conda-forge + rasterio 1.3.4 py38h485b76c_0 conda-forge/linux-64 9MB - re2 2022.06.01 h27087fc_1 conda-forge + re2 2023.02.02 hcb278e6_0 conda-forge/linux-64 201kB - s2n 1.0.10 h9b69904_0 conda-forge + s2n 1.3.41 h3358134_0 conda-forge/linux-64 362kB - tiledb 2.11.3 h1e4a385_1 conda-forge + tiledb 2.13.2 hd532e3d_0 conda-forge/linux-64 5MB - xeus 2.4.1 h70bab47_0 conda-forge + xeus 3.0.5 hac2b420_1 conda-forge/linux-64 343kB - xeus-python 0.14.3 py38h9f61dbb_1 conda-forge + xeus-python 0.15.8 py38hfbd4bf9_1 conda-forge/linux-64 769kB Downgrade: ─────────────────────────────────────────────────────────────────────────────────────────── - streamlit 1.15.1 pyhd8ed1ab_0 conda-forge + streamlit 1.8.0 pyhd8ed1ab_0 conda-forge/noarch 8MB Summary: Install: 21 packages Change: 11 packages Upgrade: 41 packages Downgrade: 1 packages Total download: 251MB ─────────────────────────────────────────────────────────────────────────────────────────── Confirm changes: [Y/n] Looking for: ['esgf-pyclient'] Pinned packages: - python 3.8.* Preparing transaction: ...working... done Verifying transaction: ...working... done Downloading xncml-0.2-py2.py3-none-any.whl (869 kB) ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 869.2/869.2 kB 8.3 MB/s eta 0:00:00 Requirement already satisfied: xarray in /opt/conda/envs/birdy/lib/python3.8/site-packages (from xncml) (2022.10.0) Collecting xmltodict Downloading xmltodict-0.13.0-py2.py3-none-any.whl (10.0 kB) Collecting xsdata Downloading xsdata-22.12-py3-none-any.whl (185 kB) ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 185.7/185.7 kB 18.0 MB/s eta 0:00:00 Requirement already satisfied: cftime in /opt/conda/envs/birdy/lib/python3.8/site-packages (from xncml) (1.6.2) Requirement already satisfied: netCDF4 in /opt/conda/envs/birdy/lib/python3.8/site-packages (from xncml) (1.6.2) Requirement already satisfied: dask in /opt/conda/envs/birdy/lib/python3.8/site-packages (from xncml) (2022.11.1) Requirement already satisfied: numpy>1.13.3 in /opt/conda/envs/birdy/lib/python3.8/site-packages (from cftime->xncml) (1.23.5) Requirement already satisfied: pyyaml>=5.3.1 in /opt/conda/envs/birdy/lib/python3.8/site-packages (from dask->xncml) (6.0) Requirement already satisfied: fsspec>=0.6.0 in /opt/conda/envs/birdy/lib/python3.8/site-packages (from dask->xncml) (2022.11.0) Requirement already satisfied: click>=7.0 in /opt/conda/envs/birdy/lib/python3.8/site-packages (from dask->xncml) (8.1.3) Requirement already satisfied: partd>=0.3.10 in /opt/conda/envs/birdy/lib/python3.8/site-packages (from dask->xncml) (1.3.0) Requirement already satisfied: packaging>=20.0 in /opt/conda/envs/birdy/lib/python3.8/site-packages (from dask->xncml) (21.3) Requirement already satisfied: toolz>=0.8.2 in /opt/conda/envs/birdy/lib/python3.8/site-packages (from dask->xncml) (0.12.0) Requirement already satisfied: cloudpickle>=1.1.1 in /opt/conda/envs/birdy/lib/python3.8/site-packages (from dask->xncml) (2.2.0) Requirement already satisfied: pandas>=1.3 in /opt/conda/envs/birdy/lib/python3.8/site-packages (from xarray->xncml) (1.3.5) Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/envs/birdy/lib/python3.8/site-packages (from packaging>=20.0->dask->xncml) (3.0.9) Requirement already satisfied: python-dateutil>=2.7.3 in /opt/conda/envs/birdy/lib/python3.8/site-packages (from pandas>=1.3->xarray->xncml) (2.8.2) Requirement already satisfied: pytz>=2017.3 in /opt/conda/envs/birdy/lib/python3.8/site-packages (from pandas>=1.3->xarray->xncml) (2022.6) Requirement already satisfied: locket in /opt/conda/envs/birdy/lib/python3.8/site-packages (from partd>=0.3.10->dask->xncml) (1.0.0) Requirement already satisfied: six>=1.5 in /opt/conda/envs/birdy/lib/python3.8/site-packages (from python-dateutil>=2.7.3->pandas>=1.3->xarray->xncml) (1.16.0) Installing collected packages: xsdata, xmltodict, xncml Successfully installed xmltodict-0.13.0 xncml-0.2 xsdata-22.12 WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv Removing intermediate container 11f0bd71b9ff ---> 00cd0ac38b2b Step 5/5 : USER jenkins ---> Running in 9b00ba912ac4 ``` --- docker/Dockerfile.testing | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.testing b/docker/Dockerfile.testing index 9dc5527..8aad5b1 100644 --- a/docker/Dockerfile.testing +++ b/docker/Dockerfile.testing @@ -1,13 +1,20 @@ # For testing quickly without having to do a full rebuild. -FROM pavics/workflow-tests:220728 +FROM pavics/workflow-tests:221130 + +# To avoid error "PROJ: proj_create_from_database: Open of /opt/conda/envs/birdy/share/proj failed" +# This simulates a real `conda activate birdy`. +ENV PROJ_DATA="/opt/conda/envs/birdy/share/proj" USER root # Use 'update' for existing and 'install' for new package. # Keep same channel ordering to not revert anything. +# esgf-pyclient for pavics-sdi esgf-dap.ipynb, PR https://github.com/Ouranosinc/pavics-sdi/pull/269 +# xncml for gen_catalog refactoring, PR https://github.com/Ouranosinc/pavics-vdb/pull/46 RUN umask 0000 \ - && mamba install -c conda-forge -c cdat -c bokeh -c plotly -c pyston -c pyviz/label/dev -c defaults -n birdy geopy + && mamba install -c conda-forge -c cdat -c bokeh -c plotly -c pyston -c pyviz/label/dev -c defaults -n birdy esgf-pyclient \ + && pip install xncml # && pip uninstall -y ravenpy \ # && mamba install -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy ravenpy aiohttp From dc474ecf1be9b4e50a7018ab78a43a4dbb4e6e4b Mon Sep 17 00:00:00 2001 From: Long Vu Date: Mon, 3 Apr 2023 16:57:44 -0400 Subject: [PATCH 2/3] release: update to use image pavics/workflow-tests:221130-update230403 --- Jenkinsfile | 2 +- binder/Dockerfile | 2 +- launchcontainer | 2 +- launchnotebook | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3c00e67..32d5cc6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,7 @@ pipeline { // https://jenkins.io/doc/book/pipeline/syntax/ agent { docker { - image "pavics/workflow-tests:221130" + image "pavics/workflow-tests:221130-update230403" label 'linux && docker' } } diff --git a/binder/Dockerfile b/binder/Dockerfile index 19bc1bf..59d9d24 100644 --- a/binder/Dockerfile +++ b/binder/Dockerfile @@ -1,4 +1,4 @@ -FROM pavics/workflow-tests:221130 +FROM pavics/workflow-tests:221130-update230403 USER root diff --git a/launchcontainer b/launchcontainer index ace0595..313c0e9 100755 --- a/launchcontainer +++ b/launchcontainer @@ -1,7 +1,7 @@ #!/bin/sh -x if [ -z "$DOCKER_IMAGE" ]; then - DOCKER_IMAGE="pavics/workflow-tests:221130" + DOCKER_IMAGE="pavics/workflow-tests:221130-update230403" fi if [ -z "$CONTAINER_NAME" ]; then diff --git a/launchnotebook b/launchnotebook index e899603..cfd758e 100755 --- a/launchnotebook +++ b/launchnotebook @@ -7,7 +7,7 @@ if [ -z "$PORT" ]; then fi if [ -z "$DOCKER_IMAGE" ]; then - DOCKER_IMAGE="pavics/workflow-tests:221130" + DOCKER_IMAGE="pavics/workflow-tests:221130-update230403" fi if [ -z "$CONTAINER_NAME" ]; then From e665901a4336b2d90cfdd898ce3f6cf1655a79f6 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Mon, 3 Apr 2023 17:05:06 -0400 Subject: [PATCH 3/3] jenkins: lower build retention since manual builds are done on a separated job --- Jenkinsfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 32d5cc6..6784ef0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -136,9 +136,8 @@ Note this is another run, will double the time and no guaranty to have same erro ansiColor('xterm') timestamps() timeout(time: 2, unit: 'HOURS') - // trying to keep 2 months worth of history with buffer for manual - // build trigger on failed builds or manual test after each production - // deployment or test deployment - buildDiscarder(logRotator(numToKeepStr: '200')) + // trying to keep 3 months worth of history + // assuming manual build requests are done on a separated job + buildDiscarder(logRotator(numToKeepStr: '100')) } }