Merge pull request #59 from Gregor-Mendel-Institute/1.3.1

Merge 1.3.1 > master
Gregor-Mendel-Institute · May 14, 2021 · 00f5ead · 00f5ead
2 parents 749f4a2 + 01150b7
commit 00f5ead
Show file tree

Hide file tree

Showing 26 changed files with 130 additions and 80 deletions.
diff --git a/.github/workflows/docker_base.yml b/.github/workflows/docker_base.yml
@@ -21,7 +21,7 @@ jobs:
         run: podman login -u ${QUAY_USER} -p ${QUAY_PASSWORD} quay.io
 
       - name: Build container
-        run: podman build --tag beckerlab/aradeepopsis-base containers/base
+        run: podman build --format docker --tag beckerlab/aradeepopsis-base containers/base
 
       - name: Push container
         run: podman push beckerlab/aradeepopsis-base docker://quay.io/beckerlab/aradeepopsis-base:${GITHUB_REF##*/}
diff --git a/.github/workflows/docker_dpp.yml b/.github/workflows/docker_dpp.yml
@@ -21,7 +21,7 @@ jobs:
         run: podman login -u ${QUAY_USER} -p ${QUAY_PASSWORD} quay.io
 
       - name: Build container
-        run: podman build --tag beckerlab/aradeepopsis-dpp containers/addons/dpp
+        run: podman build --format docker --tag beckerlab/aradeepopsis-dpp containers/addons/dpp
 
       - name: Push container
         run: podman push beckerlab/aradeepopsis-dpp docker://quay.io/beckerlab/aradeepopsis-dpp:${GITHUB_REF##*/}
diff --git a/.github/workflows/docker_shiny.yml b/.github/workflows/docker_shiny.yml
@@ -21,7 +21,7 @@ jobs:
         run: podman login -u ${QUAY_USER} -p ${QUAY_PASSWORD} quay.io
 
       - name: Build container
-        run: podman build --tag beckerlab/aradeepopsis-shiny containers/shiny
+        run: podman build --format docker --tag beckerlab/aradeepopsis-shiny containers/shiny
 
       - name: Push container
         run: podman push beckerlab/aradeepopsis-shiny docker://quay.io/beckerlab/aradeepopsis-shiny:${GITHUB_REF##*/}
diff --git a/.github/workflows/minimal_test.yml b/.github/workflows/minimal_test.yml
@@ -11,18 +11,54 @@ on:
 jobs:
   CI:
     env:
+      PROFILE: ${{ matrix.engine }}
       NXF_VER: ${{ matrix.nxf_ver }}
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        nxf_ver: ["20.07.1", ""]
+        engine: ["docker", "podman", "charliecloud", "conda"]
+        nxf_ver: ["20.07.1",""]
+        os: ["ubuntu-latest", "macos-latest"]
+        exclude:
+          - engine: "charliecloud"
+            nxf_ver: "20.07.1" # charliecloud needs 21.04.0 or later
+          - os: "macos-latest"
+            engine: "docker"
+          - os: "macos-latest"
+            engine: "podman"
+          - os: "macos-latest"
+            engine: "charliecloud"
     steps:
       - uses: actions/checkout@v2
 
+      - name: Set up Python
+        if: ${{ matrix.engine == 'charliecloud' }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+
+      - name: Install Charliecloud
+        if: ${{ matrix.engine == 'charliecloud' }}
+        run: |
+          python -m pip install lark-parser requests
+          wget -qO- https://github.com/hpc/charliecloud/releases/download/v0.22/charliecloud-0.22.tar.gz | tar -xvz
+          cd charliecloud-0.22
+          ./configure
+          make
+          sudo make install
+
       - name: Install Nextflow
         run: |
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
-      - name: Run aradeepopsis on test images
+
+      - name: Run ARADEEPOPSIS on ubuntu
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: |
+          nextflow run ${GITHUB_WORKSPACE} --save_rosette --save_histogram --save_hull --summary_diagnostics --shiny false -profile test,${PROFILE}
+
+      - name: Run ARADEEPOPSIS on macOS
+        if: ${{ matrix.os == 'macos-latest' }}
         run: |
-          nextflow run ${GITHUB_WORKSPACE} --save_rosette --save_histogram --save_hull --summary_diagnostics --shiny false -profile test,docker
+          sudo chown -R ${UID} ${CONDA}
+          nextflow run ${GITHUB_WORKSPACE} --save_rosette --save_histogram --save_hull --summary_diagnostics --shiny false -profile test,conda
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,19 @@
 # Changelog
 
+## [v1.3.1](https://github.com/Gregor-Mendel-Institute/aradeepopsis/releases/tag/v1.3.1) - 2021-05-14
+
+* fixed illegible text in Markdown documents when browsing Github in Dark Mode (thanks @greymonroe for pointing this out)
+* added config profile for BioHPC Genomics cluster at LRZ
+* removed unneeded `tensorflow-estimator` dependency
+* updated `tensorflow-base` `2.0.0` > `2.4.1`
+* updated `imagemagick` dependency `7.0.10_28` > `7.0.11_12`
+* updated `shiny` dependency `1.5.0` > `1.6.0`
+* updated `tidyverse` dependency `1.3.0` > `1.3.1`
+* updated `shinythemes` dependency `1.1.2` > `1.2.0`
+* updated `corrplot` dependency `0.84` > `0.88`
+* changed container base to micromamba for smaller container images and faster builds
+* added CI tests for `conda` profile on macOS and additional container engines on linux (`charliecloud` and `podman`)
+
 ## [v1.3](https://github.com/Gregor-Mendel-Institute/aradeepopsis/releases/tag/v1.3) - 2021-01-11
 
 * changed container registry from docker.io to quay.io

diff --git a/README.md b/README.md
@@ -4,8 +4,12 @@
 ![Latest Release](https://img.shields.io/github/v/release/Gregor-Mendel-Institute/aradeepopsis?sort=semver)
 ![License](https://img.shields.io/github/license/Gregor-Mendel-Institute/aradeepopsis)
 [![DOI](https://zenodo.org/badge/240287274.svg)](https://zenodo.org/badge/latestdoi/240287274)
-[![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A520.07.1-important.svg)](https://www.nextflow.io/)
-[![conda](https://img.shields.io/badge/install%20with-conda-brightgreen.svg)](https://conda.io/)
+[![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A520.07.1-important.svg)](https://www.nextflow.io/)  
+[![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/)
+[![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/)
+[![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/)
+[![run with podman](https://img.shields.io/badge/run%20with-podman-1d355c.svg?labelColor=000000&logo=podman)](https://docs.podman.io/)
+[![run with charliecloud](https://img.shields.io/badge/run%20with-charliecloud-1d355c.svg?labelColor=000000)](https://hpc.github.io/charliecloud/)
 
 # Introduction
 
@@ -62,7 +66,7 @@ Running the pipeline with the provided models requires at least 6GB of memory fo
 
 1. Install [`Nextflow`](https://www.nextflow.io/index.html#GetStarted)
 
-2. Install either [`conda`](https://docs.conda.io/projects/conda/en/latest/user-guide/install/), [`Docker`](https://docs.docker.com/install/), [`podman`](https://podman.io/getting-started/installation) or [`Singularity`](https://sylabs.io/guides/3.0/user-guide/installation.html)
+2. Install either [`conda`](https://docs.conda.io/projects/conda/en/latest/user-guide/install/), [`Docker`](https://docs.docker.com/install/), [`podman`](https://podman.io/getting-started/installation), [`Charliecloud`](https://hpc.github.io/charliecloud/) or [`Singularity`](https://sylabs.io/guides/3.0/user-guide/installation.html).
 
 > Note: To run the pipeline on MacOS with Docker, it is necessary to increase the allowed memory usage from the default setting of 2GB to 6GB (See the [Docker documentation](https://docs.docker.com/docker-for-mac/#resources) for instructions)
 
@@ -71,7 +75,7 @@ Running the pipeline with the provided models requires at least 6GB of memory fo
 To run the pipeline you have to provide single-pot plant images:
 
 ```bash
-nextflow run Gregor-Mendel-Institute/aradeepopsis --images 'path/to/images/*{png|jpg}' -profile {conda|docker|podman|singularity}
+nextflow run Gregor-Mendel-Institute/aradeepopsis --images 'path/to/images/*{png|jpg}' -profile {conda|docker|podman|singularity|charliecloud}
 ```
 
 ### Example to run on the CBE cluster using Singularity

diff --git a/conf/base.config b/conf/base.config
@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2019-2020 Patrick Hüther
+Copyright (C) 2019-2021 Patrick Hüther
 
 This file is part of ARADEEPOPSIS.
 ARADEEPOPSIS is free software: you can redistribute it and/or modify
@@ -23,41 +23,43 @@ along with ARADEEPOPSIS.  If not, see <https://www.gnu.org/licenses/>.
 
 env {
   PYTHONPATH = "${PYTHONPATH}:$baseDir/bin"
-  TF_CPP_MIN_LOG_LEVEL = 2
+  PYTHONNOUSERSITE = 1
+  R_PROFILE_USER = "/.Rprofile"
+  R_ENVIRON_USER = "/.Renviron"
 }
 
 process {
   withName: build_records {
-    container = 'quay.io/beckerlab/aradeepopsis-base:1.3'
+    container = 'quay.io/beckerlab/aradeepopsis-base:1.3.1'
     cpus = 1
     memory = 2.GB
     time = 1.h
   }
   withName: run_predictions {
-    container = 'quay.io/beckerlab/aradeepopsis-base:1.3'
+    container = 'quay.io/beckerlab/aradeepopsis-base:1.3.1'
     cpus = 2
     memory = params.multiscale ? 12.GB : 6.GB
     time = 4.h
   }
   withName: run_predictions_DPP {
-    container = 'quay.io/beckerlab/aradeepopsis-dpp:1.3'
+    container = 'quay.io/beckerlab/aradeepopsis-dpp:1.3.1'
     cpus = 2
     memory = 2.GB
     time = 4.h
   }
   withName: extract_traits {
-    container = 'quay.io/beckerlab/aradeepopsis-base:1.3'
+    container = 'quay.io/beckerlab/aradeepopsis-base:1.3.1'
     cpus = 1
     memory = 2.GB
     time = 1.h
   }
   withName: draw_diagnostics {
-    container = 'quay.io/beckerlab/aradeepopsis-base:1.3'
+    container = 'quay.io/beckerlab/aradeepopsis-base:1.3.1'
     cpus = 1
     memory = 2.GB
     time = 1.h
   }
   withName: launch_shiny {
-    container = 'quay.io/beckerlab/aradeepopsis-shiny:1.3'
+    container = 'quay.io/beckerlab/aradeepopsis-shiny:1.3.1'
   }
 }
diff --git a/conf/lrz.config → conf/biohpc_gen.config b/conf/lrz.config → conf/biohpc_gen.config
@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2019-2020 Patrick Hüther, Niklas Schandry
+Copyright (C) 2019-2021 Patrick Hüther, Niklas Schandry
 
 This file is part of ARADEEPOPSIS.
 ARADEEPOPSIS is free software: you can redistribute it and/or modify
@@ -17,17 +17,23 @@ along with ARADEEPOPSIS.  If not, see <https://www.gnu.org/licenses/>.
 */
 
 /* -------------------------------------------------
- * Nextflow config file for LRZ Linux Cluster (CM2) 
+ * Nextflow config file for BioHPC Genomics
  * -------------------------------------------------
  */
 
+env {
+  SLURM_CLUSTERS='biohpc_gen'
+}
+
+charliecloud {
+  enabled = true
+}
+
 process {
   executor = 'slurm'
-  queue = 'cm2_std'
-  queueSize = 50
-  clusterOptions = "--clusters cm2 --qos cm2_std"
-  errorStrategy = { task.exitStatus in [104,134,135,137,139,140,143] ? 'retry' : 'finish' }
-  maxRetries = 3
+  queue = { task.memory <= 1536.GB ? (task.time > 2.d || task.memory > 384.GB ? 'biohpc_gen_production' : 'biohpc_gen_normal') : 'biohpc_gen_highmem' }
+  beforeScript = 'module use /dss/dsslegfs02/pn73se/pn73se-dss-0000/spack/modules/x86_avx2/linux*'
+  module = 'charliecloud/0.22:miniconda3'
 
   withName: build_records {
     cpus = { 1 * task.attempt }

diff --git a/conf/cbe.config b/conf/cbe.config
@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2019-2020 Patrick Hüther
+Copyright (C) 2019-2021 Patrick Hüther
 
 This file is part of ARADEEPOPSIS.
 ARADEEPOPSIS is free software: you can redistribute it and/or modify
@@ -27,7 +27,7 @@ process {
   executor = 'slurm'
   module = 'anaconda3/2019.10'
   queue = 'c'
-  clusterOptions = { task.time <= 8.h ? '--qos short': task.time <= 48.h ? '--qos medium' : '--qos long' }
+  clusterOptions = { task.time <= 1.h ? '--qos rapid' : task.time <= 8.h ? '--qos short': task.time <= 48.h ? '--qos medium' : '--qos long' }
 
   errorStrategy = { task.exitStatus in [104,134,135,137,139,140,143] ? 'retry' : 'finish' }
   maxRetries = 3

diff --git a/conf/charliecloud.config b/conf/charliecloud.config
@@ -21,15 +21,10 @@ along with ARADEEPOPSIS.  If not, see <https://www.gnu.org/licenses/>.
  * -------------------------------------------------
  */
 
-env {
-  //for charliecloud, the PATH environment within the Docker image needs to be passed explicitly
-  PATH = "/opt/conda/envs/aradeepopsis-dpp-v1.3/bin:/opt/conda/envs/aradeepopsis-shiny-v1.3/bin:/opt/conda/envs/aradeepopsis-base-v1.3/bin:$PATH"
-}
-
 charliecloud {
   enabled = true
 }
 
 manifest {
-    nextflowVersion = '>=20.12.0-edge'
-}
+    nextflowVersion = '>=21.04.0'
+}
diff --git a/conf/conda.config b/conf/conda.config
@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2019-2020 Patrick Hüther
+Copyright (C) 2019-2021 Patrick Hüther
 
 This file is part of ARADEEPOPSIS.
 ARADEEPOPSIS is free software: you can redistribute it and/or modify
@@ -21,10 +21,6 @@ along with ARADEEPOPSIS.  If not, see <https://www.gnu.org/licenses/>.
  * -------------------------------------------------
  */
 
-env {
-  PYTHONPATH = "${PYTHONPATH}:$baseDir/bin"
-}
-
 process {
   withName: build_records {
     conda = "$baseDir/containers/base/environment.yml"

diff --git a/containers/addons/dpp/Dockerfile b/containers/addons/dpp/Dockerfile
@@ -1,8 +1,9 @@
-FROM continuumio/miniconda3
+FROM mambaorg/micromamba:0.13.0
 LABEL authors="patrick.huether@gmi.oeaw.ac.at" \
     description="Container image containing DPP dependencies for ARADEEPOPSIS"
 
 COPY environment.yml /
-RUN apt-get update && apt-get install -y procps graphviz && apt-get clean -y
-RUN conda env create -f /environment.yml && conda clean -afy
-ENV PATH /opt/conda/envs/aradeepopsis-dpp-v1.3/bin:$PATH
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y procps \
+    && micromamba install -y -n base -f /environment.yml \
+    && micromamba clean -a && rm -rf /var/lib/{apt,dpkg,cache,log}
diff --git a/containers/addons/dpp/environment.yml b/containers/addons/dpp/environment.yml
@@ -1,15 +1,10 @@
-name: aradeepopsis-dpp-v1.3
+name: aradeepopsis-dpp
 channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.7
-  - pip
-  - conda-forge::scipy=1.5.0
-  - conda-forge::opt-einsum=3.0.0
-  - conda-forge::tqdm=4.47.0
-  - conda-forge::pillow=7.1.2
+  - conda-forge::python=3.7
+  - conda-forge::pip=21.1.1
   - pip:
-    - tensorflow==1.15
-    - opencv-python==4.2.0.34
-    - https://github.com/p2irc/deepplantphenomics/archive/2.1.0.tar.gz
+    - opencv-python-headless==4.5.2.52
+    - https://github.com/p2irc/deepplantphenomics/archive/2.1.0.tar.gz
diff --git a/containers/base/Dockerfile b/containers/base/Dockerfile
@@ -1,8 +1,9 @@
-FROM continuumio/miniconda3
+FROM mambaorg/micromamba:0.13.0
 LABEL authors="patrick.huether@gmi.oeaw.ac.at" \
-    description="Container image containing base dependencies for ARADEEPOPSIS"
+    description="Container image containing DPP dependencies for ARADEEPOPSIS"
 
 COPY environment.yml /
-RUN apt-get update && apt-get install -y procps graphviz && apt-get clean -y
-RUN conda env create -f /environment.yml && conda clean -afy
-ENV PATH /opt/conda/envs/aradeepopsis-base-v1.3/bin:$PATH
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y procps \
+    && micromamba install -y -n base -f /environment.yml \
+    && micromamba clean -a && rm -rf /var/lib/{apt,dpkg,cache,log}
diff --git a/containers/base/environment.yml b/containers/base/environment.yml
@@ -1,9 +1,8 @@
-name: aradeepopsis-base-v1.3
+name: aradeepopsis-base
 channels:
   - conda-forge
   - defaults
 dependencies:
-  - anaconda::tensorflow-base=2.0.0
-  - anaconda::tensorflow-estimator=2.0.0
+  - conda-forge::tensorflow-base=2.4.1
   - conda-forge::scikit-image=0.18.1
-  - conda-forge::imagemagick=7.0.10_28
+  - conda-forge::imagemagick=7.0.11_12
diff --git a/containers/shiny/Dockerfile b/containers/shiny/Dockerfile
@@ -1,10 +1,11 @@
-FROM continuumio/miniconda3
+FROM mambaorg/micromamba:0.13.0
 LABEL authors="patrick.huether@gmi.oeaw.ac.at" \
-    description="Container image containing shiny dependencies for ARADEEPOPSIS"
+    description="Container image containing DPP dependencies for ARADEEPOPSIS"
 
 COPY environment.yml /
-RUN apt-get update && apt-get install -y procps graphviz && apt-get clean -y
-RUN conda env create -f /environment.yml && conda clean -afy
-ENV PATH /opt/conda/envs/aradeepopsis-shiny-v1.3/bin:$PATH
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y procps \
+    && micromamba install -y -n base -f /environment.yml \
+    && micromamba clean -a && rm -rf /var/lib/{apt,dpkg,cache,log}
 
-EXPOSE 44333
+EXPOSE 44333
diff --git a/containers/shiny/environment.yml b/containers/shiny/environment.yml
@@ -1,14 +1,14 @@
-name: aradeepopsis-shiny-v1.3
+name: aradeepopsis-shiny
 channels:
   - conda-forge
   - r
   - defaults
 dependencies:
-  - conda-forge::r-tidyverse=1.3.0
-  - conda-forge::r-shiny=1.5.0
-  - conda-forge::r-shinythemes=1.1.2
+  - conda-forge::r-tidyverse=1.3.1
+  - conda-forge::r-shiny=1.6.0
+  - conda-forge::r-shinythemes=1.2.0
   - conda-forge::r-shinycssloaders=1.0.0
   - conda-forge::r-slickr=0.5.0
-  - conda-forge::r-corrplot=0.84
+  - conda-forge::r-corrplot=0.88
   - conda-forge::r-jpeg=0.1_8.1
-  - r::r-radarchart=0.3.1
+  - r::r-radarchart=0.3.1
diff --git a/docs/img/BGR.png b/docs/img/BGR.png
diff --git a/docs/img/CB.png b/docs/img/CB.png
diff --git a/docs/img/CD.png b/docs/img/CD.png
diff --git a/docs/img/CR.png b/docs/img/CR.png
diff --git a/docs/img/S_green.png b/docs/img/S_green.png
diff --git a/docs/img/pipeline_graph.png b/docs/img/pipeline_graph.png
diff --git a/docs/img/traits.png b/docs/img/traits.png