From 84a43c1c58554b278a8e00e4347e433c30628e2f Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 12 Sep 2025 08:56:42 +0200 Subject: [PATCH 1/4] add action for qp-qiime2 --- .github/workflows/buildContainer.yaml | 157 +++++++++++++------------- 1 file changed, 81 insertions(+), 76 deletions(-) diff --git a/.github/workflows/buildContainer.yaml b/.github/workflows/buildContainer.yaml index 19bcc2c..c1aa20d 100644 --- a/.github/workflows/buildContainer.yaml +++ b/.github/workflows/buildContainer.yaml @@ -3,9 +3,12 @@ name: docker on: + pull_request: + branches: + - tinqiita push: branches: - - 'restructure_github_actions' + - tinqiita jobs: make_certificates: @@ -170,89 +173,91 @@ jobs: file: Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile tags: ghcr.io/${{ github.repository }}/${{ matrix.plugin }}:testcandidate - # build_mulit_plugins: - # needs: - # - build_plugins - # strategy: - # matrix: - # plugin: ["qp-qiime2"] - # runs-on: ubuntu-latest - # steps: - # - name: Checkout tinqiita repo - # uses: actions/checkout@v4 - - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v3 - - # - name: Read image version from dockerfile - # id: vars - # run: | - # VERSION=$(head -n 1 Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile | cut -d ":" -f 2- | tr -d " ") - # echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV - # cp Images/${{ matrix.plugin }}/start_${{ matrix.plugin }}.sh Images/test_plugin.sh Images/${{ matrix.plugin }}/requirements.txt Images/trigger_noconda.py Images/trigger.py . - - # - name: Download certificates from job build_main - # uses: actions/download-artifact@v4 - # with: - # name: certificates - # path: ./ - # - name: Download fake references - # uses: actions/download-artifact@v4 - # with: - # name: fake_references - # path: ./ - - # - name: Log in to GitHub Container Registry - # uses: docker/login-action@v3 - # with: - # registry: ghcr.io - # username: ${{ github.actor }} - # password: ${{ secrets.GITHUB_TOKEN }} - - # - name: Build plugin images - # uses: docker/build-push-action@v6 - # with: - # context: . - # load: true - # file: Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile - # tags: ghcr.io/${{ github.repository }}/${{ matrix.plugin }}:testcandidate - # cache-from: type=gha,scope=tinqiita-${{ github.ref_name }} - # cache-to: type=gha,scope=tinqiita-${{ github.ref_name }},mode=max - - # - name: adapt compose file to select specific plugin - # run: | - # sed -i "s|||" compose_github.yaml - # sed -i "s/MATRIXPLUGIN/${{ matrix.plugin }}/g" compose_github.yaml - - # - name: Run docker compose - # uses: hoverkraft-tech/compose-action@v2.0.1 - # with: - # compose-file: "compose_github.yaml" - # services: | - # nginx - - # - name: Execute tests in the running services - # run: | - # sleep 5 - # docker compose exec ${{ matrix.plugin }} /bin/bash -c "bash /test_plugin.sh" - - # - name: Push image to ghcr (only if tests passed) - # if: success() - # uses: docker/build-push-action@v6 - # with: - # context: . - # push: true - # file: Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile - # tags: ghcr.io/${{ github.repository }}/${{ matrix.plugin }}:testcandidate + # the qp-qiime2 plugin cannot be tested in isolation, it also needs qtp-diversity and qtp-visualization to be active in qiita + build_mulit_plugins: + needs: + - build_plugins + strategy: + matrix: + multiplugin: ["qp-qiime2"] + runs-on: ubuntu-latest + steps: + - name: Checkout tinqiita repo + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Read image version from dockerfile + id: vars + run: | + VERSION=$(head -n 1 Images/${{ matrix.multiplugin }}/${{ matrix.multiplugin }}.dockerfile | cut -d ":" -f 2- | tr -d " ") + echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV + cp Images/${{ matrix.multiplugin }}/start_${{ matrix.multiplugin }}.sh Images/test_plugin.sh Images/${{ matrix.multiplugin }}/requirements.txt Images/trigger_noconda.py Images/trigger.py . + + - name: Download certificates from job build_main + uses: actions/download-artifact@v4 + with: + name: certificates + path: ./ + - name: Download fake references + uses: actions/download-artifact@v4 + with: + name: fake_references + path: ./ + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build plugin images + uses: docker/build-push-action@v6 + with: + context: . + load: true + file: Images/${{ matrix.multiplugin }}/${{ matrix.multiplugin }}.dockerfile + tags: ghcr.io/${{ github.repository }}/${{ matrix.multiplugin }}:testcandidate + cache-from: type=gha,scope=tinqiita-${{ github.ref_name }} + cache-to: type=gha,scope=tinqiita-${{ github.ref_name }},mode=max + + - name: adapt compose file to select specific plugin + run: | + if [[ "${{ matrix.multiplugin }}" == "qp-qiime2" ]]; then sed -i 's|- QIITA_PLUGINS="MATRIXPLUGIN:"|- QIITA_PLUGINS="${{ matrix.multiplugin }}:qtp-diversity:qtp-visualization:"|' compose_github.yaml; sed -i 's|MATRIXPLUGIN:|${{ matrix.multiplugin }}:\n condition: service_started\n qtp-diversity:\n condition: service_started\n qtp-visualization:|' compose_github.yaml; fi; + + - name: Run docker compose + uses: hoverkraft-tech/compose-action@v2.0.1 + with: + compose-file: "compose_github.yaml" + services: | + nginx + + - name: Execute tests in the running services + run: | + sleep 5 + docker compose exec ${{ matrix.multiplugin }} /bin/bash -c "bash /test_plugin.sh" + + - name: Push image to ghcr (only if tests passed) + if: success() + uses: docker/build-push-action@v6 + with: + context: . + push: true + file: Images/${{ matrix.multiplugin }}/${{ matrix.multiplugin }}.dockerfile + tags: ghcr.io/${{ github.repository }}/${{ matrix.multiplugin }}:testcandidate publish_images: needs: - build_plugins - build_main + - build_mulit_plugins strategy: matrix: image: ["qp-deblur", "qtp-biom", "qtp-sequencing", "qtp-visualization", "qtp-diversity", "qp-target-gene", "qtp-job-output-folder", - "nginx", "qiita", "plugin_collector"] + "nginx", "qiita", "plugin_collector", + "qp-qiime2"] runs-on: ubuntu-latest steps: - name: Checkout tinqiita repo From 12027b57b065c3f6be17dd6e36517abfe3a207c0 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 12 Sep 2025 12:29:43 +0200 Subject: [PATCH 2/4] adding misisng file --- Images/qp-qiime2/requirements.txt | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 Images/qp-qiime2/requirements.txt diff --git a/Images/qp-qiime2/requirements.txt b/Images/qp-qiime2/requirements.txt new file mode 100644 index 0000000..4493267 --- /dev/null +++ b/Images/qp-qiime2/requirements.txt @@ -0,0 +1,9 @@ +tornado +q2-umap +q2-greengenes2 +certifi +pip-system-certs + +-e /qiita_client +-e /qiita-files +-e /qp-qiime2 From 26ee53688cd91207ccb7263c935532a0583c4786 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 12 Sep 2025 13:36:18 +0200 Subject: [PATCH 3/4] some some gigs of image size by cleaning up conda --- Images/qp-qiime2/qp-qiime2.dockerfile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Images/qp-qiime2/qp-qiime2.dockerfile b/Images/qp-qiime2/qp-qiime2.dockerfile index 0cc7639..499f542 100644 --- a/Images/qp-qiime2/qp-qiime2.dockerfile +++ b/Images/qp-qiime2/qp-qiime2.dockerfile @@ -28,14 +28,16 @@ RUN wget https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_ # install tornado based trigger layer in base environment RUN pip install -U pip -RUN conda install tornado +RUN pip install tornado COPY trigger.py /trigger.py # Download qiime2 yaml RUN wget --quiet https://data.qiime2.org/distro/core/qiime2-2023.5-py38-linux-conda.yml # Create conda env -RUN conda env create --name qiime2 -y --file qiime2-2023.5-py38-linux-conda.yml +RUN conda env create --name qiime2 -y --file qiime2-2023.5-py38-linux-conda.yml \ + && conda clean --all -y \ + && rm -rf /opt/conda/pkgs # Make RUN commands use the new environment: # append --format docker to the build command, see https://github.com/containers/podman/issues/8477 SHELL ["conda", "run", "-p", "/opt/conda/envs/qiime2", "/bin/bash", "-c"] @@ -61,8 +63,9 @@ RUN export QP_QIIME2_DBS=/databases # configuring the filtering QZAs available for QIIME 2 RUN mkdir /filtering -RUN wget -O /filtering/bloom-analyses.zip https://github.com/knightlab-analyses/bloom-analyses/archive/refs/heads/master.zip -RUN unzip -j /filtering/bloom-analyses.zip bloom-analyses-master/data/qiime2-artifacts-for-qiita/*.qza -d /filtering/ +RUN wget -O /filtering/bloom-analyses.zip https://github.com/knightlab-analyses/bloom-analyses/archive/refs/heads/master.zip \ + && unzip -j /filtering/bloom-analyses.zip bloom-analyses-master/data/qiime2-artifacts-for-qiita/*.qza -d /filtering/ \ + && rm -f /filtering/bloom-analyses.zip RUN export QP_QIIME2_FILTER_QZA=/filtering/ # TODO: should the plugin get the server configuration?! From bac91fa4997fad2265e438dceed0fb936ed8b5cd Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 12 Sep 2025 14:29:18 +0200 Subject: [PATCH 4/4] activate conda env for qp-qiime2 --- Images/test_plugin.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Images/test_plugin.sh b/Images/test_plugin.sh index a7eaa10..2f03968 100644 --- a/Images/test_plugin.sh +++ b/Images/test_plugin.sh @@ -11,8 +11,10 @@ else REQUESTS_CA_BUNDLE="" pip install pytest; fi; -# clone plugin repository -git clone https://github.com/qiita-spots/${PLUGIN} +if [ "qp-qiime2" != "$PLUGIN" ]; then + # clone plugin repository + git clone https://github.com/qiita-spots/${PLUGIN} +fi; # NOTE: client api reset only works when communicating with Qitta Master, # thus, you need to directly address the port of the master container. Don't @@ -42,4 +44,8 @@ export QIITA_PORT=21174 export QIITA_ROOTCA_CERT=$SSL_CERT_FILE # change into plugin source directory and execute actual tests -cd ${PLUGIN} && pytest +if [ "qp-qiime2" == "$PLUGIN" ]; then + source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/conda/envs/qiime2; cd ${PLUGIN} && pytest; +else + cd ${PLUGIN} && pytest; +fi;