replace individual plugin start bash scripts with ONE general one, wh…

.github/workflows/buildContainer.yaml

@@ -49,9 +49,9 @@ jobs:
         run: |
           VERSION=$(head -n 1 Images/${{ matrix.container }}/${{ matrix.container }}.dockerfile | cut -d ":" -f 2- | tr -d " ")
           echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV
-          cp Images/${{ matrix.container }}/start_${{ matrix.container }}.sh Images/test_plugin.sh Images/nginx/nginx_qiita.conf Images/${{ matrix.container }}/requirements.txt .
+          cp Images/start_plugin.sh Images/test_plugin.sh Images/nginx/nginx_qiita.conf Images/${{ matrix.container }}/requirements.txt Images/nginx/start_nginx.sh .
           cp Images/qiita/config_portal.cfg Images/qiita/config_qiita_oidc.cfg Images/qiita/drop_workflows.py Images/qiita/start_plugin.py Images/qiita/start_qiita-initDB.sh Images/qiita/start_qiita.sh .
-          cp Images/plugin_collector/collect_configs.py Images/plugin_collector/fix_test_db.py Images/plugin_collector/stefan_cert.conf Images/plugin_collector/stefan_csr.conf .
+          cp Images/plugin_collector/collect_configs.py Images/plugin_collector/fix_test_db.py Images/plugin_collector/stefan_cert.conf Images/plugin_collector/stefan_csr.conf Images/plugin_collector/start_plugin_collector.sh .
 
       - name: Download certificates from job build_main
         uses: actions/download-artifact@v4
@@ -118,7 +118,7 @@ jobs:
         run: |
           VERSION=$(head -n 1 Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile | cut -d ":" -f 2- | tr -d " ")
           echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV
-          cp Images/${{ matrix.plugin }}/start_${{ matrix.plugin }}.sh Images/test_plugin.sh Images/${{ matrix.plugin }}/requirements.txt Images/trigger_noconda.py Images/trigger.py .
+          cp Images/start_plugin.sh Images/test_plugin.sh Images/${{ matrix.plugin }}/requirements.txt Images/trigger_noconda.py Images/trigger.py .
 
       - name: Download certificates from job build_main
         uses: actions/download-artifact@v4
@@ -193,7 +193,7 @@ jobs:
         run: |
           VERSION=$(head -n 1 Images/${{ matrix.multiplugin }}/${{ matrix.multiplugin }}.dockerfile | cut -d ":" -f 2- | tr -d " ")
           echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV
-          cp Images/${{ matrix.multiplugin }}/start_${{ matrix.multiplugin }}.sh Images/test_plugin.sh Images/${{ matrix.multiplugin }}/requirements.txt Images/trigger_noconda.py Images/trigger.py .
+          cp Images/start_plugin.sh Images/test_plugin.sh Images/${{ matrix.multiplugin }}/requirements.txt Images/trigger_noconda.py Images/trigger.py .
 
       - name: Download certificates from job build_main
         uses: actions/download-artifact@v4
@@ -223,6 +223,12 @@ jobs:
           cache-from: type=gha,scope=tinqiita-${{ github.ref_name }}
           cache-to: type=gha,scope=tinqiita-${{ github.ref_name }},mode=max
 
+      - name: Prune build cache after build (saving disc space)
+        run: |
+          df -h /
+          docker buildx prune -af
+          df -h /
+
       - name: adapt compose file to select specific plugin
         run: |
           if [[ "${{ matrix.multiplugin }}" == "qp-qiime2" ]]; then sed -i 's|- QIITA_PLUGINS="MATRIXPLUGIN:"|- QIITA_PLUGINS="${{ matrix.multiplugin }}:qtp-diversity:qtp-visualization:"|' compose_github.yaml; sed -i 's|MATRIXPLUGIN:|${{ matrix.multiplugin }}:\n        condition: service_started\n      qtp-diversity:\n        condition: service_started\n      qtp-visualization:|' compose_github.yaml; fi;
@@ -241,12 +247,8 @@ jobs:
 
       - name: Push image to ghcr (only if tests passed)
         if: success()
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          push: true
-          file: Images/${{ matrix.multiplugin }}/${{ matrix.multiplugin }}.dockerfile
-          tags: ghcr.io/${{ github.repository }}/${{ matrix.multiplugin }}:testcandidate
+        run: |
+          docker push ghcr.io/${{ github.repository }}/${{ matrix.multiplugin }}:testcandidate
 
   publish_images:
     needs:

Images/qp-deblur/qp-deblur.dockerfile

@@ -110,8 +110,8 @@ COPY trigger_noconda.py /trigger.py
 
 WORKDIR /
 
-COPY start_qp-deblur.sh .
-RUN chmod 755 start_qp-deblur.sh
+COPY start_plugin.sh .
+RUN chmod 755 start_plugin.sh
 
 RUN mkdir -p /unshared_plugins
 ENV QIITA_PLUGINS_DIR=/unshared_plugins/
@@ -139,5 +139,5 @@ RUN sed -i "s|source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/cond
 # for testing
 COPY test_plugin.sh /test_plugin.sh
 
-CMD ["./start_qp-deblur.sh"]
+CMD ["./start_plugin.sh"]
 # ^^ 848 MB

Images/qp-qiime2/qp-qiime2.dockerfile

@@ -86,8 +86,8 @@ RUN dpkg-reconfigure -f noninteractive tzdata
 
 WORKDIR /
 
-COPY start_qp-qiime2.sh .
-RUN chmod 755 start_qp-qiime2.sh
+COPY start_plugin.sh .
+RUN chmod 755 start_plugin.sh
 
 RUN mkdir -p /unshared_plugins
 ENV QIITA_PLUGINS_DIR=/unshared_plugins/
@@ -108,4 +108,4 @@ RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qp-q
 # for testing
 COPY test_plugin.sh /test_plugin.sh
 
-CMD ["./start_qp-qiime2.sh"]
+CMD ["./start_plugin.sh"]

Images/qp-target-gene/qp-target-gene.dockerfile

@@ -105,8 +105,8 @@ COPY --from=builder /opt/conda/envs/qp-target-gene/lib/libpython2.7.so.1.0 /usr/
 # "install" pigz
 COPY --from=builder /opt/conda/envs/qp-target-gene/bin/pigz /usr/local/bin/
 
-COPY start_qp-target-gene.sh .
-RUN chmod 755 start_qp-target-gene.sh
+COPY start_plugin.sh .
+RUN chmod 755 start_plugin.sh
 
 RUN mkdir -p /unshared_plugins
 ENV QIITA_PLUGINS_DIR=/unshared_plugins/
@@ -133,4 +133,4 @@ RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qp-t
 # for testing
 COPY test_plugin.sh /test_plugin.sh
 
-CMD ["./start_qp-target-gene.sh"]
+CMD ["./start_plugin.sh"]

Images/qtp-biom/qtp-biom.dockerfile

@@ -113,7 +113,7 @@ RUN conda install cython
 RUN pip wheel --no-cache-dir --wheel-dir /wheels -r requirements.txt
 RUN pip install iow
 
-CMD ["./start_qtp-biom.sh"]
+CMD ["./start_plugin.sh"]
 
 # ==========================
 # Stage 2: Runtime
@@ -139,8 +139,8 @@ COPY trigger_noconda.py /trigger.py
 
 WORKDIR /
 
-COPY start_qtp-biom.sh .
-RUN chmod 755 start_qtp-biom.sh
+COPY start_plugin.sh .
+RUN chmod 755 start_plugin.sh
 
 RUN mkdir -p /unshared_plugins
 ENV QIITA_PLUGINS_DIR=/unshared_plugins/
@@ -173,6 +173,6 @@ RUN sed -i "s/'display.max_colwidth', -1/'display.max_colwidth', None/" /usr/loc
 # for testing
 COPY test_plugin.sh /test_plugin.sh
 
-CMD ["./start_qtp-biom.sh"]
+CMD ["./start_plugin.sh"]
 
 # python -c "import qiime2.plugins.feature_table"

Images/qtp-diversity/qtp-diversity.dockerfile

@@ -112,8 +112,8 @@ COPY --from=builder /wheels /wheels
 RUN pip install --no-cache-dir /wheels/* \
 	&& rm -rf rm -rf `find /usr/local/lib/python3.8/site-packages -type d -name "tests" | grep -v numpy`
 
-COPY start_qtp-diversity.sh .
-RUN chmod 755 start_qtp-diversity.sh
+COPY start_plugin.sh .
+RUN chmod 755 start_plugin.sh
 
 RUN mkdir -p /unshared_plugins
 ENV QIITA_PLUGINS_DIR=/unshared_plugins/
@@ -144,11 +144,11 @@ COPY --from=builder /opt/conda/envs/qiime2/lib/libhdf5_cpp.so.103.2.0    /usr/lo
 COPY --from=builder /opt/conda/envs/qiime2/lib/libhdf5_hl_cpp.so.100.1.4 /usr/local/lib/libhdf5_hl_cpp.so.100
 COPY --from=builder /opt/conda/envs/qiime2/lib/libhdf5_hl.so.100.1.3     /usr/local/lib/libhdf5_hl.so.100
 COPY --from=builder /opt/conda/envs/qiime2/lib/libhdf5.so.103.2.0        /usr/local/lib/libhdf5.so.103
-COPY --from=builder /opt/conda/envs/qiime2/lib/libcrypto.so.1.1          /usr/local/lib/libcrypto.so.1.1
+COPY --from=builder /opt/conda/envs/qiime2/lib/libcrypto.so*             /usr/local/lib/
 COPY --from=builder /opt/conda/envs/qiime2/lib/libcurl.so.4.8.0          /usr/local/lib/libcurl.so.4
-COPY --from=builder /opt/conda/envs/qiime2/lib/libnghttp2.so.14.24.1     /usr/local/lib/libnghttp2.so.14
+COPY --from=builder /opt/conda/envs/qiime2/lib/libnghttp2.so.*           /usr/local/lib/
 COPY --from=builder /opt/conda/envs/qiime2/lib/libssh2.so.1.0.1          /usr/local/lib/libssh2.so.1
-COPY --from=builder /opt/conda/envs/qiime2/lib/libssl.so.1.1             /usr/local/lib/libssl.so.1.1
+COPY --from=builder /opt/conda/envs/qiime2/lib/libssl.so*                /usr/local/lib/
 RUN ln -s /usr/local/lib/libopenblasp-r0.3.25.so /usr/local/lib/libcblas.so.3
 RUN ln -s /usr/local/lib/libopenblasp-r0.3.25.so /usr/local/lib/liblapacke.so.3
 RUN for f in `echo "libssu.so libhdf5_cpp.so.103 liblapacke.so.3 libcblas.so.3 libhdf5_hl_cpp.so.100 libhdf5_hl.so.100 libhdf5.so.103 libcrypto.so.1.1 libcurl.so.4 libgfortran.so.5 libnghttp2.so.14 libssh2.so.1 libssl.so.1.1 libquadmath.so.0"`; do ln -s /usr/local/lib/$f /lib/x86_64-linux-gnu/$f; done
@@ -180,4 +180,4 @@ RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qtp-
 # for testing
 COPY test_plugin.sh /test_plugin.sh
 
-CMD ["./start_qtp-diversity.sh"]
+CMD ["./start_plugin.sh"]

Images/qtp-job-output-folder/qtp-job-output-folder.dockerfile

@@ -76,8 +76,8 @@ RUN pip install --no-cache-dir /wheels/* \
 
 COPY trigger_noconda.py /trigger.py
 
-COPY start_qtp-job-output-folder.sh .
-RUN chmod 755 start_qtp-job-output-folder.sh
+COPY start_plugin.sh .
+RUN chmod 755 start_plugin.sh
 
 RUN mkdir -p /unshared_plugins
 ENV QIITA_PLUGINS_DIR=/unshared_plugins/
@@ -96,4 +96,4 @@ RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qtp-
 # for testing
 COPY test_plugin.sh /test_plugin.sh
 
-CMD ["./start_qtp-job-output-folder.sh"]
+CMD ["./start_plugin.sh"]

Images/qtp-sequencing/qtp-sequencing.dockerfile

@@ -85,8 +85,8 @@ RUN ln -s /usr/local/bin/quast.py /usr/local/bin/quast
 
 # WORKDIR /
 
-COPY start_qtp-sequencing.sh .
-RUN chmod 755 start_qtp-sequencing.sh
+COPY start_plugin.sh .
+RUN chmod 755 start_plugin.sh
 
 RUN mkdir -p /unshared_plugins
 ENV QIITA_PLUGINS_DIR=/unshared_plugins/
@@ -112,4 +112,4 @@ RUN mkdir -p /usr/share/man/man1 && \
 # for testing
 COPY test_plugin.sh /test_plugin.sh
 
-CMD ["./start_qtp-sequencing.sh"]
+CMD ["./start_plugin.sh"]

Images/qtp-visualization/qtp-visualization.dockerfile

@@ -96,8 +96,8 @@ COPY --from=builder /wheels /wheels
 RUN pip install --no-cache-dir /wheels/* \
 	&& rm -rf rm -rf `find /usr/local/lib/python3.8/site-packages -type d -name "tests" | grep -v numpy`
 
-COPY start_qtp-visualization.sh .
-RUN chmod 755 start_qtp-visualization.sh
+COPY start_plugin.sh .
+RUN chmod 755 start_plugin.sh
 
 RUN mkdir -p /unshared_plugins
 ENV QIITA_PLUGINS_DIR=/unshared_plugins/
@@ -119,4 +119,4 @@ RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qtp-
 # for testing
 COPY test_plugin.sh /test_plugin.sh
 
-CMD ["./start_qtp-visualization.sh"]
+CMD ["./start_plugin.sh"]

Images/start_plugin.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# proper listening and reacting to SIGTERM
+cleanup() {
+    echo "Received SIGTERM, stopping..."
+    kill "$PY_PID" 2>/dev/null
+    wait "$PY_PID"
+    exit 0
+}
+trap cleanup SIGTERM SIGINT
+
+STARTSCRIPT=
+if [ -z "$PLUGIN" ]; then
+    echo "Environment variable PLUGIN is empty. It should habe been set during the docker build process. Double check your dockerfile!"
+    exit 1
+fi
+case "$PLUGIN" in
+    qp-deblur)
+        STARTSCRIPT=start_deblur
+        ;;
+    qp-qiime2)
+        STARTSCRIPT=start_qiime2
+        ;;
+    qp-target-gene)
+        STARTSCRIPT=start_target_gene
+        ;;
+    qtp-biom)
+        STARTSCRIPT=start_biom
+        ;;
+    qtp-diversity)
+        STARTSCRIPT=start_diversity_types
+        ;;
+    qtp-job-output-folder)
+        STARTSCRIPT=start_qtp_job_output_folder
+        ;;
+    qtp-sequencing)
+        STARTSCRIPT=start_qtp_sequencing
+        ;;
+    qtp-visualization)
+        STARTSCRIPT=start_visualization_types
+        ;;
+    *)
+        echo "unknown qiita plugin $PLUGIN"
+        exit 1
+        ;;
+esac
+
+# start tornado server and safe PID in variable
+cd /
+if [ "$PLUGIN" = "qp-qiime2" ]; then
+    # as this plugin still uses a conda environment
+    python trigger.py qiime2 $STARTSCRIPT /qp-qiime2 &
+elif [ "$PLUGIN" = "qp-target-gene" ]; then
+    # as this plugin uses old py27
+    python3 trigger.py $STARTSCRIPT &
+else
+    python trigger.py $STARTSCRIPT &
+fi
+PY_PID=$!
+
+# wait till trigger.py process is terminated
+wait "$PY_PID"