restructure github actions

.github/workflows/buildContainer.yaml

@@ -1,87 +1,292 @@
+# todo: "make" files once and push around through artifacts!
+
 name: docker
 
 on:
   push:
     branches:
-      - 'smaller_deblur'
+      - 'restructure_github_actions'
 
 jobs:
-  docker:
+  make_certificates:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout tinqiita repo
+        uses: actions/checkout@v4
+
+      - name: Create certificate
+        # second copy of "qiita_server_certificates" is necessary to match path for docker build, first copy for mounting into container
+        run: |
+          make ./references/qiita_server_certificates ./environments/qiita_db.env ./environments/qiita.env config
+          cp -r ./references/qiita_server_certificates ./qiita_server_certificates
+
+      - name: Store certifactes for follow up jobs
+        uses: actions/upload-artifact@v4
+        with:
+          name: certificates
+          path: |
+            ./qiita_server_certificates
+            ./references/qiita_server_certificates
+
+  build_main:
+    needs: make_certificates
     strategy:
       matrix:
-        plugin: ["qp-deblur", "qtp-biom", "qtp-sequencing", "qtp-visualization", "qtp-diversity", "qp-target-gene", "qtp-job-output-folder",
-                 "nginx", "qiita", "plugin_collector"
-                ]
+        container: ["nginx", "qiita", "plugin_collector"]
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout repository
+      - name: Checkout tinqiita repo
         uses: actions/checkout@v4
 
-      - name: Login to DockerHub
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Read image version from dockerfile
+        id: vars
+        run: |
+          VERSION=$(head -n 1 Images/${{ matrix.container }}/${{ matrix.container }}.dockerfile | cut -d ":" -f 2- | tr -d " ")
+          echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV
+          cp Images/${{ matrix.container }}/start_${{ matrix.container }}.sh Images/test_plugin.sh Images/nginx/nginx_qiita.conf Images/${{ matrix.container }}/requirements.txt .
+          cp Images/qiita/config_portal.cfg Images/qiita/config_qiita_oidc.cfg Images/qiita/drop_workflows.py Images/qiita/start_plugin.py Images/qiita/start_qiita-initDB.sh Images/qiita/start_qiita.sh .
+          cp Images/plugin_collector/collect_configs.py Images/plugin_collector/fix_test_db.py Images/plugin_collector/stefan_cert.conf Images/plugin_collector/stefan_csr.conf .
+
+      - name: Download certificates from job build_main
+        uses: actions/download-artifact@v4
+        with:
+          name: certificates
+          path: ./          
+
+      - name: Log in to GitHub Container Registry
         uses: docker/login-action@v3
         with:
-          registry: harbor.computational.bio.uni-giessen.de
-          username: ${{ vars.HARBOR_CB_USERNAME }}
-          password: ${{ secrets.HARBOR_CB_SECRET }}
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build main qiita images and push to github's own registry
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          file: Images/${{ matrix.container }}/${{ matrix.container }}.dockerfile
+          tags: ghcr.io/${{ github.repository }}/${{ matrix.container }}:testcandidate
+          cache-from: type=gha,scope=tinqiita-${{ github.ref_name }}
+          cache-to: type=gha,scope=tinqiita-${{ github.ref_name }},mode=max
+
+  make_references:
+    needs: build_main
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout tinqiita repo
+        uses: actions/checkout@v4
+
+      - name: Create partially fake reference databases
+        run: |
+          mkdir -p ./references/qp-deblur/ ./references/qp-target-gene
+          make ./environments/qiita_db.env ./environments/qiita.env config ./references/qp-deblur/reference-gg-raxml-bl.tre
+          for f in `echo "references/qp-target-gene/97_otus.fasta references/qp-target-gene/97_otus.tree references/qp-target-gene/97_otu_taxonomy.txt"`; do echo "fake" > $f; done
+
+      - name: Store fake references for follow up jobs
+        uses: actions/upload-artifact@v4
+        with:
+          name: fake_references
+          path: |
+            ./references/qp-deblur
+            ./references/qp-target-gene
+            ./environments
+
+  build_plugins:
+    needs:
+      - build_main
+      - make_references
+    strategy:
+      matrix:
+        plugin: ["qp-deblur", "qtp-biom", "qtp-sequencing", "qtp-visualization", "qtp-diversity", "qp-target-gene", "qtp-job-output-folder"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout tinqiita repo
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Read version from file
+      - name: Read image version from dockerfile
         id: vars
         run: |
           VERSION=$(head -n 1 Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile | cut -d ":" -f 2- | tr -d " ")
           echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV
-          cp Images/${{ matrix.plugin }}/start_${{ matrix.plugin }}.sh Images/test_plugin.sh Images/trigger_noconda.py Images/trigger.py Images/nginx/nginx_qiita.conf Images/${{ matrix.plugin }}/requirements.txt .
-          cp Images/qiita/config_portal.cfg Images/qiita/config_qiita_oidc.cfg Images/qiita/drop_workflows.py Images/qiita/start_plugin.py Images/qiita/start_qiita-initDB.sh Images/qiita/start_qiita.sh .
-          cp Images/plugin_collector/collect_configs.py Images/plugin_collector/fix_test_db.py Images/plugin_collector/stefan_cert.conf Images/plugin_collector/stefan_csr.conf .
+          cp Images/${{ matrix.plugin }}/start_${{ matrix.plugin }}.sh Images/test_plugin.sh Images/${{ matrix.plugin }}/requirements.txt Images/trigger_noconda.py Images/trigger.py .
 
-      - name: Make tinqiita targets
-        # second copy of "qiita_server_certificates" is necessary to match path for docker build, first copy for mounting into container
-        run: |
-          make ./references/qiita_server_certificates ./references/qp-deblur/reference-gg-raxml-bl.tre ./references/qp-target-gene ./environments/qiita_db.env ./environments/qiita.env config
-          cp -r ./references/qiita_server_certificates ./qiita_server_certificates
+      - name: Download certificates from job build_main
+        uses: actions/download-artifact@v4
+        with:
+          name: certificates
+          path: ./
+      - name: Download fake references
+        uses: actions/download-artifact@v4
+        with:
+          name: fake_references
+          path: ./
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build Image (but do not push yet)
+      - name: Build plugin images
         uses: docker/build-push-action@v6
         with:
           context: .
-          push: false
           load: true
           file: Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile
-          tags: tinqiita/${{ matrix.plugin }}:testcandidate
-
-      - name: debug docker
-        run: |
-          docker image ls -a
-          docker ps -a
+          tags: ghcr.io/${{ github.repository }}/${{ matrix.plugin }}:testcandidate
+          cache-from: type=gha,scope=tinqiita-${{ github.ref_name }}
+          cache-to: type=gha,scope=tinqiita-${{ github.ref_name }},mode=max
 
       - name: adapt compose file to select specific plugin
         run: |
           sed -i "s/MATRIXPLUGIN/${{ matrix.plugin }}/g" compose_github.yaml
-          sed -i "s|image: janssenlab/${{ matrix.plugin }}:latest|image: tinqiita/${{ matrix.plugin }}:testcandidate|" compose_github.yaml
 
       - name: Run docker compose
-        if: ${{ !contains('nginx,qiita,plugin_collector', matrix.plugin) }}
         uses: hoverkraft-tech/compose-action@v2.0.1
         with:
           compose-file: "compose_github.yaml"
           services: |
             nginx
-      - name: Execute tests in the running services (wait till qiita plugins are registered)
-        if: ${{ !contains('nginx,qiita,plugin_collector', matrix.plugin) }}
+
+      - name: Execute tests in the running services
         run: |
           sleep 5
-          docker compose exec qiita /bin/bash -c "cat /logs/*; cat /qiita_plugins/*"
-          docker compose exec ${{ matrix.plugin }} /bin/bash -c "PLUGIN=${{ matrix.plugin }} bash /test_plugin.sh"
+          docker compose exec ${{ matrix.plugin }} /bin/bash -c "bash /test_plugin.sh"
 
-      - name: Push production image (only if tests passed)
+      - name: Push image to ghcr (only if tests passed)
         if: success()
         uses: docker/build-push-action@v6
         with:
           context: .
           push: true
           file: Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile
-          tags: |
-            harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.plugin }}:${{ env.IMAGE_TAG }}
-            harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.plugin }}:latest
+          tags: ghcr.io/${{ github.repository }}/${{ matrix.plugin }}:testcandidate
+
+  # build_mulit_plugins:
+  #   needs:
+  #     - build_plugins
+  #   strategy:
+  #     matrix:
+  #       plugin: ["qp-qiime2"]
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Checkout tinqiita repo
+  #       uses: actions/checkout@v4
+
+  #     - name: Set up Docker Buildx
+  #       uses: docker/setup-buildx-action@v3
+
+  #     - name: Read image version from dockerfile
+  #       id: vars
+  #       run: |
+  #         VERSION=$(head -n 1 Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile | cut -d ":" -f 2- | tr -d " ")
+  #         echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV
+  #         cp Images/${{ matrix.plugin }}/start_${{ matrix.plugin }}.sh Images/test_plugin.sh Images/${{ matrix.plugin }}/requirements.txt Images/trigger_noconda.py Images/trigger.py .
+
+  #     - name: Download certificates from job build_main
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: certificates
+  #         path: ./
+  #     - name: Download fake references
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: fake_references
+  #         path: ./
+
+  #     - name: Log in to GitHub Container Registry
+  #       uses: docker/login-action@v3
+  #       with:
+  #         registry: ghcr.io
+  #         username: ${{ github.actor }}
+  #         password: ${{ secrets.GITHUB_TOKEN }}
+
+  #     - name: Build plugin images
+  #       uses: docker/build-push-action@v6
+  #       with:
+  #         context: .
+  #         load: true
+  #         file: Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile
+  #         tags: ghcr.io/${{ github.repository }}/${{ matrix.plugin }}:testcandidate
+  #         cache-from: type=gha,scope=tinqiita-${{ github.ref_name }}
+  #         cache-to: type=gha,scope=tinqiita-${{ github.ref_name }},mode=max
+
+  #     - name: adapt compose file to select specific plugin
+  #       run: |
+  #         sed -i "s|||" compose_github.yaml
+  #         sed -i "s/MATRIXPLUGIN/${{ matrix.plugin }}/g" compose_github.yaml
+
+  #     - name: Run docker compose
+  #       uses: hoverkraft-tech/compose-action@v2.0.1
+  #       with:
+  #         compose-file: "compose_github.yaml"
+  #         services: |
+  #           nginx
+
+  #     - name: Execute tests in the running services
+  #       run: |
+  #         sleep 5
+  #         docker compose exec ${{ matrix.plugin }} /bin/bash -c "bash /test_plugin.sh"
+
+  #     - name: Push image to ghcr (only if tests passed)
+  #       if: success()
+  #       uses: docker/build-push-action@v6
+  #       with:
+  #         context: .
+  #         push: true
+  #         file: Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile
+  #         tags: ghcr.io/${{ github.repository }}/${{ matrix.plugin }}:testcandidate
+
+  publish_images:
+    needs:
+      - build_plugins
+      - build_main
+    strategy:
+      matrix:
+        image: ["qp-deblur", "qtp-biom", "qtp-sequencing", "qtp-visualization", "qtp-diversity", "qp-target-gene", "qtp-job-output-folder",
+                 "nginx", "qiita", "plugin_collector"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout tinqiita repo
+        uses: actions/checkout@v4
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Login to computational.bio registry
+        uses: docker/login-action@v3
+        with:
+          registry: harbor.computational.bio.uni-giessen.de
+          username: ${{ vars.HARBOR_CB_USERNAME }}
+          password: ${{ secrets.HARBOR_CB_SECRET }}
+
+      - name: Pull image from GHCR
+        run: docker pull ghcr.io/${{ github.repository }}/${{ matrix.image }}:testcandidate
+
+      - name: Read image version from dockerfile
+        id: vars
+        run: |
+          VERSION=$(head -n 1 Images/${{ matrix.image }}/${{ matrix.image }}.dockerfile | cut -d ":" -f 2- | tr -d " ")
+          echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV
+
+      - name: Retag image for Docker Hub
+        run: |
+          docker tag ghcr.io/${{ github.repository }}/${{ matrix.image }}:testcandidate harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:${{ env.IMAGE_TAG }}
+          docker tag ghcr.io/${{ github.repository }}/${{ matrix.image }}:testcandidate harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:latest
+
+      - name: Push image to Docker Hub
+        run: |
+          docker push harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:${{ env.IMAGE_TAG }}
+          docker push harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:latest

Images/plugin_collector/plugin_collector.dockerfile

@@ -14,4 +14,4 @@ COPY fix_test_db.py /fix_test_db.py
 COPY start_plugin_collector.sh /start_plugin_collector.sh
 RUN chmod u+x /start_plugin_collector.sh
 
-CMD /start_plugin_collector.sh
+CMD ["/start_plugin_collector.sh"]

Images/qp-qiime2/qp-qiime2.dockerfile

@@ -1,3 +1,5 @@
+# VERSION: 2025.09.12
+
 FROM ubuntu:24.04
 
 ARG MINIFORGE_VERSION=24.1.2-0