diff --git a/.github/workflows/buildContainer.yaml b/.github/workflows/buildContainer.yaml index 6112c90..7f7c79b 100644 --- a/.github/workflows/buildContainer.yaml +++ b/.github/workflows/buildContainer.yaml @@ -1,87 +1,292 @@ +# todo: "make" files once and push around through artifacts! + name: docker on: push: branches: - - 'smaller_deblur' + - 'restructure_github_actions' jobs: - docker: + make_certificates: + runs-on: ubuntu-latest + steps: + - name: Checkout tinqiita repo + uses: actions/checkout@v4 + + - name: Create certificate + # second copy of "qiita_server_certificates" is necessary to match path for docker build, first copy for mounting into container + run: | + make ./references/qiita_server_certificates ./environments/qiita_db.env ./environments/qiita.env config + cp -r ./references/qiita_server_certificates ./qiita_server_certificates + + - name: Store certifactes for follow up jobs + uses: actions/upload-artifact@v4 + with: + name: certificates + path: | + ./qiita_server_certificates + ./references/qiita_server_certificates + + build_main: + needs: make_certificates strategy: matrix: - plugin: ["qp-deblur", "qtp-biom", "qtp-sequencing", "qtp-visualization", "qtp-diversity", "qp-target-gene", "qtp-job-output-folder", - "nginx", "qiita", "plugin_collector" - ] + container: ["nginx", "qiita", "plugin_collector"] runs-on: ubuntu-latest steps: - - name: Checkout repository + - name: Checkout tinqiita repo uses: actions/checkout@v4 - - name: Login to DockerHub + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Read image version from dockerfile + id: vars + run: | + VERSION=$(head -n 1 Images/${{ matrix.container }}/${{ matrix.container }}.dockerfile | cut -d ":" -f 2- | tr -d " ") + echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV + cp Images/${{ matrix.container }}/start_${{ matrix.container }}.sh Images/test_plugin.sh Images/nginx/nginx_qiita.conf Images/${{ matrix.container }}/requirements.txt . + cp Images/qiita/config_portal.cfg Images/qiita/config_qiita_oidc.cfg Images/qiita/drop_workflows.py Images/qiita/start_plugin.py Images/qiita/start_qiita-initDB.sh Images/qiita/start_qiita.sh . + cp Images/plugin_collector/collect_configs.py Images/plugin_collector/fix_test_db.py Images/plugin_collector/stefan_cert.conf Images/plugin_collector/stefan_csr.conf . + + - name: Download certificates from job build_main + uses: actions/download-artifact@v4 + with: + name: certificates + path: ./ + + - name: Log in to GitHub Container Registry uses: docker/login-action@v3 with: - registry: harbor.computational.bio.uni-giessen.de - username: ${{ vars.HARBOR_CB_USERNAME }} - password: ${{ secrets.HARBOR_CB_SECRET }} + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build main qiita images and push to github's own registry + uses: docker/build-push-action@v6 + with: + context: . + push: true + file: Images/${{ matrix.container }}/${{ matrix.container }}.dockerfile + tags: ghcr.io/${{ github.repository }}/${{ matrix.container }}:testcandidate + cache-from: type=gha,scope=tinqiita-${{ github.ref_name }} + cache-to: type=gha,scope=tinqiita-${{ github.ref_name }},mode=max + + make_references: + needs: build_main + runs-on: ubuntu-latest + steps: + - name: Checkout tinqiita repo + uses: actions/checkout@v4 + + - name: Create partially fake reference databases + run: | + mkdir -p ./references/qp-deblur/ ./references/qp-target-gene + make ./environments/qiita_db.env ./environments/qiita.env config ./references/qp-deblur/reference-gg-raxml-bl.tre + for f in `echo "references/qp-target-gene/97_otus.fasta references/qp-target-gene/97_otus.tree references/qp-target-gene/97_otu_taxonomy.txt"`; do echo "fake" > $f; done + + - name: Store fake references for follow up jobs + uses: actions/upload-artifact@v4 + with: + name: fake_references + path: | + ./references/qp-deblur + ./references/qp-target-gene + ./environments + + build_plugins: + needs: + - build_main + - make_references + strategy: + matrix: + plugin: ["qp-deblur", "qtp-biom", "qtp-sequencing", "qtp-visualization", "qtp-diversity", "qp-target-gene", "qtp-job-output-folder"] + runs-on: ubuntu-latest + steps: + - name: Checkout tinqiita repo + uses: actions/checkout@v4 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Read version from file + - name: Read image version from dockerfile id: vars run: | VERSION=$(head -n 1 Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile | cut -d ":" -f 2- | tr -d " ") echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV - cp Images/${{ matrix.plugin }}/start_${{ matrix.plugin }}.sh Images/test_plugin.sh Images/trigger_noconda.py Images/trigger.py Images/nginx/nginx_qiita.conf Images/${{ matrix.plugin }}/requirements.txt . - cp Images/qiita/config_portal.cfg Images/qiita/config_qiita_oidc.cfg Images/qiita/drop_workflows.py Images/qiita/start_plugin.py Images/qiita/start_qiita-initDB.sh Images/qiita/start_qiita.sh . - cp Images/plugin_collector/collect_configs.py Images/plugin_collector/fix_test_db.py Images/plugin_collector/stefan_cert.conf Images/plugin_collector/stefan_csr.conf . + cp Images/${{ matrix.plugin }}/start_${{ matrix.plugin }}.sh Images/test_plugin.sh Images/${{ matrix.plugin }}/requirements.txt Images/trigger_noconda.py Images/trigger.py . - - name: Make tinqiita targets - # second copy of "qiita_server_certificates" is necessary to match path for docker build, first copy for mounting into container - run: | - make ./references/qiita_server_certificates ./references/qp-deblur/reference-gg-raxml-bl.tre ./references/qp-target-gene ./environments/qiita_db.env ./environments/qiita.env config - cp -r ./references/qiita_server_certificates ./qiita_server_certificates + - name: Download certificates from job build_main + uses: actions/download-artifact@v4 + with: + name: certificates + path: ./ + - name: Download fake references + uses: actions/download-artifact@v4 + with: + name: fake_references + path: ./ + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Build Image (but do not push yet) + - name: Build plugin images uses: docker/build-push-action@v6 with: context: . - push: false load: true file: Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile - tags: tinqiita/${{ matrix.plugin }}:testcandidate - - - name: debug docker - run: | - docker image ls -a - docker ps -a + tags: ghcr.io/${{ github.repository }}/${{ matrix.plugin }}:testcandidate + cache-from: type=gha,scope=tinqiita-${{ github.ref_name }} + cache-to: type=gha,scope=tinqiita-${{ github.ref_name }},mode=max - name: adapt compose file to select specific plugin run: | sed -i "s/MATRIXPLUGIN/${{ matrix.plugin }}/g" compose_github.yaml - sed -i "s|image: janssenlab/${{ matrix.plugin }}:latest|image: tinqiita/${{ matrix.plugin }}:testcandidate|" compose_github.yaml - name: Run docker compose - if: ${{ !contains('nginx,qiita,plugin_collector', matrix.plugin) }} uses: hoverkraft-tech/compose-action@v2.0.1 with: compose-file: "compose_github.yaml" services: | nginx - - name: Execute tests in the running services (wait till qiita plugins are registered) - if: ${{ !contains('nginx,qiita,plugin_collector', matrix.plugin) }} + + - name: Execute tests in the running services run: | sleep 5 - docker compose exec qiita /bin/bash -c "cat /logs/*; cat /qiita_plugins/*" - docker compose exec ${{ matrix.plugin }} /bin/bash -c "PLUGIN=${{ matrix.plugin }} bash /test_plugin.sh" + docker compose exec ${{ matrix.plugin }} /bin/bash -c "bash /test_plugin.sh" - - name: Push production image (only if tests passed) + - name: Push image to ghcr (only if tests passed) if: success() uses: docker/build-push-action@v6 with: context: . push: true file: Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile - tags: | - harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.plugin }}:${{ env.IMAGE_TAG }} - harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.plugin }}:latest + tags: ghcr.io/${{ github.repository }}/${{ matrix.plugin }}:testcandidate + + # build_mulit_plugins: + # needs: + # - build_plugins + # strategy: + # matrix: + # plugin: ["qp-qiime2"] + # runs-on: ubuntu-latest + # steps: + # - name: Checkout tinqiita repo + # uses: actions/checkout@v4 + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + + # - name: Read image version from dockerfile + # id: vars + # run: | + # VERSION=$(head -n 1 Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile | cut -d ":" -f 2- | tr -d " ") + # echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV + # cp Images/${{ matrix.plugin }}/start_${{ matrix.plugin }}.sh Images/test_plugin.sh Images/${{ matrix.plugin }}/requirements.txt Images/trigger_noconda.py Images/trigger.py . + + # - name: Download certificates from job build_main + # uses: actions/download-artifact@v4 + # with: + # name: certificates + # path: ./ + # - name: Download fake references + # uses: actions/download-artifact@v4 + # with: + # name: fake_references + # path: ./ + + # - name: Log in to GitHub Container Registry + # uses: docker/login-action@v3 + # with: + # registry: ghcr.io + # username: ${{ github.actor }} + # password: ${{ secrets.GITHUB_TOKEN }} + + # - name: Build plugin images + # uses: docker/build-push-action@v6 + # with: + # context: . + # load: true + # file: Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile + # tags: ghcr.io/${{ github.repository }}/${{ matrix.plugin }}:testcandidate + # cache-from: type=gha,scope=tinqiita-${{ github.ref_name }} + # cache-to: type=gha,scope=tinqiita-${{ github.ref_name }},mode=max + + # - name: adapt compose file to select specific plugin + # run: | + # sed -i "s|||" compose_github.yaml + # sed -i "s/MATRIXPLUGIN/${{ matrix.plugin }}/g" compose_github.yaml + + # - name: Run docker compose + # uses: hoverkraft-tech/compose-action@v2.0.1 + # with: + # compose-file: "compose_github.yaml" + # services: | + # nginx + + # - name: Execute tests in the running services + # run: | + # sleep 5 + # docker compose exec ${{ matrix.plugin }} /bin/bash -c "bash /test_plugin.sh" + + # - name: Push image to ghcr (only if tests passed) + # if: success() + # uses: docker/build-push-action@v6 + # with: + # context: . + # push: true + # file: Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile + # tags: ghcr.io/${{ github.repository }}/${{ matrix.plugin }}:testcandidate + + publish_images: + needs: + - build_plugins + - build_main + strategy: + matrix: + image: ["qp-deblur", "qtp-biom", "qtp-sequencing", "qtp-visualization", "qtp-diversity", "qp-target-gene", "qtp-job-output-folder", + "nginx", "qiita", "plugin_collector"] + runs-on: ubuntu-latest + steps: + - name: Checkout tinqiita repo + uses: actions/checkout@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to computational.bio registry + uses: docker/login-action@v3 + with: + registry: harbor.computational.bio.uni-giessen.de + username: ${{ vars.HARBOR_CB_USERNAME }} + password: ${{ secrets.HARBOR_CB_SECRET }} + + - name: Pull image from GHCR + run: docker pull ghcr.io/${{ github.repository }}/${{ matrix.image }}:testcandidate + + - name: Read image version from dockerfile + id: vars + run: | + VERSION=$(head -n 1 Images/${{ matrix.image }}/${{ matrix.image }}.dockerfile | cut -d ":" -f 2- | tr -d " ") + echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV + + - name: Retag image for Docker Hub + run: | + docker tag ghcr.io/${{ github.repository }}/${{ matrix.image }}:testcandidate harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:${{ env.IMAGE_TAG }} + docker tag ghcr.io/${{ github.repository }}/${{ matrix.image }}:testcandidate harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:latest + + - name: Push image to Docker Hub + run: | + docker push harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:${{ env.IMAGE_TAG }} + docker push harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:latest diff --git a/Images/plugin_collector/plugin_collector.dockerfile b/Images/plugin_collector/plugin_collector.dockerfile index 799da21..452c5b0 100644 --- a/Images/plugin_collector/plugin_collector.dockerfile +++ b/Images/plugin_collector/plugin_collector.dockerfile @@ -14,4 +14,4 @@ COPY fix_test_db.py /fix_test_db.py COPY start_plugin_collector.sh /start_plugin_collector.sh RUN chmod u+x /start_plugin_collector.sh -CMD /start_plugin_collector.sh \ No newline at end of file +CMD ["/start_plugin_collector.sh"] diff --git a/Images/qp-qiime2/qp-qiime2.dockerfile b/Images/qp-qiime2/qp-qiime2.dockerfile index 6f63281..0cc7639 100644 --- a/Images/qp-qiime2/qp-qiime2.dockerfile +++ b/Images/qp-qiime2/qp-qiime2.dockerfile @@ -1,3 +1,5 @@ +# VERSION: 2025.09.12 + FROM ubuntu:24.04 ARG MINIFORGE_VERSION=24.1.2-0 diff --git a/compose_github.yaml b/compose_github.yaml index ddb6791..443a6f9 100644 --- a/compose_github.yaml +++ b/compose_github.yaml @@ -27,7 +27,7 @@ services: # start_period: 20s qiita-initialize-db: - image: harbor.computational.bio.uni-giessen.de/tinqiita/qiita:latest + image: ghcr.io/jlab/qiita-keycloak/qiita:testcandidate command: ['/start_qiita-initDB.sh'] depends_on: - qiita-db @@ -46,7 +46,7 @@ services: - qiita-net qiita: - image: harbor.computational.bio.uni-giessen.de/tinqiita/qiita:latest + image: ghcr.io/jlab/qiita-keycloak/qiita:testcandidate build: # image wird hier direkt gebaut context: ./Images/qiita dockerfile: Dockerfile @@ -82,7 +82,7 @@ services: - "21174:21174" qiita-worker: - image: harbor.computational.bio.uni-giessen.de/tinqiita/qiita:latest + image: ghcr.io/jlab/qiita-keycloak/qiita:testcandidate build: # image wird hier direkt gebaut context: ./Images/qiita dockerfile: Dockerfile @@ -142,7 +142,7 @@ services: # start_period: 20s nginx: - image: harbor.computational.bio.uni-giessen.de/tinqiita/nginx:latest + image: ghcr.io/jlab/qiita-keycloak/nginx:testcandidate build: context: ./Images/nginx dockerfile: Dockerfile @@ -170,7 +170,7 @@ services: # start_period: 10s qtp-biom: - image: tinqiita/qtp-biom:testcandidate + image: ghcr.io/jlab/qiita-keycloak/qtp-biom:testcandidate command: ['./start_qtp-biom.sh'] # network_mode: host # stdin_open: true @@ -186,7 +186,7 @@ services: - qiita-net qtp-sequencing: - image: tinqiita/qtp-sequencing:testcandidate + image: ghcr.io/jlab/qiita-keycloak/qtp-sequencing:testcandidate command: ['./start_qtp-sequencing.sh'] # network_mode: host # stdin_open: true @@ -208,7 +208,7 @@ services: # start_period: 3s qp-target-gene: - image: tinqiita/qp-target-gene:testcandidate + image: ghcr.io/jlab/qiita-keycloak/qp-target-gene:testcandidate command: ['./start_qp-target-gene.sh'] # network_mode: host # stdin_open: true @@ -225,7 +225,7 @@ services: - qiita-net qtp-visualization: - image: tinqiita/qtp-visualization:testcandidate + image: ghcr.io/jlab/qiita-keycloak/qtp-visualization:testcandidate command: ['./start_qtp-visualization.sh'] # network_mode: host # stdin_open: true @@ -241,7 +241,7 @@ services: - qiita-net qtp-diversity: - image: tinqiita/qtp-diversity:testcandidate + image: ghcr.io/jlab/qiita-keycloak/qtp-diversity:testcandidate command: ['./start_qtp-diversity.sh'] # network_mode: host # stdin_open: true @@ -257,7 +257,7 @@ services: - qiita-net qp-deblur: - image: tinqiita/qp-deblur:testcandidate + image: ghcr.io/jlab/qiita-keycloak/qp-deblur:testcandidate command: ['./start_qp-deblur.sh'] restart: no volumes: @@ -271,7 +271,7 @@ services: - qiita-net qp-qiime2: - image: tinqiita/qp-qiime2:testcandidate + image: ghcr.io/jlab/qiita-keycloak/qp-qiime2:testcandidate command: ['./start_qp-qiime2.sh'] # network_mode: host # stdin_open: true @@ -288,7 +288,7 @@ services: - qiita-net qtp-job-output-folder: - image: tinqiita/qtp-job-output-folder:testcandidate + image: ghcr.io/jlab/qiita-keycloak/qtp-job-output-folder:testcandidate command: ['./start_qtp-job-output-folder.sh'] # network_mode: host # stdin_open: true @@ -306,7 +306,7 @@ services: plugin-collector: # prior to qiita (master and worker) start up, iterates through the QIITA_PLUGINS : separated list of plugin containers # to compile all q*.conf files from plugin containers in the server-plugin-configs volume - image: harbor.computational.bio.uni-giessen.de/tinqiita/plugin_collector:latest + image: ghcr.io/jlab/qiita-keycloak/plugin_collector:testcandidate restart: no networks: - qiita-net