diff --git a/.github/workflows/buildContainer.yaml b/.github/workflows/buildContainer.yaml index 737c167..65ce5fa 100644 --- a/.github/workflows/buildContainer.yaml +++ b/.github/workflows/buildContainer.yaml @@ -162,7 +162,9 @@ jobs: - name: Execute tests in the running services run: | sleep 5 - docker compose exec ${{ matrix.plugin }} /bin/bash -c "bash /test_plugin.sh" + fork=`if [[ "${{ matrix.plugin }}" == "qp-deblur" || "${{ matrix.plugin }}" == "qp-target-gene" || "${{ matrix.plugin }}" == "qtp-sequencing" || "${{ matrix.plugin }}" == "qtp-biom" || "${{ matrix.plugin }}" == "qtp-visualization" || "${{ matrix.plugin }}" == "qtp-diversity" ]]; then echo "jlab"; else echo "qiita-spots"; fi` + branch=`if [[ "${{ matrix.plugin }}" == "qp-deblur" ]]; then echo "uncouplePlugins"; elif [[ "${{ matrix.plugin }}" == "qp-target-gene" || "${{ matrix.plugin }}" == "qtp-sequencing" || "${{ matrix.plugin }}" == "qtp-biom" || "${{ matrix.plugin }}" == "qtp-visualization" || "${{ matrix.plugin }}" == "qtp-diversity" ]]; then echo "uncouplePlugin"; elif [[ "${{ matrix.plugin }}" == "qtp-job-output-folder" ]]; then echo "main"; else echo "master"; fi` + docker compose exec ${{ matrix.plugin }} /bin/bash -c "PLUGIN_FORK=$fork PLUGIN_BRANCH=$branch bash /test_plugin.sh" - name: Push image to ghcr (only if tests passed) if: success() diff --git a/Images/qiita/qiita.dockerfile b/Images/qiita/qiita.dockerfile index 0a1ab71..2f7aab2 100644 --- a/Images/qiita/qiita.dockerfile +++ b/Images/qiita/qiita.dockerfile @@ -47,9 +47,14 @@ RUN pip install \ psycopg2-binary -# Clone the Qiita Repo +# Clone the Qiita Repo: currently we need the oidc changes from our jlab fork + changes in the tornado_FetchFileFromCentralHandler branch, which send files if requested directly from tornado instead of nginx (happens in testing) # RUN git clone -b master https://github.com/qiita-spots/qiita.git -RUN git clone -b auth_oidc https://github.com/jlab/qiita.git +RUN git clone -b auth_oidc https://github.com/jlab/qiita.git \ + && cd qiita \ + && git config pull.rebase false \ + && git config --global user.email "jlab@uni-giessen.de" \ + && git config --global user.name "Stefan" && \ + git pull origin tornado_FetchFileFromCentralHandler # should tests re-populate the DB, ensure private plugin, qtp-biom and qp-target-gene use the correct conda env RUN sed -i "s|'source /home/runner/.profile; conda activate qiita'|'source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/conda/envs/qiita'|" /qiita/qiita_db/support_files/populate_test_db.sql diff --git a/Images/qp-deblur/qp-deblur.dockerfile b/Images/qp-deblur/qp-deblur.dockerfile index 067fe51..186e67e 100644 --- a/Images/qp-deblur/qp-deblur.dockerfile +++ b/Images/qp-deblur/qp-deblur.dockerfile @@ -1,4 +1,4 @@ -# VERSION: 2025.08.22 +# VERSION: 2025.09.12 # ========================== # Stage 1: Build wheels @@ -37,7 +37,8 @@ SHELL ["conda", "run", "-p", "/opt/conda/envs/deblur", "/bin/bash", "-c"] ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 -RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +# RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +RUN git clone -b enable_pluginprotocol_change https://github.com/jlab/qiita_client.git RUN sed -i "s/f'Entered BaseQiitaPlugin._register_command({command.name})'/'Entered BaseQiitaPlugin._register_command(%s)' % command.name/" qiita_client/qiita_client/plugin.py RUN cd qiita_client && pip install --no-cache-dir . @@ -130,7 +131,7 @@ RUN cd qiita_client && pip install . RUN mkdir -p /qiita_server_certificates/ COPY qiita_server_certificates/*_server.* /qiita_server_certificates/ -RUN /usr/local/bin/configure_deblur --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` filesystem +RUN /usr/local/bin/configure_deblur --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` https RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qp-deblur/" /unshared_plugins/*.conf # remove conda command from tigger.py diff --git a/Images/qp-target-gene/qp-target-gene.dockerfile b/Images/qp-target-gene/qp-target-gene.dockerfile index dc3b9f7..5482552 100644 --- a/Images/qp-target-gene/qp-target-gene.dockerfile +++ b/Images/qp-target-gene/qp-target-gene.dockerfile @@ -1,4 +1,4 @@ -# VERSION: 2025.08.28 +# VERSION: 2025.09.17 # ========================== # Stage 1: Build wheels @@ -44,11 +44,13 @@ RUN python2.7 get-pip2.7.py --force-reinstall RUN pip install -U pip #RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip -RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +#RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +RUN git clone -b enable_pluginprotocol_change https://github.com/jlab/qiita_client.git + RUN cd /qiita_client && pip install --no-cache-dir . RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip -RUN git clone https://github.com/qiita-spots/qp-target-gene.git +RUN git clone -b uncouplePlugin https://github.com/jlab/qp-target-gene.git WORKDIR /qp-target-gene RUN pip install biom-format RUN pip install -e . @@ -127,7 +129,7 @@ RUN export QIITA_ROOTCA_CERT=/unshared_certificates/ci_rootca.crt COPY qiita_server_certificates/*_server.* /qiita_server_certificates/ RUN sed -i "s|^#\!.*|#\!/usr/bin/python2|" /usr/local/bin/configure_target_gene RUN sed -i "s|^#\!.*|#\!/usr/bin/python2|" /usr/local/bin/start_target_gene -RUN configure_target_gene --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` +RUN configure_target_gene --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` https RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qp-target-gene/" /unshared_plugins/*.conf # for testing diff --git a/Images/qtp-biom/qtp-biom.dockerfile b/Images/qtp-biom/qtp-biom.dockerfile index 98d4841..b461f0b 100644 --- a/Images/qtp-biom/qtp-biom.dockerfile +++ b/Images/qtp-biom/qtp-biom.dockerfile @@ -1,4 +1,4 @@ -# VERSION: 2025.08.29 +# VERSION: 2025.11.04 FROM ubuntu:24.04 AS builder @@ -57,14 +57,17 @@ SHELL ["conda", "run", "-p", "/opt/conda/envs/qtp-biom", "/bin/bash", "-c"] RUN pip install -U pip # RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip -RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +# RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +RUN git clone -b enable_pluginprotocol_change https://github.com/jlab/qiita_client.git RUN cd qiita_client && pip install --no-cache-dir . # RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip RUN git clone -b master https://github.com/qiita-spots/qiita-files.git # COPY ./qiita-files /qiita-files RUN cd /qiita-files && pip install -e . -v -RUN git clone https://github.com/qiita-spots/qtp-biom.git +#RUN git clone https://github.com/qiita-spots/qtp-biom.git +RUN git clone -b uncouplePlugin https://github.com/jlab/qtp-biom.git + # COPY ./qtp-biom /qtp-biom WORKDIR /qtp-biom RUN sed -i "s|'qiita-files @ https://github.com/qiita-spots/'||" setup.py @@ -161,7 +164,7 @@ ENV SSL_CERT_FILE=/qiita_server_certificates/qiita_server_certificates.pem #RUN mkdir -p /qiita_server_certificates/ COPY qiita_server_certificates/*_server.* /qiita_server_certificates/ -RUN configure_biom --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` +RUN configure_biom --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` https RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qtp-biom/" /unshared_plugins/*.conf # fix an pandas deprecation issue, i.e. patch q2templates code diff --git a/Images/qtp-diversity/qtp-diversity.dockerfile b/Images/qtp-diversity/qtp-diversity.dockerfile index f7d79cc..4bd48ff 100644 --- a/Images/qtp-diversity/qtp-diversity.dockerfile +++ b/Images/qtp-diversity/qtp-diversity.dockerfile @@ -1,4 +1,4 @@ -# VERSION: 2025.09.11 +# VERSION: 2025.11.04 FROM ubuntu:24.04 AS builder @@ -50,7 +50,7 @@ SHELL ["conda", "run", "-p", "/opt/conda/envs/qiime2", "/bin/bash", "-c"] RUN pip install -U pip # RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip -RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +RUN git clone -b enable_pluginprotocol_change https://github.com/jlab/qiita_client.git RUN cd qiita_client && pip install --no-cache-dir . # RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip @@ -58,7 +58,7 @@ RUN git clone -b master https://github.com/qiita-spots/qiita-files.git RUN cd /qiita-files && pip install -e . -v #RUN pip install https://github.com/biocore/q2-mislabeled/archive/refs/heads/main.zip -RUN git clone https://github.com/qiita-spots/qtp-diversity.git +RUN git clone -b uncouplePlugin https://github.com/jlab/qtp-diversity.git WORKDIR /qtp-diversity RUN sed -i "s|'qiita-files @ https://github.com/'||" setup.py RUN sed -i "s|'qiita-spots/qiita-files/archive/master.zip',||" setup.py @@ -174,7 +174,7 @@ ENV SSL_CERT_FILE=/qiita_server_certificates/qiita_server_certificates.pem COPY qiita_server_certificates/*_server.* /qiita_server_certificates/ RUN chmod u+x /usr/local/bin/configure_diversity_types /usr/local/bin/start_diversity_types -RUN configure_diversity_types --env-script "true" --ca-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` +RUN configure_diversity_types --env-script "true" --ca-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` https RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qtp-diversity/" /unshared_plugins/*.conf # for testing diff --git a/Images/qtp-job-output-folder/qtp-job-output-folder.dockerfile b/Images/qtp-job-output-folder/qtp-job-output-folder.dockerfile index 3ea824a..f68a89c 100644 --- a/Images/qtp-job-output-folder/qtp-job-output-folder.dockerfile +++ b/Images/qtp-job-output-folder/qtp-job-output-folder.dockerfile @@ -1,4 +1,4 @@ -# VERSION: 2025.09.11 +# VERSION: 2025.11.05 # ========================== # Stage 1: Build wheels @@ -40,14 +40,15 @@ SHELL ["conda", "run", "-p", "/opt/conda/envs/qtp-job-output-folder", "/bin/bash RUN pip install -U pip #RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip -RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +#RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +RUN git clone -b enable_pluginprotocol_change https://github.com/jlab/qiita_client.git RUN cd qiita_client && pip install --no-cache-dir . # RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip RUN git clone -b master https://github.com/qiita-spots/qiita-files.git RUN cd /qiita-files && pip install -e . -v -RUN git clone https://github.com/qiita-spots/qtp-job-output-folder.git +RUN git clone -b uncouplePlugin https://github.com/jlab/qtp-job-output-folder.git WORKDIR /qtp-job-output-folder RUN sed -i "s|'qiita-files @ https://github.com/'||" setup.py RUN sed -i "s|'qiita-spots/qiita-files/archive/master.zip',||" setup.py @@ -90,7 +91,7 @@ ENV SSL_CERT_FILE=/qiita_server_certificates/qiita_server_certificates.pem #RUN export QIITA_ROOTCA_CERT=/unshared_certificates/ci_rootca.crt RUN chmod u+x /usr/local/bin/configure_qtp_job_output_folder /usr/local/bin/start_qtp_job_output_folder COPY qiita_server_certificates/*_server.* /qiita_server_certificates/ -RUN configure_qtp_job_output_folder --env-script "true" --ca-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` +RUN configure_qtp_job_output_folder --env-script "true" --ca-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` https RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qtp-job-output-folder/" /unshared_plugins/*.conf # for testing diff --git a/Images/qtp-sequencing/qtp-sequencing.dockerfile b/Images/qtp-sequencing/qtp-sequencing.dockerfile index 20cbd1d..74d553a 100644 --- a/Images/qtp-sequencing/qtp-sequencing.dockerfile +++ b/Images/qtp-sequencing/qtp-sequencing.dockerfile @@ -1,4 +1,4 @@ -# VERSION: 2025.09.08 +# VERSION: 2025.10.27 # ========================== # Stage 1: Build wheels (~5.8 GB) @@ -36,15 +36,20 @@ SHELL ["conda", "run", "-p", "/opt/conda/envs/qtp-sequencing", "/bin/bash", "-c" RUN pip install -U pip #RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip -RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +#RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +RUN git clone -b enable_pluginprotocol_change https://github.com/jlab/qiita_client.git + RUN cd qiita_client && pip install --no-cache-dir . # RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip RUN git clone -b master https://github.com/qiita-spots/qiita-files.git RUN cd /qiita-files && pip install -e . -v -RUN git clone https://github.com/qiita-spots/qtp-sequencing.git +#RUN git clone https://github.com/qiita-spots/qtp-sequencing.git +RUN git clone -b uncouplePlugin https://github.com/jlab/qtp-sequencing.git WORKDIR /qtp-sequencing +# report the actually clone commit hash of the source repo +RUN git rev-parse HEAD RUN sed -i "s|'qiita-files @ https://github.com/'||" setup.py RUN sed -i "s|'qiita-spots/qiita-files/archive/master.zip',||" setup.py RUN sed -i "s|'qiita_client @ https://github.com/'||" setup.py diff --git a/Images/qtp-visualization/qtp-visualization.dockerfile b/Images/qtp-visualization/qtp-visualization.dockerfile index e08ad3e..87b38bb 100644 --- a/Images/qtp-visualization/qtp-visualization.dockerfile +++ b/Images/qtp-visualization/qtp-visualization.dockerfile @@ -1,4 +1,4 @@ -# VERSION: 2025.08.29 +# VERSION: 2025.11.04 FROM ubuntu:24.04 AS builder @@ -52,14 +52,14 @@ ENV LANG=C.UTF-8 RUN pip install -U pip #RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip -RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +RUN git clone -b enable_pluginprotocol_change https://github.com/jlab/qiita_client.git RUN cd qiita_client && pip install --no-cache-dir . #RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip RUN git clone -b master https://github.com/qiita-spots/qiita-files.git RUN cd /qiita-files && pip install -e . -v -RUN git clone https://github.com/qiita-spots/qtp-visualization.git +RUN git clone -b uncouplePlugin https://github.com/jlab/qtp-visualization.git WORKDIR /qtp-visualization RUN sed -i "s|'qiita_client', 'click >= 3.3', 'qiime2'|'click >= 3.3'|" setup.py RUN pip install -e . @@ -113,7 +113,7 @@ RUN chmod u+x /usr/local/bin/configure_visualization_types /usr/local/bin/start_ COPY qiita_server_certificates/*_server.* /qiita_server_certificates/ # qiime2 expects to have a CONDA_PREFIX set, see https://github.com/qiime2/qiime2/blob/812fd09cf80b4ed76c1f39827ae2dba729448436/qiime2/sdk/parallel_config.py#L30 ENV CONDA_PREFIX=/usr/local -RUN configure_visualization_types --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` +RUN configure_visualization_types --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` https RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qtp-visualization/" /unshared_plugins/*.conf # for testing diff --git a/Images/test_plugin.sh b/Images/test_plugin.sh index 2f03968..82c31c3 100644 --- a/Images/test_plugin.sh +++ b/Images/test_plugin.sh @@ -2,6 +2,11 @@ echo "plugin to be tested is: '$PLUGIN'" +# by default, clone from master. But can also clone different branches +branch="${PLUGIN_BRANCH:-master}" +# by default, clone from qiita-spots branch. But can also use alternative forks +fork="${PLUGIN_FORK:-qiita-spots}" + # install dependencies apt-get update apt-get -y --fix-missing install git @@ -13,7 +18,8 @@ fi; if [ "qp-qiime2" != "$PLUGIN" ]; then # clone plugin repository - git clone https://github.com/qiita-spots/${PLUGIN} + git clone -b ${branch} https://github.com/${fork}/${PLUGIN}; + echo "Clone from '${fork}', branch '${branch}'" 1>&2; fi; # NOTE: client api reset only works when communicating with Qitta Master, @@ -43,6 +49,9 @@ done export QIITA_PORT=21174 export QIITA_ROOTCA_CERT=$SSL_CERT_FILE +# either (old) plugins ignore this at all OR adapted plugins switch to https file transfer +export QIITA_PLUGINCOUPLING=https + # change into plugin source directory and execute actual tests if [ "qp-qiime2" == "$PLUGIN" ]; then source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/conda/envs/qiime2; cd ${PLUGIN} && pytest; diff --git a/README.md b/README.md index ec93ee9..86a3d82 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -## Howto start-up qiita through docker compose +## Howto start-up qiita through docker compose Note: this does currently **not** work with podman :-( So strictly stick to docker here. 1. We assume you operate on your local computer, i.e. not within the BCF cluster as you won't have docker, on a Ubuntu/Mint like OS. You will need approx. 55 GB free disk space. diff --git a/compose.yaml b/compose.yaml index 53a4b4d..c11c1f5 100644 --- a/compose.yaml +++ b/compose.yaml @@ -199,7 +199,7 @@ services: # tty: true restart: no volumes: - - qiita-data:/qiita_data + #- qiita-data:/qiita_data - ./references/qiita_server_certificates:/qiita_server_certificates - ./src/qtp-biom:/qtp-biom:U - test_tmp_dir:/tmp @@ -232,7 +232,7 @@ services: # tty: true restart: no volumes: - - qiita-data:/qiita_data + # - qiita-data:/qiita_data - ./references/qiita_server_certificates:/qiita_server_certificates - test_tmp_dir:/tmp - ./references/qp-target-gene:/databases/gg/13_8/rep_set @@ -249,7 +249,7 @@ services: # tty: true restart: no volumes: - - qiita-data:/qiita_data + #- qiita-data:/qiita_data - ./references/qiita_server_certificates:/qiita_server_certificates - test_tmp_dir:/tmp environment: @@ -265,7 +265,7 @@ services: # tty: true restart: no volumes: - - qiita-data:/qiita_data + #- qiita-data:/qiita_data - ./references/qiita_server_certificates:/qiita_server_certificates - test_tmp_dir:/tmp environment: @@ -281,7 +281,7 @@ services: # tty: true restart: no volumes: - - qiita-data:/qiita_data + #- qiita-data:/qiita_data - ./references/qiita_server_certificates:/qiita_server_certificates - ./references/qp-deblur:/opt/conda/envs/deblur/share/fragment-insertion/ref #- ./src/qtp-biom:/qtp-biom:U @@ -316,7 +316,7 @@ services: # tty: true restart: no volumes: - - qiita-data:/qiita_data + #- qiita-data:/qiita_data - ./references/qiita_server_certificates:/qiita_server_certificates - test_tmp_dir:/tmp environment: diff --git a/compose_github.yaml b/compose_github.yaml index 443a6f9..788f7f6 100644 --- a/compose_github.yaml +++ b/compose_github.yaml @@ -177,7 +177,7 @@ services: # tty: true restart: no volumes: - - qiita-data:/qiita_data + #- qiita-data:/qiita_data - ./references/qiita_server_certificates:/qiita_server_certificates - test_tmp_dir:/tmp environment: @@ -193,7 +193,7 @@ services: # tty: true restart: no volumes: - - qiita-data:/qiita_data + #- qiita-data:/qiita_data - ./references/qiita_server_certificates:/qiita_server_certificates - test_tmp_dir:/tmp environment: @@ -215,7 +215,7 @@ services: # tty: true restart: no volumes: - - qiita-data:/qiita_data + #- qiita-data:/qiita_data - ./references/qiita_server_certificates:/qiita_server_certificates - test_tmp_dir:/tmp - ./references/qp-target-gene:/databases/gg/13_8/rep_set @@ -232,7 +232,7 @@ services: # tty: true restart: no volumes: - - qiita-data:/qiita_data + #- qiita-data:/qiita_data - ./references/qiita_server_certificates:/qiita_server_certificates - test_tmp_dir:/tmp environment: @@ -248,7 +248,7 @@ services: # tty: true restart: no volumes: - - qiita-data:/qiita_data + #- qiita-data:/qiita_data - ./references/qiita_server_certificates:/qiita_server_certificates - test_tmp_dir:/tmp environment: @@ -261,7 +261,7 @@ services: command: ['./start_qp-deblur.sh'] restart: no volumes: - - qiita-data:/qiita_data + # - qiita-data:/qiita_data - ./references/qiita_server_certificates:/qiita_server_certificates - ./references/qp-deblur:/opt/conda/envs/deblur/share/fragment-insertion/ref - test_tmp_dir:/tmp @@ -295,7 +295,7 @@ services: # tty: true restart: no volumes: - - qiita-data:/qiita_data + #- qiita-data:/qiita_data - ./references/qiita_server_certificates:/qiita_server_certificates - test_tmp_dir:/tmp environment: @@ -349,4 +349,4 @@ volumes: server-plugin-configs: # a shared directory on qiita server side (master and workers) to collect qiita plugin configuration files qiita-logs: test_tmp_dir: # many of plugin tests are written under the assumption that they are executed on same machine as qiita main and thus share /tmp - which is not the case in our docker compose scenario! - \ No newline at end of file +