From 8f612601e8354179aa44e241f9f6c4160c51f4f7 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 11 Mar 2025 12:05:52 +0100 Subject: [PATCH 1/8] refactor in a sense that docker prefix "qiita-container-anna-" must no longer given at too many positions --- Images/qiita/drop_workflows.py | 67 ++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 Images/qiita/drop_workflows.py diff --git a/Images/qiita/drop_workflows.py b/Images/qiita/drop_workflows.py new file mode 100644 index 0000000..aa45d90 --- /dev/null +++ b/Images/qiita/drop_workflows.py @@ -0,0 +1,67 @@ +import qiita_db as qdb +import sys + + +def remove(self): + # store for later, after table entry is dropped + workflow_name = self.name + + def _get_workflow_id(name): + with qdb.sql_connection.TRN: + sql = """SELECT default_workflow_id + FROM qiita.default_workflow + WHERE name = %s""" + qdb.sql_connection.TRN.add(sql, [name]) + return qdb.sql_connection.TRN.execute_fetchlast() + def _get_node_ids(workflow_id): + with qdb.sql_connection.TRN: + sql = """SELECT default_workflow_node_id + FROM qiita.default_workflow_node + WHERE default_workflow_id = %s""" + qdb.sql_connection.TRN.add(sql, [workflow_id]) + return qdb.sql_connection.TRN.execute_fetchflatten() + def _get_edge_ids(node_ids): + if len(node_ids) > 0: + with qdb.sql_connection.TRN: + sql = """SELECT default_workflow_edge_id + FROM qiita.default_workflow_edge + WHERE parent_id in %s OR child_id in %s""" + qdb.sql_connection.TRN.add(sql, [tuple(node_ids), tuple(node_ids)]) + return qdb.sql_connection.TRN.execute_fetchflatten() + else: + return [] + + workflow_id = _get_workflow_id(self.name) + node_ids = _get_node_ids(workflow_id) + edge_ids = _get_edge_ids(node_ids) + with qdb.sql_connection.TRN: + if len(edge_ids) > 0: + sql = """DELETE FROM qiita.default_workflow_edge_connections + WHERE default_workflow_edge_id in %s""" + qdb.sql_connection.TRN.add(sql, [tuple(edge_ids)]) + + sql = """DELETE FROM qiita.default_workflow_edge + WHERE default_workflow_edge_id in %s""" + qdb.sql_connection.TRN.add(sql, [tuple(edge_ids)]) + + if workflow_id is not None: + sql = """DELETE FROM qiita.default_workflow_node + WHERE default_workflow_id = %s""" + qdb.sql_connection.TRN.add(sql, [workflow_id]) + + sql = """DELETE FROM qiita.default_workflow_data_type + WHERE default_workflow_id = %s""" + qdb.sql_connection.TRN.add(sql, [workflow_id]) + + sql = """DELETE FROM qiita.default_workflow + WHERE default_workflow_id = %s""" + qdb.sql_connection.TRN.add(sql, [workflow_id]) + print("removed workflow '%s': ID=%i with %i nodes and %i edges" % (workflow_name, workflow_id, len(node_ids), len(edge_ids)), file=sys.stderr) + +def remove_workflows(): + for w in qdb.software.DefaultWorkflow.iter(): + w.remove = remove + w.remove(w) + + +remove_workflows() From 3f4318bdb7f331fa7c97592acea9c727a2345491 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 11 Mar 2025 15:44:23 +0100 Subject: [PATCH 2/8] make certificates in tmp dir + inject docker prefix --- Makefile | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 2e73dc5..adb8949 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,8 @@ PODMAN_FLAGS = PODMAN_BIN = docker buildx CERTNAME=stefan OPENSSL=/bin/openssl +# docker compose prepends name of directory to containers +DOCKER_PREFIX=$(shell basename `pwd`)- TMPDIR := $(shell mktemp -d) ifeq ($(origin tmpdir), undefined) @@ -12,13 +14,14 @@ Certificates/: Images/plugin_collector/stefan_csr.conf Images/plugin_collector/s # === create own certificates === mkdir -p Certificates/ # Generate a new root CA private key and certificate - cd $@/ && $(OPENSSL) req -x509 -sha256 -days 356 -nodes -newkey rsa:2048 -subj "/CN=qiita-container-anna-qiita-1/C=DE/L=Giessen" -keyout $(CERTNAME)_rootca.key -out $(CERTNAME)_rootca.crt + cd $@/ && $(OPENSSL) req -x509 -sha256 -days 356 -nodes -newkey rsa:2048 -subj "/CN=$(DOCKER_PREFIX)-1/C=DE/L=Giessen" -keyout $(CERTNAME)_rootca.key -out $(CERTNAME)_rootca.crt # Generate a new server private key cd $@/ && $(OPENSSL) genrsa -out $(CERTNAME)_server.key 2048 # Copy the following to a new file named csr.conf and modify to suit your needs # Copy the following to a new file named cert.conf and modify to suit your needs # Nils: alt_names is the important aspect. Make entries for all valid hostnames with which services shall be addressed - cp $^ $@/ + for f in `echo "$^"`; do cat $$f | sed "s/PREFIX/$(DOCKER_PREFIX)/g" > $@/`basename $$f`; done + #cp $^ $@/ # Generate a certificate signing request cd $@/ && $(OPENSSL) req -new -key $(CERTNAME)_server.key -out $(CERTNAME)_server.csr -config $(CERTNAME)_csr.conf # Generate a new signed server.crt to use with your server.key @@ -93,8 +96,9 @@ plugin: Images/qtp-biom/trigger.py Certificates/ .built_image_plugin_collector: Images/plugin_collector/plugin_collector.dockerfile Images/plugin_collector/fix_test_db.py Images/plugin_collector/collect_configs.py Images/plugin_collector/startup_plugin_collector.sh tmpdir=$(TMPDIR) $(MAKE) plugin - cp -r Certificates/ Images/plugin_collector/ - cd Images/plugin_collector && $(PODMAN_BIN) build . -f `basename $<` $(PODMAN_FLAGS) -t local-plugin_collector + cp $^ $(TMPDIR) + cp -r Certificates/ $(tmpdir)/ + $(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-plugin_collector touch .built_image_plugin_collector images: .built_image_qtp-biom .built_image_nginx .built_image_qiita .built_image_plugin_collector .built_image_qtp-sequencing .built_image_qp-target-gene .built_image_qtp-visualization .built_image_qtp-diversity .built_image_qp-deblur .built_image_qp-qiime2 .built_image_qp-qiime2 .built_image_qtp-job-output-folder @@ -108,4 +112,8 @@ environments/qiita.env: environments/qiita.env.example config: environments/qiita_db.env environments/qiita.env +make clean: + rm .built_image_* + rm -rf Certificates + all: config images From eb71b4b048448dd63a44efa4982e5c3fa23611ca Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 11 Mar 2025 15:45:04 +0100 Subject: [PATCH 3/8] remove unneccessary lines from startup scripts --- Images/qp-deblur/start_qp-deblur.sh | 12 ------------ Images/qp-qiime2/start_qp-qiime2.sh | 12 ------------ Images/qp-target-gene/start_qp-target-gene.sh | 12 ------------ Images/qtp-biom/start_qtp-biom.sh | 12 ------------ Images/qtp-diversity/start_qtp-diversity.sh | 12 ------------ .../start_qtp-job-output-folder.sh | 12 ------------ Images/qtp-sequencing/start_qtp-sequencing.sh | 12 ------------ Images/qtp-visualization/start_qtp-visualization.sh | 12 ------------ 8 files changed, 96 deletions(-) diff --git a/Images/qp-deblur/start_qp-deblur.sh b/Images/qp-deblur/start_qp-deblur.sh index c1639e8..a8b9bac 100644 --- a/Images/qp-deblur/start_qp-deblur.sh +++ b/Images/qp-deblur/start_qp-deblur.sh @@ -1,17 +1,5 @@ #!/bin/bash -#export QIITA_ROOTCA_CERT=/qiita/qiita_core/support_files/ci_server.crt -export QIITA_CONFIG_FP=/qiita_configurations/qiita_server.cfg -CONDA_DIR=/opt/conda -ENV_NAME=qp-deblur - -# Commented out because I wanted to jump onto the container to crawl into the code (it crashes during the start_biom step) - -#configure_biom --env-script "source /opt/conda/bin/activate ; conda activate qtp-biom" --server-cert $QIITA_ROOTCA_CERT - -#start_biom https://localhost:8383 register ignored -#start_biom http://qiita:8383 register ignored -#source $CONDA_DIR/etc/profile.d/conda.sh; conda activate $CONDA_DIR/envs/$ENV_NAME; cd / && python trigger.py cd / && python trigger.py deblur start_deblur /qp-deblur tail -f /dev/null diff --git a/Images/qp-qiime2/start_qp-qiime2.sh b/Images/qp-qiime2/start_qp-qiime2.sh index 72337ab..50069a0 100644 --- a/Images/qp-qiime2/start_qp-qiime2.sh +++ b/Images/qp-qiime2/start_qp-qiime2.sh @@ -1,17 +1,5 @@ #!/bin/bash -#export QIITA_ROOTCA_CERT=/qiita/qiita_core/support_files/ci_server.crt -export QIITA_CONFIG_FP=/qiita_configurations/qiita_server.cfg -CONDA_DIR=/opt/conda -ENV_NAME=qp-qiime2 - -# Commented out because I wanted to jump onto the container to crawl into the code (it crashes during the start_biom step) - -#configure_biom --env-script "source /opt/conda/bin/activate ; conda activate qtp-biom" --server-cert $QIITA_ROOTCA_CERT - -#start_biom https://localhost:8383 register ignored -#start_biom http://qiita:8383 register ignored -#source $CONDA_DIR/etc/profile.d/conda.sh; conda activate $CONDA_DIR/envs/$ENV_NAME; cd / && python trigger.py cd / && python trigger.py qiime2 start_qiime2 /qp-qiime2 tail -f /dev/null diff --git a/Images/qp-target-gene/start_qp-target-gene.sh b/Images/qp-target-gene/start_qp-target-gene.sh index 5fdfed7..9749354 100644 --- a/Images/qp-target-gene/start_qp-target-gene.sh +++ b/Images/qp-target-gene/start_qp-target-gene.sh @@ -1,17 +1,5 @@ #!/bin/bash -#export QIITA_ROOTCA_CERT=/qiita/qiita_core/support_files/ci_server.crt -export QIITA_CONFIG_FP=/qiita_configurations/qiita_server.cfg -CONDA_DIR=/opt/conda -ENV_NAME=qp-target-gene - -# Commented out because I wanted to jump onto the container to crawl into the code (it crashes during the start_biom step) - -#configure_biom --env-script "source /opt/conda/bin/activate ; conda activate qtp-biom" --server-cert $QIITA_ROOTCA_CERT - -#start_biom https://localhost:8383 register ignored -#start_biom http://qiita:8383 register ignored -#source $CONDA_DIR/etc/profile.d/conda.sh; conda activate $CONDA_DIR/envs/$ENV_NAME; cd / && python trigger.py cd / && python trigger.py qp-target-gene start_target_gene /qp-target-gene tail -f /dev/null diff --git a/Images/qtp-biom/start_qtp-biom.sh b/Images/qtp-biom/start_qtp-biom.sh index 1e2d9ba..3ab12c4 100644 --- a/Images/qtp-biom/start_qtp-biom.sh +++ b/Images/qtp-biom/start_qtp-biom.sh @@ -1,17 +1,5 @@ #!/bin/bash -#export QIITA_ROOTCA_CERT=/qiita/qiita_core/support_files/ci_server.crt -export QIITA_CONFIG_FP=/qiita_configurations/qiita_server.cfg -CONDA_DIR=/opt/conda -ENV_NAME=qtp-biom - -# Commented out because I wanted to jump onto the container to crawl into the code (it crashes during the start_biom step) - -#configure_biom --env-script "source /opt/conda/bin/activate ; conda activate qtp-biom" --server-cert $QIITA_ROOTCA_CERT - -#start_biom https://localhost:8383 register ignored -#start_biom http://qiita:8383 register ignored -#source $CONDA_DIR/etc/profile.d/conda.sh; conda activate $CONDA_DIR/envs/$ENV_NAME; cd / && python trigger.py cd / && python trigger.py qtp-biom start_biom /qtp-biom tail -f /dev/null diff --git a/Images/qtp-diversity/start_qtp-diversity.sh b/Images/qtp-diversity/start_qtp-diversity.sh index ba217c8..889173f 100644 --- a/Images/qtp-diversity/start_qtp-diversity.sh +++ b/Images/qtp-diversity/start_qtp-diversity.sh @@ -1,17 +1,5 @@ #!/bin/bash -#export QIITA_ROOTCA_CERT=/qiita/qiita_core/support_files/ci_server.crt -export QIITA_CONFIG_FP=/qiita_configurations/qiita_server.cfg -CONDA_DIR=/opt/conda -ENV_NAME=qtp-sequencing - -# Commented out because I wanted to jump onto the container to crawl into the code (it crashes during the start_biom step) - -#configure_biom --env-script "source /opt/conda/bin/activate ; conda activate qtp-biom" --server-cert $QIITA_ROOTCA_CERT - -#start_biom https://localhost:8383 register ignored -#start_biom http://qiita:8383 register ignored -#source $CONDA_DIR/etc/profile.d/conda.sh; conda activate $CONDA_DIR/envs/$ENV_NAME; cd / && python trigger.py cd / && python trigger.py qiime2 start_diversity_types /qtp-diversity tail -f /dev/null diff --git a/Images/qtp-job-output-folder/start_qtp-job-output-folder.sh b/Images/qtp-job-output-folder/start_qtp-job-output-folder.sh index 8863b74..cdc5f27 100644 --- a/Images/qtp-job-output-folder/start_qtp-job-output-folder.sh +++ b/Images/qtp-job-output-folder/start_qtp-job-output-folder.sh @@ -1,17 +1,5 @@ #!/bin/bash -#export QIITA_ROOTCA_CERT=/qiita/qiita_core/support_files/ci_server.crt -export QIITA_CONFIG_FP=/qiita_configurations/qiita_server.cfg -CONDA_DIR=/opt/conda -ENV_NAME=qtp-sequencing - -# Commented out because I wanted to jump onto the container to crawl into the code (it crashes during the start_biom step) - -#configure_biom --env-script "source /opt/conda/bin/activate ; conda activate qtp-biom" --server-cert $QIITA_ROOTCA_CERT - -#start_biom https://localhost:8383 register ignored -#start_biom http://qiita:8383 register ignored -#source $CONDA_DIR/etc/profile.d/conda.sh; conda activate $CONDA_DIR/envs/$ENV_NAME; cd / && python trigger.py cd / && python trigger.py qtp-job-output-folder start_qtp_job_output_folder /qtp-job-output-folder tail -f /dev/null diff --git a/Images/qtp-sequencing/start_qtp-sequencing.sh b/Images/qtp-sequencing/start_qtp-sequencing.sh index d4d0494..a740816 100644 --- a/Images/qtp-sequencing/start_qtp-sequencing.sh +++ b/Images/qtp-sequencing/start_qtp-sequencing.sh @@ -1,17 +1,5 @@ #!/bin/bash -#export QIITA_ROOTCA_CERT=/qiita/qiita_core/support_files/ci_server.crt -export QIITA_CONFIG_FP=/qiita_configurations/qiita_server.cfg -CONDA_DIR=/opt/conda -ENV_NAME=qtp-sequencing - -# Commented out because I wanted to jump onto the container to crawl into the code (it crashes during the start_biom step) - -#configure_biom --env-script "source /opt/conda/bin/activate ; conda activate qtp-biom" --server-cert $QIITA_ROOTCA_CERT - -#start_biom https://localhost:8383 register ignored -#start_biom http://qiita:8383 register ignored -#source $CONDA_DIR/etc/profile.d/conda.sh; conda activate $CONDA_DIR/envs/$ENV_NAME; cd / && python trigger.py cd / && python trigger.py qtp-sequencing start_qtp_sequencing /qtp-sequencing tail -f /dev/null diff --git a/Images/qtp-visualization/start_qtp-visualization.sh b/Images/qtp-visualization/start_qtp-visualization.sh index ec5e941..f3aeebe 100644 --- a/Images/qtp-visualization/start_qtp-visualization.sh +++ b/Images/qtp-visualization/start_qtp-visualization.sh @@ -1,17 +1,5 @@ #!/bin/bash -#export QIITA_ROOTCA_CERT=/qiita/qiita_core/support_files/ci_server.crt -export QIITA_CONFIG_FP=/qiita_configurations/qiita_server.cfg -CONDA_DIR=/opt/conda -ENV_NAME=qtp-sequencing - -# Commented out because I wanted to jump onto the container to crawl into the code (it crashes during the start_biom step) - -#configure_biom --env-script "source /opt/conda/bin/activate ; conda activate qtp-biom" --server-cert $QIITA_ROOTCA_CERT - -#start_biom https://localhost:8383 register ignored -#start_biom http://qiita:8383 register ignored -#source $CONDA_DIR/etc/profile.d/conda.sh; conda activate $CONDA_DIR/envs/$ENV_NAME; cd / && python trigger.py cd / && python trigger.py qtp-visualization start_visualization_types /qtp-visualization tail -f /dev/null From 1047bcbb26e6c7bc86ba05f250220c14e7843fdb Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 11 Mar 2025 15:45:45 +0100 Subject: [PATCH 4/8] make certificate creation more flexible with regards to docker prefix --- Images/plugin_collector/stefan_cert.conf | 10 +++++----- Images/plugin_collector/stefan_csr.conf | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Images/plugin_collector/stefan_cert.conf b/Images/plugin_collector/stefan_cert.conf index 9479793..cb9531d 100644 --- a/Images/plugin_collector/stefan_cert.conf +++ b/Images/plugin_collector/stefan_cert.conf @@ -5,8 +5,8 @@ subjectAltName = @alt_names [alt_names] DNS.1 = localhost -DNS.2 = qiita-container-anna-qiita-1 -DNS.3 = qiita-container-anna-qiita-worker-1 -DNS.4 = qiita-container-anna-qiita-worker-2 -DNS.5 = qiita-container-anna-qiita-worker-3 -DNS.6 = qiita-container-anna-nginx-1 \ No newline at end of file +DNS.2 = PREFIXqiita-1 +DNS.3 = PREFIXqiita-worker-1 +DNS.4 = PREFIXqiita-worker-2 +DNS.5 = PREFIXqiita-worker-3 +DNS.6 = PREFIXnginx-1 diff --git a/Images/plugin_collector/stefan_csr.conf b/Images/plugin_collector/stefan_csr.conf index 2f42bb4..ca7ed1c 100644 --- a/Images/plugin_collector/stefan_csr.conf +++ b/Images/plugin_collector/stefan_csr.conf @@ -19,8 +19,8 @@ subjectAltName = @alt_names [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1 -DNS.2 = qiita-container-anna-qiita-1 -DNS.3 = qiita-container-anna-qiita-worker-1 -DNS.4 = qiita-container-anna-qiita-worker-2 -DNS.5 = qiita-container-anna-qiita-worker-3 -DNS.6 = qiita-container-anna-nginx-1 \ No newline at end of file +DNS.2 = PREFIXqiita-1 +DNS.3 = PREFIXqiita-worker-1 +DNS.4 = PREFIXqiita-worker-2 +DNS.5 = PREFIXqiita-worker-3 +DNS.6 = PREFIXnginx-1 From 78b16e3c94ea76223f79351f381ed1b9720bb65c Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 11 Mar 2025 15:46:32 +0100 Subject: [PATCH 5/8] avoid hard coding of docker prefix --- Images/plugin_collector/collect_configs.py | 3 ++- Images/qiita/start_plugin.py | 3 ++- compose.yaml | 15 +++++++++++++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Images/plugin_collector/collect_configs.py b/Images/plugin_collector/collect_configs.py index 24c745b..5dfb7a8 100644 --- a/Images/plugin_collector/collect_configs.py +++ b/Images/plugin_collector/collect_configs.py @@ -10,6 +10,7 @@ raise ValueError("No qiita plugins given for which configuration files should be retrieved! Environment variable '%s' not set!" % ENV_PLUGINS) var_plugins = os.environ['QIITA_PLUGINS'] +docker_prefix = os.environ['DOCKER_PREFIX'] # strip potential quotes if var_plugins.startswith('"') or var_plugins.startswith("'"): var_plugins = var_plugins[1:] @@ -23,7 +24,7 @@ if container == "": continue print(' (%i/%i) %s' % (i+1, len(containers), container), end="", file=sys.stderr) - url = 'http://qiita-container-anna-%s-1:%s/%s' % (container, PORT, API_ENDPOINT) + url = 'http://%s%s-1:%s/%s' % (docker_prefix, container, PORT, API_ENDPOINT) print(" '%s'" % url, end="", file=sys.stderr) req = requests.get(url) diff --git a/Images/qiita/start_plugin.py b/Images/qiita/start_plugin.py index d1df576..c5cc114 100644 --- a/Images/qiita/start_plugin.py +++ b/Images/qiita/start_plugin.py @@ -8,7 +8,8 @@ pluginname, qiita_server_url, job_id, output_dir = sys.argv[1:] -req = requests.post('http://qiita-container-anna-%s-1:%s/run' % (pluginname, PORT), +docker_prefix = os.environ['DOCKER_PREFIX'] +req = requests.post('http://%s%s-1:%s/run' % (docker_prefix, pluginname, PORT), json={'url': qiita_server_url, 'job_id': job_id, 'output_dir': output_dir}) diff --git a/compose.yaml b/compose.yaml index 544db6c..6704f82 100644 --- a/compose.yaml +++ b/compose.yaml @@ -61,6 +61,7 @@ services: - PORT=21174 - MASTER=--master - QIITA_CLIENT_DEBUG_LEVEL=DEBUG + - DOCKER_PREFIX=qiita-container-anna- volumes: - qiita-data:/qiita_data - ./src/qiita:/qiita:U @@ -102,6 +103,7 @@ services: - PORT=21175 - MASTER= - QIITA_CLIENT_DEBUG_LEVEL=DEBUG + - DOCKER_PREFIX=qiita-container-anna- volumes: - qiita-data:/qiita_data - ./logs:/logs @@ -197,11 +199,13 @@ services: volumes: - qiita-data:/qiita_data - ./Images/qiita/config_qiita_oidc.cfg:/qiita_configurations/qiita_server.cfg:r # TODO: do we really want to expose server settings to the plugin? + - ./src/qtp-biom:/qtp-biom:U environment: # TODO: is there a more elegant way to obtain this path? - REQUESTS_CA_BUNDLE=/opt/conda/envs/qtp-biom/lib/python3.8/site-packages/certifi/cacert.pem - SSL_CERT_FILE=/opt/conda/envs/qtp-biom/lib/python3.8/site-packages/certifi/cacert.pem - QIITA_CLIENT_DEBUG_LEVEL=DEBUG + - QIITA_CONFIG_FP=/qiita_configurations/qiita_server.cfg networks: - qiita-net @@ -220,6 +224,7 @@ services: - REQUESTS_CA_BUNDLE=/opt/conda/envs/qtp-sequencing/lib/python3.9/site-packages/certifi/cacert.pem - SSL_CERT_FILE=/opt/conda/envs/qtp-sequencing/lib/python3.9/site-packages/certifi/cacert.pem - QIITA_CLIENT_DEBUG_LEVEL=DEBUG + - QIITA_CONFIG_FP=/qiita_configurations/qiita_server.cfg networks: - qiita-net @@ -238,6 +243,7 @@ services: - REQUESTS_CA_BUNDLE=/opt/conda/envs/qp-target-gene/lib/python2.7/site-packages/certifi/cacert.pem - SSL_CERT_FILE=/opt/conda/envs/qp-target-gene/lib/python2.7/site-packages/certifi/cacert.pem - QIITA_CLIENT_DEBUG_LEVEL=DEBUG + - QIITA_CONFIG_FP=/qiita_configurations/qiita_server.cfg networks: - qiita-net @@ -256,6 +262,7 @@ services: - REQUESTS_CA_BUNDLE=/opt/conda/envs/qtp-visualization/lib/python3.6/site-packages/certifi/cacert.pem - SSL_CERT_FILE=/opt/conda/envs/qtp-visualization/lib/python3.6/site-packages/certifi/cacert.pem - QIITA_CLIENT_DEBUG_LEVEL=DEBUG + - QIITA_CONFIG_FP=/qiita_configurations/qiita_server.cfg networks: - qiita-net @@ -274,6 +281,7 @@ services: - REQUESTS_CA_BUNDLE=/opt/conda/envs/qiime2/lib/python3.8/site-packages/certifi/cacert.pem - SSL_CERT_FILE=/opt/conda/envs/qiime2/lib/python3.8/site-packages/certifi/cacert.pem - QIITA_CLIENT_DEBUG_LEVEL=DEBUG + - QIITA_CONFIG_FP=/qiita_configurations/qiita_server.cfg networks: - qiita-net @@ -287,11 +295,13 @@ services: volumes: - qiita-data:/qiita_data - ./Images/qiita/config_qiita_oidc.cfg:/qiita_configurations/qiita_server.cfg:r # TODO: do we really want to expose server settings to the plugin? + - ./src/qtp-biom:/qtp-biom:U environment: # TODO: is there a more elegant way to obtain this path? - REQUESTS_CA_BUNDLE=/opt/conda/envs/deblur/lib/python3.5/site-packages/certifi/cacert.pem - SSL_CERT_FILE=/opt/conda/envs/deblur/lib/python3.5/site-packages/certifi/cacert.pem - QIITA_CLIENT_DEBUG_LEVEL=DEBUG + - QIITA_CONFIG_FP=/qiita_configurations/qiita_server.cfg networks: - qiita-net @@ -310,6 +320,7 @@ services: - REQUESTS_CA_BUNDLE=/opt/conda/envs/qiime2/lib/python3.8/site-packages/certifi/cacert.pem - SSL_CERT_FILE=/opt/conda/envs/qiime2/lib/python3.8/site-packages/certifi/cacert.pem - QIITA_CLIENT_DEBUG_LEVEL=DEBUG + - QIITA_CONFIG_FP=/qiita_configurations/qiita_server.cfg networks: - qiita-net @@ -328,6 +339,7 @@ services: - REQUESTS_CA_BUNDLE=/opt/conda/envs/qtp-job-output-folder/lib/python3.6/site-packages/certifi/cacert.pem - SSL_CERT_FILE=/opt/conda/envs/qtp-job-output-folder/lib/python3.6/site-packages/certifi/cacert.pem - QIITA_CLIENT_DEBUG_LEVEL=DEBUG + - QIITA_CONFIG_FP=/qiita_configurations/qiita_server.cfg networks: - qiita-net @@ -343,8 +355,6 @@ services: - server-plugin-configs:/qiita_plugins - qiita-data:/qiita_data - server-certificates:/qiita_certificates - #- /Daten/Git/jlab/qiita-container-anna/Images/plugin_collector/collect_configs.py:/collect.py - #- /Daten/Git/jlab/qiita-container-anna/Images/plugin_collector/fix_test_db.py:/fix_test_db.py depends_on: qiita-initialize-db: condition: service_completed_successfully @@ -365,6 +375,7 @@ services: qtp-job-output-folder: condition: service_started environment: + - DOCKER_PREFIX=qiita-container-anna- - QIITA_PLUGINS="qtp-biom:qtp-sequencing:qp-target-gene:qtp-visualization:qtp-diversity:qp-deblur:qp-qiime2:qtp-job-output-folder:" command: ['/startup_plugin_collector.sh'] From 363a70ed1971029e7d36b0386f14f8503c7194f9 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 11 Mar 2025 16:18:56 +0100 Subject: [PATCH 6/8] found an even better way to avoid basename project names: field "name" as top level in compose file --- Images/nginx/nginx_qiita.conf | 6 +++--- Images/plugin_collector/collect_configs.py | 3 +-- Images/plugin_collector/stefan_cert.conf | 10 +++++----- Images/plugin_collector/stefan_csr.conf | 10 +++++----- Images/qiita/start_plugin.py | 3 +-- Makefile | 9 +++++---- compose.yaml | 5 ++--- 7 files changed, 22 insertions(+), 24 deletions(-) diff --git a/Images/nginx/nginx_qiita.conf b/Images/nginx/nginx_qiita.conf index ae2f2e0..8e0772e 100644 --- a/Images/nginx/nginx_qiita.conf +++ b/Images/nginx/nginx_qiita.conf @@ -13,9 +13,9 @@ http { # ports to redirect for mainqiita upstream mainqiita { server qiita:21174; - server qiita-container-anna-qiita-worker-1:21175; - server qiita-container-anna-qiita-worker-2:21175; - server qiita-container-anna-qiita-worker-3:21175; + server tinqiita-qiita-worker-1:21175; + server tinqiita-qiita-worker-2:21175; + server tinqiita-qiita-worker-3:21175; } # define variables for the actions that shall be taken for websocket handshake diff --git a/Images/plugin_collector/collect_configs.py b/Images/plugin_collector/collect_configs.py index 5dfb7a8..0637bf8 100644 --- a/Images/plugin_collector/collect_configs.py +++ b/Images/plugin_collector/collect_configs.py @@ -10,7 +10,6 @@ raise ValueError("No qiita plugins given for which configuration files should be retrieved! Environment variable '%s' not set!" % ENV_PLUGINS) var_plugins = os.environ['QIITA_PLUGINS'] -docker_prefix = os.environ['DOCKER_PREFIX'] # strip potential quotes if var_plugins.startswith('"') or var_plugins.startswith("'"): var_plugins = var_plugins[1:] @@ -24,7 +23,7 @@ if container == "": continue print(' (%i/%i) %s' % (i+1, len(containers), container), end="", file=sys.stderr) - url = 'http://%s%s-1:%s/%s' % (docker_prefix, container, PORT, API_ENDPOINT) + url = 'http://%s%s-1:%s/%s' % ('tinqiita-', container, PORT, API_ENDPOINT) print(" '%s'" % url, end="", file=sys.stderr) req = requests.get(url) diff --git a/Images/plugin_collector/stefan_cert.conf b/Images/plugin_collector/stefan_cert.conf index cb9531d..2eb1ee9 100644 --- a/Images/plugin_collector/stefan_cert.conf +++ b/Images/plugin_collector/stefan_cert.conf @@ -5,8 +5,8 @@ subjectAltName = @alt_names [alt_names] DNS.1 = localhost -DNS.2 = PREFIXqiita-1 -DNS.3 = PREFIXqiita-worker-1 -DNS.4 = PREFIXqiita-worker-2 -DNS.5 = PREFIXqiita-worker-3 -DNS.6 = PREFIXnginx-1 +DNS.2 = tinqiita-qiita-1 +DNS.3 = tinqiita-qiita-worker-1 +DNS.4 = tinqiita-qiita-worker-2 +DNS.5 = tinqiita-qiita-worker-3 +DNS.6 = tinqiita-nginx-1 diff --git a/Images/plugin_collector/stefan_csr.conf b/Images/plugin_collector/stefan_csr.conf index ca7ed1c..011124a 100644 --- a/Images/plugin_collector/stefan_csr.conf +++ b/Images/plugin_collector/stefan_csr.conf @@ -19,8 +19,8 @@ subjectAltName = @alt_names [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1 -DNS.2 = PREFIXqiita-1 -DNS.3 = PREFIXqiita-worker-1 -DNS.4 = PREFIXqiita-worker-2 -DNS.5 = PREFIXqiita-worker-3 -DNS.6 = PREFIXnginx-1 +DNS.2 = tinqiita-qiita-1 +DNS.3 = tinqiita-qiita-worker-1 +DNS.4 = tinqiita-qiita-worker-2 +DNS.5 = tinqiita-qiita-worker-3 +DNS.6 = tinqiita-nginx-1 diff --git a/Images/qiita/start_plugin.py b/Images/qiita/start_plugin.py index c5cc114..5709f63 100644 --- a/Images/qiita/start_plugin.py +++ b/Images/qiita/start_plugin.py @@ -8,8 +8,7 @@ pluginname, qiita_server_url, job_id, output_dir = sys.argv[1:] -docker_prefix = os.environ['DOCKER_PREFIX'] -req = requests.post('http://%s%s-1:%s/run' % (docker_prefix, pluginname, PORT), +req = requests.post('http://%s%s-1:%s/run' % ('tinqiita-', pluginname, PORT), json={'url': qiita_server_url, 'job_id': job_id, 'output_dir': output_dir}) diff --git a/Makefile b/Makefile index adb8949..82cb5b1 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,6 @@ PODMAN_BIN = docker buildx CERTNAME=stefan OPENSSL=/bin/openssl # docker compose prepends name of directory to containers -DOCKER_PREFIX=$(shell basename `pwd`)- TMPDIR := $(shell mktemp -d) ifeq ($(origin tmpdir), undefined) @@ -14,13 +13,13 @@ Certificates/: Images/plugin_collector/stefan_csr.conf Images/plugin_collector/s # === create own certificates === mkdir -p Certificates/ # Generate a new root CA private key and certificate - cd $@/ && $(OPENSSL) req -x509 -sha256 -days 356 -nodes -newkey rsa:2048 -subj "/CN=$(DOCKER_PREFIX)-1/C=DE/L=Giessen" -keyout $(CERTNAME)_rootca.key -out $(CERTNAME)_rootca.crt + cd $@/ && $(OPENSSL) req -x509 -sha256 -days 356 -nodes -newkey rsa:2048 -subj "/CN=tinqiita-nginx-1/C=DE/L=Giessen" -keyout $(CERTNAME)_rootca.key -out $(CERTNAME)_rootca.crt # Generate a new server private key cd $@/ && $(OPENSSL) genrsa -out $(CERTNAME)_server.key 2048 # Copy the following to a new file named csr.conf and modify to suit your needs # Copy the following to a new file named cert.conf and modify to suit your needs # Nils: alt_names is the important aspect. Make entries for all valid hostnames with which services shall be addressed - for f in `echo "$^"`; do cat $$f | sed "s/PREFIX/$(DOCKER_PREFIX)/g" > $@/`basename $$f`; done + for f in `echo "$^"`; do cat $$f > $@/`basename $$f`; done #cp $^ $@/ # Generate a certificate signing request cd $@/ && $(OPENSSL) req -new -key $(CERTNAME)_server.key -out $(CERTNAME)_server.csr -config $(CERTNAME)_csr.conf @@ -91,7 +90,7 @@ plugin: Images/qtp-biom/trigger.py Certificates/ test -d src/qiita || git clone -b auth_oidc https://github.com/jlab/qiita.git src/qiita # remove configuration and certificate files from upstream qiita repo rm -rf src/qiita/qiita_core/support_files - cd Images/qiita && $(PODMAN_BIN) build . -f `basename $<` $(PODMAN_FLAGS) -t local-qiita + cd Images/qiita && $(PODMAN_BIN) build . -f `basename $<` $(PODMAN_FLAGS) -t local-qiita --no-cache touch .built_image_qiita .built_image_plugin_collector: Images/plugin_collector/plugin_collector.dockerfile Images/plugin_collector/fix_test_db.py Images/plugin_collector/collect_configs.py Images/plugin_collector/startup_plugin_collector.sh @@ -115,5 +114,7 @@ config: environments/qiita_db.env environments/qiita.env make clean: rm .built_image_* rm -rf Certificates + rm -rf /var/lib/docker/volumes/tinqiita_server-certificates/_data/* + rm -rf /var/lib/docker/volumes/tinqiita_server-plugin-configs/_data/* all: config images diff --git a/compose.yaml b/compose.yaml index 6704f82..cfff263 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,3 +1,5 @@ +name: tinqiita + services: qiita-db: image: postgres:15 @@ -61,7 +63,6 @@ services: - PORT=21174 - MASTER=--master - QIITA_CLIENT_DEBUG_LEVEL=DEBUG - - DOCKER_PREFIX=qiita-container-anna- volumes: - qiita-data:/qiita_data - ./src/qiita:/qiita:U @@ -103,7 +104,6 @@ services: - PORT=21175 - MASTER= - QIITA_CLIENT_DEBUG_LEVEL=DEBUG - - DOCKER_PREFIX=qiita-container-anna- volumes: - qiita-data:/qiita_data - ./logs:/logs @@ -375,7 +375,6 @@ services: qtp-job-output-folder: condition: service_started environment: - - DOCKER_PREFIX=qiita-container-anna- - QIITA_PLUGINS="qtp-biom:qtp-sequencing:qp-target-gene:qtp-visualization:qtp-diversity:qp-deblur:qp-qiime2:qtp-job-output-folder:" command: ['/startup_plugin_collector.sh'] From c25c8db51c12df792e51fbcbb55ddfb6e0bd2fbf Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 11 Mar 2025 16:34:44 +0100 Subject: [PATCH 7/8] update base url --- Images/qiita/config_qiita_oidc.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Images/qiita/config_qiita_oidc.cfg b/Images/qiita/config_qiita_oidc.cfg index bc1511e..273e03d 100644 --- a/Images/qiita/config_qiita_oidc.cfg +++ b/Images/qiita/config_qiita_oidc.cfg @@ -23,7 +23,7 @@ LOG_DIR = /logs/ REQUIRE_APPROVAL = True # Base URL: DO NOT ADD TRAILING SLASH -BASE_URL = https://qiita-container-anna-nginx-1:8383 +BASE_URL = https://tinqiita-nginx-1:8383 # Download path files UPLOAD_DATA_DIR = /qiita_data/uploads/ From 2f5354d9eeb28f29dd9fbcc815229e517eb9db79 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 14 Mar 2025 12:10:47 +0100 Subject: [PATCH 8/8] add aspera for ENA submission --- Images/qiita/qiita.dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Images/qiita/qiita.dockerfile b/Images/qiita/qiita.dockerfile index b185724..2a69b06 100644 --- a/Images/qiita/qiita.dockerfile +++ b/Images/qiita/qiita.dockerfile @@ -77,4 +77,7 @@ RUN rm -f /qiita/qiita_pet/nginx_example.conf /qiita/qiita_pet/supervisor_exampl COPY drop_workflows.py /drop_workflows.py +# install aspera client for ENA submission +RUN conda install hcc::aspera-cli + # CMD ["conda", "run", "-n", "qiita"]