unify trigger_noconda.py and trigger.py

.github/workflows/buildContainer.yaml

@@ -103,6 +103,7 @@ jobs:
       - build_main
       - make_references
     strategy:
+      fail-fast: false
       matrix:
         plugin: ["qp-deblur", "qtp-biom", "qtp-sequencing", "qtp-visualization", "qtp-diversity", "qp-target-gene", "qtp-job-output-folder"]
     runs-on: ubuntu-latest
@@ -118,7 +119,7 @@ jobs:
         run: |
           VERSION=$(head -n 1 Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile | cut -d ":" -f 2- | tr -d " ")
           echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV
-          cp Images/start_plugin.sh Images/test_plugin.sh Images/${{ matrix.plugin }}/requirements.txt Images/trigger_noconda.py Images/trigger.py .
+          cp Images/start_plugin.sh Images/test_plugin.sh Images/${{ matrix.plugin }}/requirements.txt Images/trigger.py .
 
       - name: Download certificates from job build_main
         uses: actions/download-artifact@v4
@@ -193,7 +194,7 @@ jobs:
         run: |
           VERSION=$(head -n 1 Images/${{ matrix.multiplugin }}/${{ matrix.multiplugin }}.dockerfile | cut -d ":" -f 2- | tr -d " ")
           echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV
-          cp Images/start_plugin.sh Images/test_plugin.sh Images/${{ matrix.multiplugin }}/requirements.txt Images/trigger_noconda.py Images/trigger.py .
+          cp Images/start_plugin.sh Images/test_plugin.sh Images/${{ matrix.multiplugin }}/requirements.txt Images/trigger.py .
 
       - name: Download certificates from job build_main
         uses: actions/download-artifact@v4

Images/qp-deblur/qp-deblur.dockerfile

@@ -105,7 +105,7 @@ RUN sed -i "s|/opt/conda/envs/deblur/share/fragment-insertion/sepp/.sepp/bundled
 
 # install tornado based trigger layer in base environment
 RUN pip install -U --no-cache-dir tornado pip-system-certs
-COPY trigger_noconda.py /trigger.py
+COPY trigger.py /trigger.py
 # ^^ 848 MB
 
 WORKDIR /

Images/qp-target-gene/qp-target-gene.dockerfile

@@ -112,7 +112,7 @@ RUN mkdir -p /unshared_plugins
 ENV QIITA_PLUGINS_DIR=/unshared_plugins/
 
 RUN pip3 install tornado
-COPY trigger_noconda.py /trigger.py
+COPY trigger.py /trigger.py
 
 # copy sortmerna binaries
 COPY --from=builder /sortmerna-2.0/sortmerna /usr/local/bin/sortmerna

Images/qtp-biom/qtp-biom.dockerfile

@@ -135,7 +135,7 @@ RUN ln -s /usr/local/lib/python3.8/site-packages/scikit_learn.libs/libgomp-a34b3
 
 # install tornado based trigger layer in base environment
 #RUN pip install -U --no-cache-dir tornado
-COPY trigger_noconda.py /trigger.py
+COPY trigger.py /trigger.py
 
 WORKDIR /
 

Images/qtp-diversity/qtp-diversity.dockerfile

@@ -119,7 +119,7 @@ RUN mkdir -p /unshared_plugins
 ENV QIITA_PLUGINS_DIR=/unshared_plugins/
 
 # install tornado based trigger layer in base environment
-COPY trigger_noconda.py /trigger.py
+COPY trigger.py /trigger.py
 
 #COPY --from=builder /opt/conda/envs/dependencies/bin/* /opt/conda/bin/
 #COPY --from=builder /opt/conda/envs/dependencies/sbin/* /opt/conda/sbin/
@@ -156,7 +156,7 @@ RUN for f in `echo "libssu.so libhdf5_cpp.so.103 liblapacke.so.3 libcblas.so.3 l
 # fix an pandas deprecation issue, i.e. patch q2templates code
 RUN sed -i "s/'display.max_colwidth', -1/'display.max_colwidth', None/" /usr/local/lib/python3.8/site-packages/q2templates/util.py
 
-# COPY trigger_noconda.py /trigger.py
+# COPY trigger.py /trigger.py
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 

Images/qtp-diversity/requirements.txt

@@ -28,7 +28,7 @@ ijson
 pyhmmer
 frictionless
 numpy
-umap
+umap-learn
 -e /unifrac
 
 -e /qtp-diversity

Images/qtp-job-output-folder/qtp-job-output-folder.dockerfile

@@ -74,7 +74,7 @@ COPY --from=builder /wheels /wheels
 RUN pip install --no-cache-dir /wheels/* \
 	&& rm -rf rm -rf `find /usr/local/lib/python3.6/site-packages -type d -name "tests" | grep -v numpy`
 
-COPY trigger_noconda.py /trigger.py
+COPY trigger.py /trigger.py
 
 COPY start_plugin.sh .
 RUN chmod 755 start_plugin.sh

Images/qtp-sequencing/qtp-sequencing.dockerfile

@@ -78,7 +78,7 @@ COPY --from=builder /opt/conda/envs/qtp-sequencing/lib/libdeflate.so.0 /lib/x86_
 # "install" pigz
 COPY --from=builder /opt/conda/envs/qtp-sequencing/bin/pigz /usr/local/bin/
 
-COPY trigger_noconda.py /trigger.py
+COPY trigger.py /trigger.py
 
 # link to quast program
 RUN ln -s /usr/local/bin/quast.py /usr/local/bin/quast

Images/qtp-visualization/qtp-visualization.dockerfile

@@ -102,7 +102,7 @@ RUN chmod 755 start_plugin.sh
 RUN mkdir -p /unshared_plugins
 ENV QIITA_PLUGINS_DIR=/unshared_plugins/
 
-COPY trigger_noconda.py /trigger.py
+COPY trigger.py /trigger.py
 
 ##  Export cert and config filepaths
 COPY qiita_server_certificates/qiita_server_certificates.pem /qiita_server_certificates/qiita_server_certificates.pem

Images/trigger.py

@@ -6,6 +6,8 @@
 import sys
 import traceback
 import asyncio
+import sys
+import os
 
 conda_env_name = None
 plugin_start_script = None
@@ -28,7 +30,11 @@ async def post(self):
                 return
 
             # Systembefehl ausfuehren
-            cmd = 'source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/conda/envs/%s; %s/scripts/%s %s %s %s' % (conda_env_name, plugin_src_dir, plugin_start_script, qiita_worker_url, job_id, output_dir)
+            cmd = ""
+            if conda_env_name is not None:
+                cmd = 'source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/conda/envs/%s; %s/scripts/%s %s %s %s' % (conda_env_name, plugin_src_dir, plugin_start_script, qiita_worker_url, job_id, output_dir)
+            else:
+                cmd = '%s %s %s %s' % (plugin_start_script, qiita_worker_url, job_id, output_dir)
             # Asynchronen Subprozess starten
             proc = await asyncio.create_subprocess_shell(
                 cmd,
@@ -54,9 +60,12 @@ async def post(self):
             self.set_status(500)
             # a hack to learn which docker service I am in
             plugin_name = "unknown"
-            for f in glob('/start_*.sh'):
-                plugin_name = f.split('_')[-1].replace('.sh', '')
-                break
+            if conda_env_name is not None:
+                for f in glob('/start_*.sh'):
+                    plugin_name = f.split('_')[-1].replace('.sh', '')
+                    break
+            else:
+                plugin_name = os.path.basename(plugin_start_script).replace('start_', '')
             print("Error in service '%s': %s" % (plugin_name, str(e)), file=sys.stderr)
             traceback.print_exc()
             self.write({"error": str(e)})
@@ -78,11 +87,15 @@ def make_app():
     ])
 
 if __name__ == "__main__":
-    conda_env_name = sys.argv[1]
-    plugin_start_script = sys.argv[2]
-    plugin_src_dir = sys.argv[3]
+    if len(sys.argv) == 1+1:
+        _, plugin_start_script = sys.argv
+    elif len(sys.argv) == 3+1:
+        _, conda_env_name, plugin_start_script, plugin_src_dir = sys.argv
+    else:
+        print("Incorrect number of arguments provided\nUsage: trigger.py <conda_env_name> <plugin_start_script> <plugin_src_dir>  <-- for use with conda\n  or   trigger.py <plugin_start_script>  <-- for use without conda\n", file=sys.stderr)
+        exit(1)
 
     app = make_app()
     app.listen(5000)  # Server auf Port 5000 starten
-    print("Server laeuft auf http://localhost:5000", file=sys.stderr)
+    print("Server listening on http://localhost:5000", file=sys.stderr)
     tornado.ioloop.IOLoop.current().start()

Makefile

@@ -31,7 +31,7 @@ $(DIR_REFERENCES)/qiita_server_certificates: Images/plugin_collector/stefan_csr.
 	# === end: create own certificates ===
 
 # a general target, executed for each plugin
-plugin: Images/trigger.py Images/trigger_noconda.py Images/start_plugin.sh $(DIR_REFERENCES)/qiita_server_certificates Images/test_plugin.sh
+plugin: Images/trigger.py Images/start_plugin.sh $(DIR_REFERENCES)/qiita_server_certificates Images/test_plugin.sh
 	cp -r $^ $(tmpdir)/
 
 .built_image_qtp-biom: Images/qtp-biom/qtp-biom.dockerfile src/qiita-files/ src/qtp-biom/ Images/qtp-biom/requirements.txt