works!

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

.github/workflows/test-flux.yaml

@@ -5,6 +5,13 @@ on:
 jobs:
   test-flux-operator-executor:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # Note that we cannot test conda because the bind of the filesystem is too slow
+        commands: [["singularity", "cp -R ./tests/test_singularity_module/* /tmp/workflow/", "snakemake --cores 1 --jobs 2 --flux-operator --use-singularity"],
+                   ["vanilla", "", "snakemake --cores 1 --jobs 1 --flux-operator --minicluster-container-image ghcr.io/rse-ops/atacseq-vanilla:app-latest"]]
+
     steps:
     - name: Checkout the code
       uses: actions/checkout@v3
@@ -36,25 +43,34 @@ jobs:
       run: |
         export SHELL=/bin/bash
         eval $(minikube -p minikube docker-env)
-        minikube ssh docker pull ghcr.io/rse-ops/atacseq-vanilla:app-latest
-        
+        minikube ssh docker pull ghcr.io/rse-ops/mamba:app-mamba
+
+    - name: Install Flux Operator
+      run: |
+        wget https://raw.githubusercontent.com/flux-framework/flux-operator/main/examples/dist/flux-operator.yaml
+        kubectl apply -f flux-operator.yaml
+
+    - name: Make workflow directory
+      run: mkdir -p /tmp/workflow
+
     - name: Prepare workflow on Runner
+      if: (matrix.commands[0] != 'singularity') 
       run: |
         git clone --depth 1 https://github.com/snakemake/snakemake-tutorial-data /tmp/workflow
         mkdir -p /tmp/workflow/scripts
         wget -O /tmp/workflow/scripts/plot-quals.py https://raw.githubusercontent.com/rse-ops/flux-hpc/main/snakemake/atacseq/scripts/plot-quals.py
         cp ./examples/flux/operator/Snakefile /tmp/workflow/Snakefile
 
-    - name: Install Flux Operator
-      run: |
-        wget https://raw.githubusercontent.com/flux-framework/flux-operator/main/examples/dist/flux-operator.yaml
-        kubectl apply -f flux-operator.yaml
-
-    - name: Run Jobs
+    - name: Run ${{ matrix.commands[0] }} Test Job
+      env:
+        prep_command: ${{ matrix.commands[1] }}
+        run_command: ${{ matrix.commands[2] }}
       run: |
-        cd /tmp/workflow
         minikube mount /tmp/workflow:/tmp/workflow &
-        snakemake --cores 1 --jobs 1 --flux-operator    
+        ${prep_command}
+        sudo chmod -R 775 /tmp/workflow/
+        cd /tmp/workflow
+        ${run_command}
 
   test-flux-executor:
     runs-on: ubuntu-latest

docs/executor_tutorial/flux.rst

@@ -294,6 +294,69 @@ You'll notice the workflow moving faster, and this is because we have submit mor
 than one job at once! Note that we discourage using MiniKube, as the conda environments
 create a lot of tiny files that (in practice) will not finish in any amount of reasonable time.
 
+Tutorial Workflow with Singularity
+::::::::::::::::::::::::::::::::::
+
+Now let's do the same, but ask to run Singularity. 
+
+.. warning::
+
+    Using Singularity in the Flux Operator requires running the pod in priviledged
+    mode, as we are running a container inside of a container. If you cluster
+    administrator does not allow this you may encounter errors.
+
+
+We can first remove the contents we have in ``/tmp/workflow``
+and replace with a Singularity workflow (test) from Snakemake:
+
+.. code-block:: console
+
+    $ rm -rf /tmp/workflow/*
+    $ cp -R ./tests/test_singularity_module/* /tmp/workflow/
+
+And run Snakemake:
+
+.. code-block:: console
+
+    $ snakemake --cores 1 --jobs 2 --flux-operator --use-singularity
+
+Note that since we need to pull the singularity containers, this step will take slightly
+longer. You will likely get optimal peformance to have many commands run that share one or 
+more common containers. And that's it! Snakemake will run the workflows and clean up 
+the MiniClusters.
+
+
+Tutorial Workflow with udocker
+::::::::::::::::::::::::::::::
+
+Let's now run a similar test, but using a "udocker" container.
+
+.. warning::
+
+    Using udocker in the Flux Operator requires running the pod in priviledged
+    mode, as we are running a container inside of a container. If you cluster
+    administrator does not allow this you may encounter errors.
+
+We can first remove the contents we have in ``/tmp/workflow``
+and replace with a Singularity workflow (test) from Snakemake:
+
+.. code-block:: console
+
+    $ rm -rf /tmp/workflow/*
+    $ cp -R ./tests/test_containerized/* /tmp/workflow/
+
+And run Snakemake:
+
+.. code-block:: console
+
+    $ snakemake --cores 1 --jobs 2 --flux-operator
+
+Note that since we need to pull the singularity containers, this step will take slightly
+longer. You will likely get optimal peformance to have many commands run that share one or 
+more common containers. And that's it! Snakemake will run the workflows and clean up 
+the MiniClusters.
+
+
 How does it work?
 :::::::::::::::::
 

examples/flux/operator/kind-config.yaml

@@ -0,0 +1,7 @@
+apiVersion: kind.x-k8s.io/v1alpha4
+kind: Cluster
+nodes:
+      - role: control-plane
+        extraMounts:
+          - hostPath: /tmp/workflow
+            containerPath: /tmp/workflow

snakemake/executors/flux/flux_operator.py

@@ -222,7 +222,7 @@ def __init__(
             max_status_checks_per_second=10,
         )
 
-        # Set the default container image
+        # Set the default container image (has singularity and udocker)
         self.container_image = container_image or "ghcr.io/rse-ops/mamba:app-mamba"
 
         # Attach variables for easy access
@@ -235,6 +235,13 @@ def __init__(
             raise WorkflowError(
                 "Please install the fluxoperator python package (pip install fluxoperator)"
             )
+
+        # Using singularity will require priviledged
+        if self.workflow.use_singularity:
+            logger.warning(
+                f"WARNING: Singularity will require pods running as priviledged."
+            )
+
         self.ctrl = FluxManager(flux_operator_ns)
 
         # Set max nodes of k8s cluster

snakemake/script.py

@@ -617,8 +617,6 @@ def get_preamble(self):
         )
 
     def write_script(self, preamble, fd):
-        print(preamble)
-        print(self.source)
         fd.write(preamble.encode())
         fd.write(self.source.encode())