Merge faster tests (#198)

* Use Caching in test workflow to increase speed

---------

Co-authored-by: alsmith <asmith151@outlook.com>

.github/workflows/test_pipelines.yml

@@ -3,13 +3,17 @@ name: Python package
 on: [push]
 
 jobs:
+  Test:
+    if: github.event_name == 'pull_request' || github.event_name == 'push'
   Test:
     if: github.event_name == 'pull_request' || github.event_name == 'push'
     runs-on: ubuntu-latest
     strategy:
       matrix:
         test: ["config", "design", "pipeline"]
         assay: ["atac", "chip-rx", "chip", "rna-rx", "rna", "snp"]
+        test: ["config", "design", "pipeline"]
+        assay: ["atac", "chip-rx", "chip", "rna-rx", "rna", "snp"]
 
     steps:
       - uses: actions/checkout@v3
@@ -25,19 +29,29 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-env-
 
-      - name: Set up conda using miniforge
+      - name: Setup conda
         uses: conda-incubator/setup-miniconda@v3
         with:
           environment-file: testing.yml
           miniforge-variant: Mambaforge
           miniforge-version: latest
           use-mamba: true
 
-      - name: Install the package
+      - name: Install package
         shell: pwsh
         run: |
           pip install .
 
+      - name: Cache singularity
+        if: matrix.test == 'pipeline'
+        id: cache-singularity
+        uses: actions/cache@v3
+        with:
+          path: ~/.apptainer
+          key: ${{ runner.os }}-singularity-${{ hashFiles('tmp/*') }}
+          restore-keys: |
+            ${{ runner.os }}-singularity-
+
       - uses: eWaterCycle/setup-apptainer@v2
         if: matrix.test == 'pipeline'
         with:
@@ -50,7 +64,23 @@ jobs:
           apptainer remote add --no-login SylabsCloud cloud.sylabs.io
           apptainer remote use SylabsCloud
 
+      - name: Pull singularity images
+        if: matrix.test == 'pipeline' && steps.cache-singularity.outputs.cache-hit != 'true'
+        shell: bash
+        run: |
+          echo "Pulling singularity images from the remote repository..."
+          mkdir -p tmp
+          if [[ "${{ matrix.test }}" == "pipeline" ]]; then
+            apptainer pull --force library://asmith151/seqnado/seqnado_pipeline:latest
+          elif [[ "${{ matrix.assay }}" == "atac" || "${{ matrix.assay }}" == "chip" ]]; then
+            apptainer pull --force library://asmith151/seqnado/seqnado_extra:latest
+          elif [[ "${{ matrix.assay }}" == "rna-rx" ]]; then
+            apptainer pull --force library://asmith151/seqnado/seqnado_report:latest
+          fi
+
       - name: Test ${{ matrix.test }} ${{ matrix.assay }}
         shell: pwsh
+        env:
+          TMPDIR: /tmp
         run: |
-          pytest tests/test_pipelines.py::test_${{ matrix.test }}[${{ matrix.assay }}] -vv -s --cores 4
+          pytest tests/test_pipelines.py::test_${{ matrix.test }}[${{ matrix.assay }}] -vv -s --cores 4

tests/test_pipelines.py

@@ -409,13 +409,18 @@ def test_design(design, assay_type):
 
 @pytest.fixture(scope="function", autouse=True)
 def apptainer_args(indicies, test_data_path):
-
     indicies_mount = indicies.parent if not indicies.is_dir() else indicies
     tmpdir = pathlib.Path(os.environ.get("TMPDIR", "/tmp") or "/tmp")
     wd = pathlib.Path(os.getcwd()).resolve()
+    apptainer_cache_dir = pathlib.Path.home() / ".apptainer"
+
     os.environ["APPTAINER_BINDPATH"] = (
-        f"{wd}:{wd}, {test_data_path}:{test_data_path}, {indicies_mount}:{indicies_mount}"
+        f"{wd}:{wd},"
+        f"{test_data_path}:{test_data_path},"
+        f"{indicies_mount}:{indicies_mount},"
+        f"{tmpdir}:{tmpdir}"
     )
+    os.environ["APPTAINER_CACHEDIR"] = str(apptainer_cache_dir)
 
 
 @pytest.fixture(scope="function")