Merge pull request #157 from drpatelh/fixes

Add public_aws_ecr.config and use in CI tests
nf-core · May 15, 2023 · 719dde4 · 719dde4
2 parents ffc3622 + f510057
commit 719dde4
Show file tree

Hide file tree

Showing 7 changed files with 35 additions and 6 deletions.
diff --git a/.github/workflows/cloud_tests_full.yml b/.github/workflows/cloud_tests_full.yml
@@ -27,7 +27,7 @@ jobs:
           compute_env: ${{ secrets.TOWER_CE_AWS_CPU }}
           workdir: "${{ secrets.TOWER_BUCKET_AWS }}/work/fetchngs/work-${{ github.sha }}"
           run_name: "aws_fetchngs_full"
-          profiles: test_full
+          profiles: test_full,public_aws_ecr
           parameters: |
             {
                 "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",

diff --git a/.github/workflows/cloud_tests_small.yml b/.github/workflows/cloud_tests_small.yml
@@ -25,7 +25,7 @@ jobs:
           compute_env: ${{ secrets.TOWER_CE_AWS_CPU }}
           workdir: "${{ secrets.TOWER_BUCKET_AWS }}/work/fetchngs/work-${{ github.sha }}"
           run_name: "aws_fetchngs_small"
-          profiles: test
+          profiles: test,public_aws_ecr
           parameters: |
             {
                 "outdir": "${{ secrets.TOWER_BUCKET_AWS }}/fetchngs/results-test-${{ github.sha }}"

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,7 +3,7 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [[1.10.0](https://github.com/nf-core/fetchngs/releases/tag/1.10.0)] - 2023-05-12
+## [[1.10.0](https://github.com/nf-core/fetchngs/releases/tag/1.10.0)] - 2023-05-15
 
 ### Credits
 
@@ -14,6 +14,7 @@ Special thanks to the following for their contributions to the release:
 - [Maxime Garcia](https://github.com/maxulysse)
 - [Moritz E. Beber](https://github.com/Midnighter)
 - [Rob Syme](https://github.com/robsyme)
+- [sirclockalot](https://github.com/sirclockalot)
 
 Thank you to everyone else that has contributed by reporting bugs, enhancements or in any other way, shape or form.
 
@@ -26,6 +27,7 @@ Thank you to everyone else that has contributed by reporting bugs, enhancements
 - [PR #147](https://github.com/nf-core/fetchngs/pull/147) - Updated pipeline template to [nf-core/tools 2.8](https://github.com/nf-core/tools/releases/tag/2.8)
 - [PR #148](https://github.com/nf-core/fetchngs/pull/148) - Fix default metadata fields for ENA API v2.0
 - [PR #150](https://github.com/nf-core/fetchngs/pull/150) - Add infrastructure and CI for multi-cloud full-sized tests run via Nextflow Tower
+- [PR #157](https://github.com/nf-core/fetchngs/pull/157) - Add `public_aws_ecr.config` to source mulled containers when using `public.ecr.aws` Docker Biocontainer registry
 
 ### Software dependencies
 

diff --git a/conf/public_aws_ecr.config b/conf/public_aws_ecr.config
@@ -0,0 +1,24 @@
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    AWS ECR Config
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Config to set public AWS ECR images wherever possible
+    This improves speed when running on AWS infrastructure.
+    Use this as an example template when using your own private registry.
+----------------------------------------------------------------------------------------
+*/
+
+docker.registry = 'public.ecr.aws'
+podman.registry = 'public.ecr.aws'
+
+process {
+    withName: '.*:SRATOOLS_FASTERQDUMP' {
+        container = 'quay.io/biocontainers/mulled-v2-5f89fe0cd045cb1d615630b9261a1d17943a9b6a:6a9ff0e76ec016c3d0d27e0c0d362339f2d787e6-0'
+    }
+    withName: '.*:SRA_MERGE_SAMPLESHEET' {
+        container = 'quay.io/nf-core/ubuntu:20.04'
+    }
+    withName: '.*:SYNAPSE_MERGE_SAMPLESHEET' {
+        container = 'quay.io/nf-core/ubuntu:20.04'
+    }
+}
diff --git a/modules.json b/modules.json
@@ -17,7 +17,7 @@
                     },
                     "sratools/fasterqdump": {
                         "branch": "master",
-                        "git_sha": "10cb20f6a130d104fef335a8290f3ffce650f28d",
+                        "git_sha": "603ecbd9f45300c9788f197d2a15a005685b4220",
                         "installed_by": ["fastq_download_prefetch_fasterqdump_sratools"]
                     },
                     "sratools/prefetch": {

diff --git a/modules/nf-core/sratools/fasterqdump/main.nf b/modules/nf-core/sratools/fasterqdump/main.nf
diff --git a/nextflow.config b/nextflow.config
@@ -158,6 +158,9 @@ profiles {
         executor.cpus          = 16
         executor.memory        = 60.GB
     }
+    public_aws_ecr {
+        includeConfig 'conf/public_aws_ecr.config'
+    }
     test         { includeConfig 'conf/test.config'         }
     test_synapse { includeConfig 'conf/test_synapse.config' }
     test_full    { includeConfig 'conf/test_full.config'    }
@@ -208,7 +211,7 @@ manifest {
     description     = """Pipeline to fetch metadata and raw FastQ files from public databases"""
     mainScript      = 'main.nf'
     nextflowVersion = '!>=22.10.1'
-    version         = '1.10.0dev'
+    version         = '1.10.0'
     doi             = 'https://doi.org/10.5281/zenodo.5070524'
 }