Merge pull request #1484 from nf-core/dev

Release 3.4.2

.github/workflows/ci.yml

@@ -256,6 +256,10 @@ jobs:
           - tags: "tabix/tabix"
           - tags: "tiddit/sv"
           - tags: "untar"
+          - tags: "pipeline_sarek"
+        include:
+          - tags: "pipeline_sarek"
+            profile: "test,docker"
     env:
       NXF_ANSI_LOG: false
       TEST_DATA_BASE: "${{ github.workspace }}/test-datasets"

.github/workflows/cloudtest.yml

@@ -31,79 +31,76 @@ on:
         default: true
 
 jobs:
-  trigger-profile-test:
-    name: Run AWS tests
+  trigger-test:
+    name: launch
     runs-on: ubuntu-latest
+    if: ${{ github.repository == 'nf-core/sarek' }}
     strategy:
       fail-fast: false
       matrix:
         include:
           - profile: test_aws
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.test && inputs.aws ) ) }}
+            test: test
+            cloud: aws
+            compute_env: TOWER_COMPUTE_ENV
+            workdir: TOWER_BUCKET_AWS
           - profile: test_azure
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.test && inputs.azure ) ) }}
+            test: test
+            cloud: azure
+            compute_env: TOWER_CE_AZURE_CPU
+            workdir: TOWER_BUCKET_AZURE
           - profile: test_full_aws
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.somatic && inputs.aws ) ) }}
+            test: somatic
+            cloud: aws
+            compute_env: TOWER_COMPUTE_ENV
+            workdir: TOWER_BUCKET_AWS
           - profile: test_full_azure
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.somatic && inputs.azure ) ) }}
+            test: somatic
+            cloud: azure
+            compute_env: TOWER_CE_AZURE_CPU
+            workdir: TOWER_BUCKET_AZURE
           - profile: test_full_germline_aws
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.germline && inputs.aws ) ) }}
+            test: germline
+            cloud: aws
+            compute_env: TOWER_COMPUTE_ENV
+            workdir: TOWER_BUCKET_AWS
           - profile: test_full_germline_azure
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.germline && inputs.azure ) ) }}
-          - profile: test_full_germline_ncbench_agilent_aws
-            enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || ( inputs.germline_ncbench_agilent && inputs.aws ) ) }}
+            test: germline
+            cloud: azure
+            compute_env: TOWER_CE_AZURE_CPU
+            workdir: TOWER_BUCKET_AZURE
+          - profile: test_full_germline_ncbench_agilent
+            test: germline_ncbench_agilent
+            cloud: aws
+            compute_env: TOWER_COMPUTE_ENV
+            workdir: TOWER_BUCKET_AWS
 
     steps:
       # Launch workflow on AWS Batch
-      - name: AWS Launch
+      - name: Launch
         uses: seqeralabs/action-tower-launch@v2
-        if: ${{ matrix.enabled && ( github.event_name != 'workflow_dispatch' || inputs.aws ) }}
+        # If inputs item exists (i.e. workflow_dispatch), then we find matrix.test and check it is false
+        # If is false, we negate and run the job
+        if: ( !contains(inputs[matrix.test], 'false') && !contains(inputs[matrix.cloud], 'false') )
         with:
           run_name: sarek_${{ matrix.profile }}
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
-          compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
+          compute_env: ${{ secrets[matrix.compute_env] }}
           revision: ${{ github.sha }}
-          workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/sarek/work-${{ github.sha }}/${{ matrix.profile }}
+          workdir: ${{ secrets[matrix.workdir] }}/work/sarek/work-${{ github.sha }}/${{ matrix.profile }}
           parameters: |
             {
               "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
-              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/sarek/results-${{ github.sha }}/${{ matrix.profile }}/"
+              "outdir": "${{ secrets[matrix.workdir] }}/sarek/results-${{ github.sha }}/${{ matrix.profile }}/"
             }
           profiles: ${{ matrix.profile }}
 
       - uses: actions/upload-artifact@v3
-        name: Save AWS Logs
+        name: Save Logs
         if: success() || failure()
         with:
-          name: tower-aws-${{ matrix.profile }}-log
-          path: |
-            tower_action_*.log
-            tower_action_*.json
-
-      # Launch workflow using Tower CLI tool action
-      - name: Azure Launch
-        uses: seqeralabs/action-tower-launch@v2
-        if: ${{ matrix.enabled && ( github.event_name != 'workflow_dispatch' || inputs.azure ) }}
-        with:
-          run_name: sarek_${{ matrix.profile }}
-          workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
-          access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
-          compute_env: ${{ secrets.TOWER_CE_AZURE_CPU }}
-          revision: ${{ github.sha }}
-          workdir: ${{ secrets.TOWER_BUCKET_AZURE}}/sarek/work-${{ github.sha }}/${{ matrix.profile }}
-          parameters: |
-            {
-              "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
-              "outdir": "${{ secrets.TOWER_BUCKET_AZURE }}/sarek/results-${{ github.sha }}/${{ matrix.profile }}/"
-            }
-          profiles: ${{ matrix.profile }}
-
-      - uses: actions/upload-artifact@v3
-        name: Save Azure Logs
-        if: success() || failure()
-        with:
-          name: tower-azure-${{ matrix.profile }}-log
+          name: tower-${{ matrix.profile }}-log
           path: |
             tower_action_*.log
             tower_action_*.json

CHANGELOG.md

@@ -5,6 +5,52 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.4.2](https://github.com/nf-core/sarek/releases/tag/3.4.2) - Sájtáristjåhkkå
+
+Sájtáristjåhkkå is another peak (just under 2k) in the Pårte massif, it is one of the few peaks in Sweden that cannot be climbed without proper climbing equipment.
+
+### Added
+
+- [#1489](https://github.com/nf-core/sarek/pull/1489) - Added a `testdata.nf-core.sarek` key in `conf/igenomes.config` for small reference
+- [#1493](https://github.com/nf-core/sarek/pull/1493) - Added a `wave` profile
+- [#1498](https://github.com/nf-core/sarek/pull/1498) - Prepare release `3.4.2`
+
+### Changed
+
+- [#1477](https://github.com/nf-core/sarek/pull/1477) - Back to dev
+- [#1482](https://github.com/nf-core/sarek/pull/1482) - Pin `nf-prov` plugin to `1.2.2`
+- [#1485](https://github.com/nf-core/sarek/pull/1485) - Update citation for publication
+- [#1487](https://github.com/nf-core/sarek/pull/1487) - Update sentieon-modules to Sentieon `202308.02`
+- [#1490](https://github.com/nf-core/sarek/pull/1490) - Update mosdepth to `0.3.8`
+- [#1505](https://github.com/nf-core/sarek/pull/1505) - Update CITATIONS.md
+- [#1506](https://github.com/nf-core/sarek/pull/1506) - Fixing typos (`index_alignement` -> `index_alignment`)
+- [#1509](https://github.com/nf-core/sarek/pull/1509) - Update contributors
+
+### Fixed
+
+- [#1378](https://github.com/nf-core/sarek/pull/1378) - Improve cloud tests launch workflow to use matrix
+- [#1488](https://github.com/nf-core/sarek/pull/1488) - Fixing call to `GATK4_HAPLOTYPECALLER` and thereby also the test-profile `test_full_germline`
+- [#1494](https://github.com/nf-core/sarek/pull/1494) - Fix Cloud Storage objects are immutable on GCP [#1491](https://github.com/nf-core/sarek/issues/1491)
+- [#1496](https://github.com/nf-core/sarek/pull/1496) - Fix multiple DOI handling in manifest
+- [#1499](https://github.com/nf-core/sarek/pull/1499) - Remove all md5sum for mosdepth tests
+- [#1499](https://github.com/nf-core/sarek/pull/1499) - Add mosdepth dependency to all tests runnning it
+- [#1501](https://github.com/nf-core/sarek/pull/1501) - Remove string "None" param option from ascat_genome
+
+### Removed
+
+- [#1489](https://github.com/nf-core/sarek/pull/1489) - Remove `test_cache` profile
+
+### Dependencies
+
+| Dependency | Old version | New version |
+| ---------- | ----------- | ----------- |
+| `mosdepth` | 0.3.6       | 0.3.8       |
+| `sentieon` | 202308.01   | 202308.02   |
+
+### Modules / Subworkflows
+
+### Parameters
+
 ## [3.4.1](https://github.com/nf-core/sarek/releases/tag/3.4.1) - Balgattjåhkkå
 
 Balgattjåhkkå is the other top peak (over 2k m) in the Pårte massif, the other one being Pårtetjåkko (Bårddetjåhkkå).
@@ -71,16 +117,16 @@ Balgattjåhkkå is the other top peak (over 2k m) in the Pårte massif, the othe
 
 ### Dependencies
 
-| Dependency | Old version | New version |
-| ---------- | ----------- | ----------- |
-| bcftools   | 1.17        | 1.18        |
-| ensemblvep | 110.0       | 111.0       |
-| fgbio      | 2.0.2       | 2.1.0       |
-| gatk       | 4.4.0.0     | 4.5.0.0     |
-| gatk-spark | 4.4.0.0     | 4.5.0.0     |
-| mosdepth   | 0.3.3       | 0.3.6       |
-| multiqc    | 1.17        | 1.18        |
-| samtools   | 1.17        | 1.19.2      |
+| Dependency   | Old version | New version |
+| ------------ | ----------- | ----------- |
+| `bcftools`   | 1.17        | 1.18        |
+| `ensemblvep` | 110.0       | 111.0       |
+| `fgbio`      | 2.0.2       | 2.1.0       |
+| `gatk`       | 4.4.0.0     | 4.5.0.0     |
+| `gatk-spark` | 4.4.0.0     | 4.5.0.0     |
+| `mosdepth`   | 0.3.3       | 0.3.6       |
+| `multiqc`    | 1.17        | 1.18        |
+| `samtools`   | 1.17        | 1.19.2      |
 
 ### Modules / Subworkflows
 
@@ -89,9 +135,9 @@ Balgattjåhkkå is the other top peak (over 2k m) in the Pårte massif, the othe
 
 ### Parameter
 
-| Old name                   | New name                 |
-| -------------------------- | ------------------------ |
-| bcftools_annotations_index | bcftools_annotations_tbi |
+| Old name                     | New name                   |
+| ---------------------------- | -------------------------- |
+| `bcftools_annotations_index` | `bcftools_annotations_tbi` |
 
 ## [3.4.0](https://github.com/nf-core/sarek/releases/tag/3.4.0) - Pårtetjåkko
 
@@ -141,8 +187,8 @@ Pårtetjåkko is a mountain in the south of the park.
 
 | Dependency | Old version | New version |
 | ---------- | ----------- | ----------- |
-| fastqc     | 0.11.9      | 0.12.1      |
-| multiqc    | 1.15        | 1.17        |
+| `fastqc`   | 0.11.9      | 0.12.1      |
+| `multiqc`  | 1.15        | 1.17        |
 
 ### Modules / Subworkflows
 
@@ -175,9 +221,9 @@ Ráhpajávvre is the Lule Sámi spelling of Rapaselet.
 
 ### Dependencies
 
-| Dependency                       | Old version | New version |
-| -------------------------------- | ----------- | ----------- |
-| Control-FREEC/assesssignificance | 11.6b       | 11.6        |
+| Dependency                         | Old version | New version |
+| ---------------------------------- | ----------- | ----------- |
+| `Control-FREEC/assesssignificance` | 11.6b       | 11.6        |
 
 ## [3.3.1](https://github.com/nf-core/sarek/releases/tag/3.3.1) - Biellorippjávrre
 
@@ -201,9 +247,9 @@ A lake near the Rapaselet delta.
 
 ### Dependencies
 
-| Dependency    | Old version | New version |
-| ------------- | ----------- | ----------- |
-| Control-FREEC | 11.6        | 11.6b       |
+| Dependency      | Old version | New version |
+| --------------- | ----------- | ----------- |
+| `Control-FREEC` | 11.6        | 11.6b       |
 
 ## [3.3.0](https://github.com/nf-core/sarek/releases/tag/3.3.0) - Rapaselet
 
@@ -320,7 +366,7 @@ Vuoinesluobbalah is a lake close to Bierikjávrre.
 
 ### Fixed
 
-- [#1087](https://github.com/nf-core/sarek/pull/1087) - Fix wrong default memory in GATK4_CREATESEQUENCEDICTIONARY [#1085](https://github.com/nf-core/sarek/pull/1085)
+- [#1087](https://github.com/nf-core/sarek/pull/1087) - Fix wrong default memory in `GATK4_CREATESEQUENCEDICTIONARY` [#1085](https://github.com/nf-core/sarek/pull/1085)
 - [#1089](https://github.com/nf-core/sarek/pull/1089) - Remove duplicated code
 - [#1093](https://github.com/nf-core/sarek/pull/1093) - Fixing Ascat by reverting meta.id in channels allele_files, loci_files, gc_file and rt_file to baseName
 - [#1098](https://github.com/nf-core/sarek/pull/1098) - Fix Channel issue in Mutect2 subworkflow [#1094](https://github.com/nf-core/sarek/pull/1094)

CITATIONS.md

@@ -70,6 +70,8 @@
 
 - [FGBio](https://github.com/fulcrumgenomics/fgbio)
 
+  > doi: 10.5281/zenodo.10456900
+
 - [FreeBayes](https://arxiv.org/abs/1207.3907)
 
   > Garrison E, Marth G. Haplotype-based variant detection from short-read sequencing. arXiv preprint arXiv:1207.3907 [q-bio.GN] 2012. doi: 10.48550/arXiv.1207.3907

README.md

@@ -153,13 +153,15 @@ We thank the following people for their extensive assistance in the development
 - [Johannes Alneberg](https://github.com/alneberg)
 - [José Fernández Navarro](https://github.com/jfnavarro)
 - [Júlia Mir Pedrol](https://github.com/mirpedrol)
+- [Ken Brewer](https://github.com/kenibrewer)
 - [Lasse Westergaard Folkersen](https://github.com/lassefolkersen)
 - [Lucia Conde](https://github.com/lconde-ucl)
 - [Malin Larsson](https://github.com/malinlarsson)
 - [Marcel Martin](https://github.com/marcelm)
 - [Nick Smith](https://github.com/nickhsmith)
 - [Nicolas Schcolnicov](https://github.com/nschcolnicov)
 - [Nilesh Tawari](https://github.com/nilesh-tawari)
+- [Nils Homer](https://github.com/nh13)
 - [Olga Botvinnik](https://github.com/olgabot)
 - [Oskar Wacker](https://github.com/WackerO)
 - [pallolason](https://github.com/pallolason)
@@ -169,6 +171,7 @@ We thank the following people for their extensive assistance in the development
 - [Sam Minot](https://github.com/sminot)
 - [Sebastian-D](https://github.com/Sebastian-D)
 - [Silvia Morini](https://github.com/silviamorins)
+- [Simon Pearce](https://github.com/SPPearce)
 - [Solenne Correard](https://github.com/scorreard)
 - [Susanne Jodoin](https://github.com/SusiJo)
 - [Szilveszter Juhos](https://github.com/szilvajuhos)
@@ -193,7 +196,7 @@ For further information or help, don't hesitate to get in touch on the [Slack `#
 
 If you use `nf-core/sarek` for your analysis, please cite the `Sarek` article as follows:
 
-> Friederike Hanssen, Maxime U Garcia, Lasse Folkersen, Anders Sune Pedersen, Francesco Lescai, Susanne Jodoin, Edmund Miller, Oskar Wacker, Nicholas Smith, nf-core community, Gisela Gabernet, Sven Nahnsen **Scalable and efficient DNA sequencing analysis on different compute infrastructures aiding variant discovery** _bioRxiv_ [doi: 10.1101/2023.07.19.549462](https://doi.org/10.1101/2023.07.19.549462).
+> Friederike Hanssen, Maxime U Garcia, Lasse Folkersen, Anders Sune Pedersen, Francesco Lescai, Susanne Jodoin, Edmund Miller, Oskar Wacker, Nicholas Smith, nf-core community, Gisela Gabernet, Sven Nahnsen **Scalable and efficient DNA sequencing analysis on different compute infrastructures aiding variant discovery** _NAR Genomics and Bioinformatics_ Volume 6, Issue 2, June 2024, lqae031, [doi: 10.1093/nargab/lqae031](https://doi.org/10.1093/nargab/lqae031).
 
 > Garcia M, Juhos S, Larsson M et al. **Sarek: A portable workflow for whole-genome sequencing analysis of germline and somatic variants [version 2; peer review: 2 approved]** _F1000Research_ 2020, 9:63 [doi: 10.12688/f1000research.16665.2](http://dx.doi.org/10.12688/f1000research.16665.2).
 

assets/multiqc_config.yml

@@ -3,9 +3,9 @@ custom_logo_url: https://github.com/nf-core/sarek/
 custom_logo_title: "nf-core/sarek"
 
 report_comment: >
-  This report has been generated by the <a href="https://github.com/nf-core/sarek/releases/tag/3.4.1" target="_blank">nf-core/sarek</a>
+  This report has been generated by the <a href="https://github.com/nf-core/sarek/releases/tag/3.4.2" target="_blank">nf-core/sarek</a>
   analysis pipeline. For information about how to interpret these results, please see the
-  <a href="https://nf-co.re/sarek/3.4.1/docs/output" target="_blank">documentation</a>.
+  <a href="https://nf-co.re/sarek/3.4.2/docs/output" target="_blank">documentation</a>.
 report_section_order:
   "nf-core-sarek-methods-description":
     order: -1000

conf/igenomes.config

@@ -320,5 +320,25 @@ params {
             fasta                 = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Sequence/WholeGenomeFasta/genome.fa"
             readme                = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Annotation/README.txt"
         }
+        'testdata.nf-core.sarek' {
+            dbsnp                   = "${params.igenomes_base}/genomics/homo_sapiens/genome/vcf/dbsnp_146.hg38.vcf.gz"
+            dbsnp_tbi               = "${params.igenomes_base}/genomics/homo_sapiens/genome/vcf/dbsnp_146.hg38.vcf.gz.tbi"
+            dbsnp_vqsr              = '--resource:dbsnp,known=false,training=true,truth=false,prior=2.0 dbsnp_146.hg38.vcf.gz'
+            dict                    = "${params.igenomes_base}/genomics/homo_sapiens/genome/genome.dict"
+            fasta                   = "${params.igenomes_base}/genomics/homo_sapiens/genome/genome.fasta"
+            fasta_fai               = "${params.igenomes_base}/genomics/homo_sapiens/genome/genome.fasta.fai"
+            germline_resource       = "${params.igenomes_base}/genomics/homo_sapiens/genome/vcf/gnomAD.r2.1.1.vcf.gz"
+            germline_resource_tbi   = "${params.igenomes_base}/genomics/homo_sapiens/genome/vcf/gnomAD.r2.1.1.vcf.gz.tbi"
+            intervals               = "${params.igenomes_base}/genomics/homo_sapiens/genome/genome.interval_list"
+            known_indels            = "${params.igenomes_base}/genomics/homo_sapiens/genome/vcf/mills_and_1000G.indels.vcf.gz"
+            known_indels_tbi        = "${params.igenomes_base}/genomics/homo_sapiens/genome/vcf/mills_and_1000G.indels.vcf.gz.tbi"
+            known_indels_vqsr       = '--resource:mills,known=false,training=true,truth=true,prior=10.0 mills_and_1000G.indels.vcf.gz'
+            ngscheckmate_bed        = "${params.igenomes_base}/genomics/homo_sapiens/genome/chr21/germlineresources/SNP_GRCh38_hg38_wChr.bed"
+            snpeff_db               = '105'
+            snpeff_genome           = 'WBcel235'
+            vep_cache_version       = '110'
+            vep_genome              = 'WBcel235'
+            vep_species             = 'caenorhabditis_elegans'
+        }
     }
 }