diff --git a/.github/workflows/cloud_tests_full.yml b/.github/workflows/cloud_tests_full.yml new file mode 100644 index 000000000..88a03fa07 --- /dev/null +++ b/.github/workflows/cloud_tests_full.yml @@ -0,0 +1,92 @@ +name: full-sized tests on cloud providers +run-name: Submitting workflow to all cloud providers using full sized data +on: + workflow_dispatch: + inputs: + platform: + description: 'Platform to run test' + required: true + default: 'all' + type: choice + options: + - all + - aws + - azure + - gcp +jobs: + run-full-tests-on-aws: + if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'aws' }} + runs-on: self-hosted + strategy: + matrix: + aligner: ["star_salmon", "star_rsem"] + steps: + - uses: seqeralabs/action-tower-launch@v1 + with: + workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} + access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} + compute_env: ${{ secrets.TOWER_CE_AWS_CPU }} + workdir: "${{ secrets.TOWER_BUCKET_AWS }}/work/rnaseq/work-${{ github.sha }}" + run_name: "aws_rnaseq_full_${{ matrix.aligner }}" + profiles: test_full_aws + parameters: | + { + "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}", + "outdir": "${{ secrets.TOWER_BUCKET_AWS }}/rnaseq/results-${{ github.sha }}/aligner_${{ matrix.aligner }}" + } + wait: false + - uses: actions/upload-artifact@v3 + with: + name: Tower debug log file + path: tower_action_*.log + run-full-tests-on-gcp: + if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'gcp' }} + runs-on: self-hosted + strategy: + matrix: + aligner: ["star_salmon", "star_rsem"] + steps: + - uses: seqeralabs/action-tower-launch@v1 + with: + workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} + access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} + compute_env: ${{ secrets.TOWER_CE_GCP_CPU }} + workdir: "${{ secrets.TOWER_BUCKET_GCP }}/work/rnaseq/work-${{ github.sha }}" + run_name: "gcp_rnaseq_full_${{ matrix.aligner }}" + profiles: test_full_gcp + parameters: | + { + "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}", + "outdir": "${{ secrets.TOWER_BUCKET_GCP }}/rnaseq/results-${{ github.sha }}/aligner_${{ matrix.aligner }}" + } + wait: false + - uses: actions/upload-artifact@v3 + with: + name: Tower debug log file + path: tower_action_*.log + run-full-tests-on-azure: + if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'azure' }} + runs-on: self-hosted + strategy: + matrix: + aligner: ["star_salmon", "star_rsem"] + steps: + - uses: seqeralabs/action-tower-launch@v1 + with: + workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} + access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} + compute_env: ${{ secrets.TOWER_CE_AZURE_CPU }} + workdir: "${{ secrets.TOWER_BUCKET_AZURE }}/work/rnaseq/work-${{ github.sha }}" + run_name: "azure_rnaseq_full_${{ matrix.aligner }}" + profiles: test_full_azure + parameters: | + { + "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}", + "outdir": "${{ secrets.TOWER_BUCKET_AZURE }}/rnaseq/results-${{ github.sha }}/aligner_${{ matrix.aligner }}", + "igenomes_base": "${{ secrets.TOWER_IGENOMES_BASE_AZURE }}" + } + wait: false + - uses: actions/upload-artifact@v3 + with: + name: Tower debug log file + path: tower_action_*.log diff --git a/.github/workflows/cloud_tests_small.yml b/.github/workflows/cloud_tests_small.yml new file mode 100644 index 000000000..b1bd5711d --- /dev/null +++ b/.github/workflows/cloud_tests_small.yml @@ -0,0 +1,79 @@ +name: small-sized tests on cloud providers +run-name: Submitting workflow to all cloud providers using small sized data +on: + workflow_dispatch: + inputs: + platform: + description: 'Platform to run test' + required: true + default: 'all' + type: choice + options: + - all + - aws + - azure + - gcp +jobs: + run-small-tests-on-aws: + if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'aws' }} + runs-on: self-hosted + steps: + - uses: seqeralabs/action-tower-launch@v1 + with: + workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} + access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} + compute_env: ${{ secrets.TOWER_CE_AWS_CPU }} + workdir: "${{ secrets.TOWER_BUCKET_AWS }}/work/rnaseq/work-${{ github.sha }}" + run_name: "aws_rnaseq_small" + profiles: test + parameters: | + { + "outdir": "${{ secrets.TOWER_BUCKET_AWS }}/rnaseq/results-test-${{ github.sha }}" + } + wait: false + - uses: actions/upload-artifact@v3 + with: + name: Tower debug log file + path: tower_action_*.log + run-small-tests-on-gcp: + if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'gcp' }} + runs-on: self-hosted + steps: + - uses: seqeralabs/action-tower-launch@v1 + with: + workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} + access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} + compute_env: ${{ secrets.TOWER_CE_GCP_CPU }} + workdir: "${{ secrets.TOWER_BUCKET_GCP }}/work/rnaseq/work-${{ github.sha }}" + run_name: "gcp_rnaseq_small" + profiles: test + parameters: | + { + "outdir": "${{ secrets.TOWER_BUCKET_GCP }}/rnaseq/results-test-${{ github.sha }}" + } + wait: false + - uses: actions/upload-artifact@v3 + with: + name: Tower debug log file + path: tower_action_*.log + run-small-tests-on-azure: + if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'azure' }} + runs-on: self-hosted + steps: + - uses: seqeralabs/action-tower-launch@v1 + with: + workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} + access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} + compute_env: ${{ secrets.TOWER_CE_AZURE_CPU }} + workdir: "${{ secrets.TOWER_BUCKET_AZURE }}/work/rnaseq/work-${{ github.sha }}" + run_name: "azure_rnaseq_small" + profiles: test + parameters: | + { + "outdir": "${{ secrets.TOWER_BUCKET_AZURE }}/rnaseq/results-test-${{ github.sha }}" + } + wait: false + - uses: actions/upload-artifact@v3 + with: + name: Tower debug log file + path: tower_action_*.log diff --git a/.nf-core.yml b/.nf-core.yml index 40bcac74d..e6c9d79f0 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -3,4 +3,5 @@ lint: files_unchanged: - assets/email_template.html - assets/email_template.txt + - lib/NfcoreSchema.groovy - lib/NfcoreTemplate.groovy diff --git a/CHANGELOG.md b/CHANGELOG.md index 13687f2a4..5dc7f96b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [[3.11.0](https://github.com/nf-core/rnaseq/releases/tag/3.11.0)] - 2023-03-27 +## [[3.11.0](https://github.com/nf-core/rnaseq/releases/tag/3.11.0)] - 2023-03-29 ### Credits @@ -24,16 +24,17 @@ Thank you to everyone else that has contributed by reporting bugs, enhancements - Users can now select between `--trimmer trimgalore` (default) and `--trimmer fastp`. - Trim Galore! specific pipeline parameters have been deprecated: `--clip_r1`, `--clip_r2`, `--three_prime_clip_r1`, `--three_prime_clip_r2` and `--trim_nextseq` - Any additional options can now be specified via the `--extra_trimgalore_args` and `--extra_fastp_args` parameters, respectively. -- [[#663](https://github.com/nf-core/rnaseq/pull/663)] - Alternative trimming step for polyA/T removal -- [[#781](https://github.com/nf-core/rnaseq/pull/781)] - Add Warning for poly(A) libraries -- [[#878](https://github.com/nf-core/rnaseq/pull/878)] - Allow tabs in fasta header when creating decoys for salmon index -- [[#931](https://github.com/nf-core/rnaseq/pull/931)] - Save transcriptome BAM files when using `--save_umi_intermeds` / `--save_align_intermeds` +- [[#663](https://github.com/nf-core/rnaseq/issues/663)] - Alternative trimming step for polyA/T removal +- [[#781](https://github.com/nf-core/rnaseq/issues/781)] - Add Warning for poly(A) libraries +- [[#878](https://github.com/nf-core/rnaseq/issues/878)] - Allow tabs in fasta header when creating decoys for salmon index +- [[#931](https://github.com/nf-core/rnaseq/issues/931)] - Save transcriptome BAM files when using `--save_umi_intermeds` / `--save_align_intermeds` - [[#934](https://github.com/nf-core/rnaseq/pull/934)] - Union of `ext.args` and `params.extra_star_align_args` prevents parameter clashes in the STAR module -- [[#940](https://github.com/nf-core/rnaseq/issues/940)] - Bugfix in `salmon_summarizedexperiment.r` to ensure `rbind` doesn't fail when `rowdata` has no `tx` column. See ([[#941](https://github.com/nf-core/rnaseq/pull/941)]) for details. +- [[#940](https://github.com/nf-core/rnaseq/issues/940)] - Bugfix in `salmon_summarizedexperiment.r` to ensure `rbind` doesn't fail when `rowdata` has no `tx` column. - [[#944](https://github.com/nf-core/rnaseq/issues/944)] - Read clipping using clip_r1, clip_r2, three_prime_clip_r1, three_prime_clip_r2 disabled in 3.10 - [[#956](https://github.com/nf-core/rnaseq/pull/956)] - Implement 'auto' as default strandedness argument in `fastq_dir_to_samplesheet.py` script -- [[#960](https://github.com/nf-core/rnaseq/pull/960)] - Failure with awsbatch when running processes that are using `executor: local` -- [[#961](https://github.com/nf-core/rnaseq/pull/961)] - Add warnings to STDOUT for all skipped and failed strandedness check samples +- [[#960](https://github.com/nf-core/rnaseq/issues/960)] - Failure with awsbatch when running processes that are using `executor: local` +- [[#961](https://github.com/nf-core/rnaseq/issues/961)] - Add warnings to STDOUT for all skipped and failed strandedness check samples +- [[#975](https://github.com/nf-core/rnaseq/issues/975)] - `SALMON_INDEX` runs when using `--aligner star_rsem` even if samples have explicit strandedness - Remove HISAT2 from automated AWS full-sized tests ### Parameters diff --git a/assets/slackreport.json b/assets/slackreport.json index 043d02f27..fc84fa5e5 100644 --- a/assets/slackreport.json +++ b/assets/slackreport.json @@ -3,7 +3,7 @@ { "fallback": "Plain-text summary of the attachment.", "color": "<% if (success) { %>good<% } else { %>danger<%} %>", - "author_name": "sanger-tol/readmapping v${version} - ${runName}", + "author_name": "nf-core/rnaseq ${version} - ${runName}", "author_icon": "https://www.nextflow.io/docs/latest/_static/favicon.ico", "text": "<% if (success) { %>Pipeline completed successfully!<% } else { %>Pipeline completed with errors<% } %>", "fields": [ diff --git a/lib/NfcoreSchema.groovy b/lib/NfcoreSchema.groovy index 33cd4f6e8..4d2968143 100755 --- a/lib/NfcoreSchema.groovy +++ b/lib/NfcoreSchema.groovy @@ -2,6 +2,7 @@ // This file holds several functions used to perform JSON parameter validation, help and summary rendering for the nf-core pipeline template. // +import nextflow.Nextflow import org.everit.json.schema.Schema import org.everit.json.schema.loader.SchemaLoader import org.everit.json.schema.ValidationException @@ -177,7 +178,7 @@ class NfcoreSchema { } if (has_error) { - System.exit(1) + Nextflow.error('Exiting!') } } diff --git a/lib/WorkflowMain.groovy b/lib/WorkflowMain.groovy index 5dd03dd17..eaf9bce23 100755 --- a/lib/WorkflowMain.groovy +++ b/lib/WorkflowMain.groovy @@ -2,6 +2,8 @@ // This file holds several functions specific to the main.nf workflow in the nf-core/rnaseq pipeline // +import nextflow.Nextflow + class WorkflowMain { // @@ -83,8 +85,7 @@ class WorkflowMain { // Check input has been provided if (!params.input) { - log.error "Please provide an input samplesheet to the pipeline e.g. '--input samplesheet.csv'" - System.exit(1) + Nextflow.error("Please provide an input samplesheet to the pipeline e.g. '--input samplesheet.csv'") } } // diff --git a/lib/WorkflowRnaseq.groovy b/lib/WorkflowRnaseq.groovy index 64264ae37..ff577917b 100755 --- a/lib/WorkflowRnaseq.groovy +++ b/lib/WorkflowRnaseq.groovy @@ -2,6 +2,7 @@ // This file holds several functions specific to the workflow/rnaseq.nf in the nf-core/rnaseq pipeline // +import nextflow.Nextflow import groovy.json.JsonSlurper import groovy.text.SimpleTemplateEngine @@ -15,13 +16,11 @@ class WorkflowRnaseq { if (!params.fasta) { - log.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." - System.exit(1) + Nextflow.error("Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file.") } if (!params.gtf && !params.gff) { - log.error "No GTF or GFF3 annotation specified! The pipeline requires at least one of these files." - System.exit(1) + Nextflow.error("No GTF or GFF3 annotation specified! The pipeline requires at least one of these files.") } if (params.gtf) { @@ -41,51 +40,43 @@ class WorkflowRnaseq { } if (!params.skip_bbsplit && !params.bbsplit_index && !params.bbsplit_fasta_list) { - log.error "Please provide either --bbsplit_fasta_list / --bbsplit_index to run BBSplit." - System.exit(1) + Nextflow.error("Please provide either --bbsplit_fasta_list / --bbsplit_index to run BBSplit.") } if (params.remove_ribo_rna && !params.ribo_database_manifest) { - log.error "Please provide --ribo_database_manifest to remove ribosomal RNA with SortMeRNA." - System.exit(1) + Nextflow.error("Please provide --ribo_database_manifest to remove ribosomal RNA with SortMeRNA.") } if (params.with_umi && !params.skip_umi_extract) { if (!params.umitools_bc_pattern && !params.umitools_bc_pattern2) { - log.error "UMI-tools requires a barcode pattern to extract barcodes from the reads." - System.exit(1) + Nextflow.error("UMI-tools requires a barcode pattern to extract barcodes from the reads.") } } if (!params.skip_trimming) { if (!valid_params['trimmers'].contains(params.trimmer)) { - log.error "Invalid option: '${params.trimmer}'. Valid options for '--trimmer': ${valid_params['trimmers'].join(', ')}." - System.exit(1) + Nextflow.error("Invalid option: '${params.trimmer}'. Valid options for '--trimmer': ${valid_params['trimmers'].join(', ')}.") } } if (!params.skip_alignment) { if (!valid_params['aligners'].contains(params.aligner)) { - log.error "Invalid option: '${params.aligner}'. Valid options for '--aligner': ${valid_params['aligners'].join(', ')}." - System.exit(1) + Nextflow.error("Invalid option: '${params.aligner}'. Valid options for '--aligner': ${valid_params['aligners'].join(', ')}.") } } else { if (!params.pseudo_aligner) { - log.error "--skip_alignment specified without --pseudo_aligner...please specify e.g. --pseudo_aligner ${valid_params['pseudoaligners'][0]}." - System.exit(1) + Nextflow.error("--skip_alignment specified without --pseudo_aligner...please specify e.g. --pseudo_aligner ${valid_params['pseudoaligners'][0]}.") } skipAlignmentWarn(log) } if (params.pseudo_aligner) { if (!valid_params['pseudoaligners'].contains(params.pseudo_aligner)) { - log.error "Invalid option: '${params.pseudo_aligner}'. Valid options for '--pseudo_aligner': ${valid_params['pseudoaligners'].join(', ')}." - System.exit(1) + Nextflow.error("Invalid option: '${params.pseudo_aligner}'. Valid options for '--pseudo_aligner': ${valid_params['pseudoaligners'].join(', ')}.") } else { if (!(params.salmon_index || params.transcript_fasta || (params.fasta && (params.gtf || params.gff)))) { - log.error "To use `--pseudo_aligner 'salmon'`, you must provide either --salmon_index or --transcript_fasta or both --fasta and --gtf / --gff." - System.exit(1) + Nextflow.error("To use `--pseudo_aligner 'salmon'`, you must provide either --salmon_index or --transcript_fasta or both --fasta and --gtf / --gff.") } } } @@ -120,8 +111,7 @@ class WorkflowRnaseq { // Check which RSeQC modules we are running def rseqc_modules = params.rseqc_modules ? params.rseqc_modules.split(',').collect{ it.trim().toLowerCase() } : [] if ((valid_params['rseqc_modules'] + rseqc_modules).unique().size() != valid_params['rseqc_modules'].size()) { - log.error "Invalid option: ${params.rseqc_modules}. Valid options for '--rseqc_modules': ${valid_params['rseqc_modules'].join(', ')}" - System.exit(1) + Nextflow.error("Invalid option: ${params.rseqc_modules}. Valid options for '--rseqc_modules': ${valid_params['rseqc_modules'].join(', ')}") } } @@ -159,14 +149,14 @@ class WorkflowRnaseq { def chrom = lspl[0] def size = lspl[1] if (size.toInteger() > max_size) { - log.error "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + + def error_string = "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + " Contig longer than ${max_size}bp found in reference genome!\n\n" + " ${chrom}: ${size}\n\n" + " Provide the '--bam_csi_index' parameter to use a CSI instead of BAI index.\n\n" + " Please see:\n" + " https://github.com/nf-core/rnaseq/issues/744\n" + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - System.exit(1) + Nextflow.error(error_string) } } } @@ -313,12 +303,12 @@ class WorkflowRnaseq { // private static void genomeExistsError(params, log) { if (params.genomes && params.genome && !params.genomes.containsKey(params.genome)) { - log.error "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + + def error_string = "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + " Genome '${params.genome}' not found in any config files provided to the pipeline.\n" + " Currently, the available genome keys are:\n" + " ${params.genomes.keySet().join(", ")}\n" + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - System.exit(1) + Nextflow.error(error_string) } } @@ -384,13 +374,13 @@ class WorkflowRnaseq { // Print a warning if using '--aligner star_rsem' and '--with_umi' // private static void rsemUmiError(log) { - log.error "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + + def error_string = "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + " When using '--aligner star_rsem', STAR is run by RSEM itself and so it is\n" + " not possible to remove UMIs before the quantification.\n\n" + " If you would like to remove UMI barcodes using the '--with_umi' option\n" + " please use either '--aligner star_salmon' or '--aligner hisat2'.\n" + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - System.exit(1) + Nextflow.error(error_string) } // diff --git a/nextflow.config b/nextflow.config index eadc068a2..c5ca21943 100644 --- a/nextflow.config +++ b/nextflow.config @@ -205,8 +205,17 @@ profiles { executor.cpus = 16 executor.memory = 60.GB } - test { includeConfig 'conf/test.config' } - test_full { includeConfig 'conf/test_full.config' } + test { includeConfig 'conf/test.config' } + test_full { includeConfig 'conf/test_full.config' } + test_full_aws { includeConfig 'conf/test_full.config' } + test_full_gcp { + includeConfig 'conf/test_full.config' + params.input = 'https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/samplesheet/v3.10/samplesheet_full_gcp.csv' + } + test_full_azure { + includeConfig 'conf/test_full.config' + params.input = 'https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/samplesheet/v3.10/samplesheet_full_azure.csv' + } } // Load igenomes.config if required diff --git a/workflows/rnaseq.nf b/workflows/rnaseq.nf index 45555593a..87630bead 100755 --- a/workflows/rnaseq.nf +++ b/workflows/rnaseq.nf @@ -243,9 +243,15 @@ workflow RNASEQ { // // SUBWORKFLOW: Sub-sample FastQ files and pseudo-align with Salmon to auto-infer strandedness // + // Return empty channel if ch_strand_fastq.auto_strand is empty so salmon index isn't created + PREPARE_GENOME.out.fasta + .combine(ch_strand_fastq.auto_strand) + .map { it.first() } + .set { ch_genome_fasta } + FASTQ_SUBSAMPLE_FQ_SALMON ( ch_strand_fastq.auto_strand, - PREPARE_GENOME.out.fasta, + ch_genome_fasta, PREPARE_GENOME.out.transcript_fasta, PREPARE_GENOME.out.gtf, PREPARE_GENOME.out.salmon_index,