Workflow output definition

assets/schema_mappings.yml

@@ -0,0 +1,28 @@
+$schema: 'http://json-schema.org/draft-07/schema'
+$id: 'https://raw.githubusercontent.com/nf-core/fetchngs/master/assets/schema_mappings.yml'
+title: 'nf-core/fetchngs pipeline - id_mappings.csv schema'
+description: 'Schema for the mappings file produced by fetchngs'
+type: array
+items:
+  type: object
+  properties:
+    sample:
+      type: string
+    experiment_accession:
+      type: string
+    run_accession:
+      type: string
+    sample_accession:
+      type: string
+    experiment_alias:
+      type: string
+    run_alias:
+      type: string
+    sample_alias:
+      type: string
+    experiment_title:
+      type: string
+    sample_title:
+      type: string
+    sample_description:
+      type: string

assets/schema_samplesheet.yml

@@ -0,0 +1,81 @@
+$schema: 'http://json-schema.org/draft-07/schema'
+$id: 'https://raw.githubusercontent.com/nf-core/fetchngs/master/assets/schema_mappings.yml'
+title: 'nf-core/fetchngs pipeline - samplesheet.csv schema'
+description: 'Schema for the samplesheet file produced by fetchngs'
+type: array
+items:
+  type: object
+  properties:
+    sample:
+      type: string
+    fastq_1:
+      type: string
+      format: file-path
+      pattern: '^\\S+\\.f(ast)?q\\.gz$'
+    fastq_2:
+      type: string
+      format: file-path
+      pattern: '^\\S+\\.f(ast)?q\\.gz$'
+    run_accession:
+      type: string
+    experiment_accession:
+      type: string
+    sample_accession:
+      type: string
+    secondary_sample_accession:
+      type: string
+    study_accession:
+      type: string
+    secondary_study_accession:
+      type: string
+    submission_accession:
+      type: string
+    run_alias:
+      type: string
+    experiment_alias:
+      type: string
+    sample_alias:
+      type: string
+    study_alias:
+      type: string
+    library_layout:
+      type: string
+    library_selection:
+      type: string
+    library_source:
+      type: string
+    library_strategy:
+      type: string
+    library_name:
+      type: string
+    instrument_model:
+      type: string
+    instrument_platform:
+      type: string
+    base_count:
+      type: integer
+    read_count:
+      type: integer
+    tax_id:
+      type: string
+    scientific_name:
+      type: string
+    sample_title:
+      type: string
+    experiment_title:
+      type: string
+    study_title:
+      type: string
+    sample_description:
+      type: string
+    fastq_md5:
+      type: string
+      pattern: '^[0-9a-f]{32}$'
+    fastq_bytes:
+      type: integer
+    fastq_ftp:
+      type: string
+    fastq_galaxy:
+      type: string
+    fastq_aspera:
+      type: string

conf/base.config

@@ -14,12 +14,6 @@ process {
     memory = { check_max( 6.GB * task.attempt, 'memory' ) }
     time   = { check_max( 4.h  * task.attempt, 'time'   ) }
 
-    publishDir = [
-        path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" },
-        mode: params.publish_dir_mode,
-        saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-    ]
-
     errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' }
     maxRetries    = 1
     maxErrors     = '-1'

main.nf

@@ -10,6 +10,7 @@
 */
 
 nextflow.enable.dsl = 2
+nextflow.preview.output = true
 
 /*
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -83,6 +84,11 @@ workflow {
     )
 }
 
+output {
+    directory params.outdir
+    mode params.publish_dir_mode
+}
+
 /*
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     THE END

modules/local/aspera_cli/nextflow.config

@@ -1,17 +1,5 @@
 process {
     withName: 'ASPERA_CLI' {
         ext.args = '-QT -l 300m -P33001'
-        publishDir = [
-            [
-                path: { "${params.outdir}/fastq" },
-                mode: params.publish_dir_mode,
-                pattern: "*.fastq.gz"
-            ],
-            [
-                path: { "${params.outdir}/fastq/md5" },
-                mode: params.publish_dir_mode,
-                pattern: "*.md5"
-            ]
-        ]
     }
 }

modules/local/sra_fastq_ftp/nextflow.config

@@ -1,17 +1,5 @@
 process {
     withName: 'SRA_FASTQ_FTP' {
         ext.args = '-t 5 -nv -c -T 60'
-        publishDir = [
-            [
-                path: { "${params.outdir}/fastq" },
-                mode: params.publish_dir_mode,
-                pattern: "*.fastq.gz"
-            ],
-            [
-                path: { "${params.outdir}/fastq/md5" },
-                mode: params.publish_dir_mode,
-                pattern: "*.md5"
-            ]
-        ]
     }
 }

output.yml

@@ -0,0 +1,16 @@
+$schema: 'http://json-schema.org/draft-07/schema'
+$id: 'https://raw.githubusercontent.com/nf-core/fetchngs/master/output.yml'
+title: 'nf-core/fetchngs pipeline outputs'
+description: ''
+type: object
+properties:
+  id_mappings:
+    type: string
+    format: file-path
+    mimetype: text/csv
+    schema: assets/schema_mappings.yml
+  samplesheet:
+    type: string
+    format: file-path
+    mimetype: text/csv
+    schema: assets/schema_samplesheet.yml

workflows/sra/main.nf

@@ -123,6 +123,7 @@ workflow SRA {
             .fastq
             .mix(SRA_FASTQ_FTP.out.fastq)
             .mix(FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS.out.reads)
+            .tap { ch_fastq }
             .map {
                 meta, fastq ->
                     def reads = fastq instanceof List ? fastq.flatten() : [ fastq ]
@@ -153,7 +154,7 @@ workflow SRA {
         .map { it[1] }
         .collectFile(name:'tmp_samplesheet.csv', newLine: true, keepHeader: true, sort: { it.baseName })
         .map { it.text.tokenize('\n').join('\n') }
-        .collectFile(name:'samplesheet.csv', storeDir: "${params.outdir}/samplesheet")
+        .collectFile(name:'samplesheet.csv')
         .set { ch_samplesheet }
 
     SRA_TO_SAMPLESHEET
@@ -162,7 +163,7 @@ workflow SRA {
         .map { it[1] }
         .collectFile(name:'tmp_id_mappings.csv', newLine: true, keepHeader: true, sort: { it.baseName })
         .map { it.text.tokenize('\n').join('\n') }
-        .collectFile(name:'id_mappings.csv', storeDir: "${params.outdir}/samplesheet")
+        .collectFile(name:'id_mappings.csv')
         .set { ch_mappings }
 
     //
@@ -181,14 +182,25 @@ workflow SRA {
     // Collate and save software versions
     //
     softwareVersionsToYAML(ch_versions)
-        .collectFile(storeDir: "${params.outdir}/pipeline_info", name: 'nf_core_fetchngs_software_mqc_versions.yml', sort: true, newLine: true)
+        .collectFile(name: 'nf_core_fetchngs_software_mqc_versions.yml', sort: true, newLine: true)
+        .set { ch_versions_yml }
 
     emit:
     samplesheet     = ch_samplesheet
     mappings        = ch_mappings
     sample_mappings = ch_sample_mappings_yml
     sra_metadata    = ch_sra_metadata
     versions        = ch_versions.unique()
+
+    publish:
+    ch_fastq                    >> 'fastq/'
+    ASPERA_CLI.out.md5          >> 'fastq/md5/'
+    SRA_FASTQ_FTP.out.md5       >> 'fastq/md5/'
+    SRA_RUNINFO_TO_FTP.out.tsv  >> 'metadata/'
+    ch_versions_yml             >> 'pipeline_info/'
+    ch_samplesheet              >> 'samplesheet/'
+    ch_mappings                 >> 'samplesheet/'
+    ch_sample_mappings_yml      >> 'samplesheet/'
 }
 
 /*

workflows/sra/nextflow.config

@@ -1,8 +1,3 @@
-includeConfig "../../modules/local/multiqc_mappings_config/nextflow.config"
 includeConfig "../../modules/local/aspera_cli/nextflow.config"
 includeConfig "../../modules/local/sra_fastq_ftp/nextflow.config"
-includeConfig "../../modules/local/sra_ids_to_runinfo/nextflow.config"
-includeConfig "../../modules/local/sra_runinfo_to_ftp/nextflow.config"
-includeConfig "../../modules/local/sra_to_samplesheet/nextflow.config"
-includeConfig "../../modules/nf-core/sratools/prefetch/nextflow.config"
 includeConfig "../../subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config"