From a3ddd3ea6746907324e610c23afc5f1b655d03e7 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Tue, 23 Oct 2018 14:37:40 +0200 Subject: [PATCH 1/2] add publishDirMode param and docs --- annotate.nf | 14 +++++++------- buildContainers.nf | 2 +- buildReferences.nf | 8 ++++---- conf/base.config | 1 + docs/PARAMETERS.md | 15 +++++++++++++++ lib/SarekUtils.groovy | 2 ++ main.nf | 12 ++++++------ runMultiQC.nf | 4 ++-- somaticVC.nf | 26 +++++++++++++------------- 9 files changed, 51 insertions(+), 33 deletions(-) diff --git a/annotate.nf b/annotate.nf index d8a1bd9d2b..396c217dd8 100644 --- a/annotate.nf +++ b/annotate.nf @@ -103,7 +103,7 @@ vcfForVep = vcfForVep.map { process RunBcftoolsStats { tag {vcf} - publishDir directoryMap.bcftoolsStats, mode: 'link' + publishDir directoryMap.bcftoolsStats, mode: params.publishDirMode input: set variantCaller, file(vcf) from vcfForBCFtools @@ -124,7 +124,7 @@ if (params.verbose) bcfReport = bcfReport.view { process RunVcftools { tag {vcf} - publishDir directoryMap.vcftools, mode: 'link' + publishDir directoryMap.vcftools, mode: params.publishDirMode input: set variantCaller, file(vcf) from vcfForVCFtools @@ -145,7 +145,7 @@ if (params.verbose) vcfReport = vcfReport.view { process RunSnpeff { tag {"${variantCaller} - ${vcf}"} - publishDir params.outDir, mode: 'link', saveAs: { + publishDir params.outDir, mode: params.publishDirMode, saveAs: { if (it == "${vcf.simpleName}_snpEff.csv") "${directoryMap.snpeffReports}/${it}" else if (it == "${vcf.simpleName}_snpEff.ann.vcf") null else "${directoryMap.snpeff}/${it}" @@ -198,7 +198,7 @@ if('merge' in tools) { process RunVEP { tag {"${variantCaller} - ${vcf}"} - publishDir params.outDir, mode: 'link', saveAs: { + publishDir params.outDir, mode: params.publishDirMode, saveAs: { if (it == "${vcf.simpleName}_VEP.summary.html") "${directoryMap.vep}/${it}" else null } @@ -245,7 +245,7 @@ vcfToCompress = snpeffVCF.mix(vepVCF) process CompressVCF { tag {"${annotator} - ${vcf}"} - publishDir "${directoryMap."$finalannotator"}", mode: 'link' + publishDir "${directoryMap."$finalannotator"}", mode: params.publishDirMode input: set annotator, variantCaller, file(vcf) from vcfToCompress @@ -268,14 +268,14 @@ if (params.verbose) vcfCompressedoutput = vcfCompressedoutput.view { } process GetVersionSnpeff { - publishDir directoryMap.version, mode: 'link' + publishDir directoryMap.version, mode: params.publishDirMode output: file("v_*.txt") when: 'snpeff' in tools || 'merge' in tools script: QC.getVersionSnpEFF() } process GetVersionVEP { - publishDir directoryMap.version, mode: 'link' + publishDir directoryMap.version, mode: params.publishDirMode output: file("v_*.txt") when: 'vep' in tools || 'merge' in tools script: QC.getVersionVEP() diff --git a/buildContainers.nf b/buildContainers.nf index c5706dc981..6aad5e5c52 100644 --- a/buildContainers.nf +++ b/buildContainers.nf @@ -86,7 +86,7 @@ if (params.verbose) containersBuilt = containersBuilt.view { process PullSingularityContainers { tag {"${params.repository}/${container}:${params.tag}"} - publishDir "${params.containerPath}", mode: 'move' + publishDir "${params.containerPath}", mode: params.publishDirMode input: val container from singularityContainers diff --git a/buildReferences.nf b/buildReferences.nf index 52a34af469..4ce4a400e1 100644 --- a/buildReferences.nf +++ b/buildReferences.nf @@ -103,7 +103,7 @@ ch_notCompressedfiles process BuildBWAindexes { tag {f_reference} - publishDir params.outDir, mode: 'link' + publishDir params.outDir, mode: params.publishDirMode input: file(f_reference) from ch_fastaForBWA @@ -125,7 +125,7 @@ if (params.verbose) bwaIndexes.flatten().view { process BuildReferenceIndex { tag {f_reference} - publishDir params.outDir, mode: 'link' + publishDir params.outDir, mode: params.publishDirMode input: file(f_reference) from ch_fastaReference @@ -149,7 +149,7 @@ if (params.verbose) ch_referenceIndex.view { process BuildSAMToolsIndex { tag {f_reference} - publishDir params.outDir, mode: 'link' + publishDir params.outDir, mode: params.publishDirMode input: file(f_reference) from ch_fastaForSAMTools @@ -170,7 +170,7 @@ if (params.verbose) ch_samtoolsIndex.view { process BuildVCFIndex { tag {f_reference} - publishDir params.outDir, mode: 'link' + publishDir params.outDir, mode: params.publishDirMode input: file(f_reference) from ch_vcfFile diff --git a/conf/base.config b/conf/base.config index 02e12faa23..e215a04432 100644 --- a/conf/base.config +++ b/conf/base.config @@ -27,6 +27,7 @@ params { nucleotidesPerSecond = 1000.0 // To estimate interval size by default onlyQC = false // All process will be run and not only the QC tools outDir = "${PWD}" // Path to output directory + publishDirMode = 'link' // publishDir mode is 'link' by default push = false // Don't push container to DockerHub repository = wf_repository // DockerHub containers repository singularity = false // Don't use singularity to build buildContainers.nf diff --git a/docs/PARAMETERS.md b/docs/PARAMETERS.md index 399fade15c..ff9a304c42 100644 --- a/docs/PARAMETERS.md +++ b/docs/PARAMETERS.md @@ -21,6 +21,21 @@ Choose an output directory Specify a project number ID on a UPPMAX cluster. (optional if not on such a cluster) +## --publishDirMode + +Specify wich mode `publishDir` directive need to follow, in Sarek link is the default mode. + +From [Nextflow documentation](https://www.nextflow.io/docs/latest/process.html#publishdir): + +| Mode | Description | +|--------------|-------------| +| copy | Copies the output files into the published directory | +| copyNoFollow | Copies the output files into the published directory without following symlinks ie. copies the links themselves | +| link | Creates a hard link in the published directory for each process output file (default) | +| move | Moves the output files into the published directory. Note: this is only supposed to be used for a terminating process i.e. a process whose output is not consumed by any other downstream process | +| rellink | Creates a relative symbolic link in the published directory for each process output file | +| symlink | Creates an absolute symbolic link in the published directory for each process output file | + ### --sample `file.tsv` Use the given TSV file as sample (cf [TSV documentation](TSV.md)). diff --git a/lib/SarekUtils.groovy b/lib/SarekUtils.groovy index 7b3f33df35..dd90e8f264 100644 --- a/lib/SarekUtils.groovy +++ b/lib/SarekUtils.groovy @@ -71,6 +71,8 @@ class SarekUtils { 'outDir', 'params', 'project', + 'publish-dir-mode', + 'publishDirMode', 'push', 'ref-dir', 'refDir', diff --git a/main.nf b/main.nf index 2ddad4e215..423565a090 100644 --- a/main.nf +++ b/main.nf @@ -123,7 +123,7 @@ if (params.verbose) bamFiles = bamFiles.view { process RunFastQC { tag {idPatient + "-" + idRun} - publishDir "${directoryMap.fastQC}/${idRun}", mode: 'link' + publishDir "${directoryMap.fastQC}/${idRun}", mode: params.publishDirMode input: set idPatient, status, idSample, idRun, file(fastqFile1), file(fastqFile2) from fastqFilesforFastQC @@ -226,7 +226,7 @@ if (params.verbose) mergedBam = mergedBam.view { process MarkDuplicates { tag {idPatient + "-" + idSample} - publishDir params.outDir, mode: 'link', + publishDir params.outDir, mode: params.publishDirMode, saveAs: { if (it == "${bam}.metrics") "${directoryMap.markDuplicatesQC}/${it}" else "${directoryMap.duplicateMarked}/${it}" @@ -283,7 +283,7 @@ if (params.verbose) duplicateMarkedBams = duplicateMarkedBams.view { process CreateRecalibrationTable { tag {idPatient + "-" + idSample} - publishDir directoryMap.duplicateMarked, mode: 'link', overwrite: false + publishDir directoryMap.duplicateMarked, mode: params.publishDirMode, overwrite: false input: set idPatient, status, idSample, file(bam), file(bai) from mdBam // realignedBam @@ -350,7 +350,7 @@ recalTables = recalTables.map { [it[0]] + it[2..-1] } // remove status process RecalibrateBam { tag {idPatient + "-" + idSample} - publishDir directoryMap.recalibrated, mode: 'link' + publishDir directoryMap.recalibrated, mode: params.publishDirMode input: set idPatient, status, idSample, file(bam), file(bai), file(recalibrationReport) from recalibrationTable @@ -398,7 +398,7 @@ if (params.verbose) recalibratedBam = recalibratedBam.view { process RunSamtoolsStats { tag {idPatient + "-" + idSample} - publishDir directoryMap.samtoolsStats, mode: 'link' + publishDir directoryMap.samtoolsStats, mode: params.publishDirMode input: set idPatient, status, idSample, file(bam), file(bai) from bamForSamToolsStats @@ -419,7 +419,7 @@ if (params.verbose) samtoolsStatsReport = samtoolsStatsReport.view { process RunBamQC { tag {idPatient + "-" + idSample} - publishDir directoryMap.bamQC, mode: 'link' + publishDir directoryMap.bamQC, mode: params.publishDirMode input: set idPatient, status, idSample, file(bam), file(bai) from bamForBamQC diff --git a/runMultiQC.nf b/runMultiQC.nf index 497e1f7505..9cfbac610d 100644 --- a/runMultiQC.nf +++ b/runMultiQC.nf @@ -47,7 +47,7 @@ directoryMap = SarekUtils.defineDirectoryMap(params.outDir) startMessage() process GetVersionAll { - publishDir directoryMap.multiQC, mode: 'link' + publishDir directoryMap.multiQC, mode: params.publishDirMode input: file(versions) from Channel.fromPath("${directoryMap.version}/*").collect().ifEmpty(file ("empty")) @@ -94,7 +94,7 @@ reportsForMultiQC = Channel.empty() ).collect() process RunMultiQC { - publishDir directoryMap.multiQC, mode: 'link' + publishDir directoryMap.multiQC, mode: params.publishDirMode input: file (multiqcConfig) from createMultiQCconfig() diff --git a/somaticVC.nf b/somaticVC.nf index ab77e879b4..89a814da2c 100644 --- a/somaticVC.nf +++ b/somaticVC.nf @@ -113,7 +113,7 @@ if (params.verbose) recalibratedBam = recalibratedBam.view { process RunSamtoolsStats { tag {idPatient + "-" + idSample} - publishDir directoryMap.samtoolsStats, mode: 'link' + publishDir directoryMap.samtoolsStats, mode: params.publishDirMode input: set idPatient, status, idSample, file(bam), file(bai) from bamForSamToolsStats @@ -134,7 +134,7 @@ if (params.verbose) samtoolsStatsReport = samtoolsStatsReport.view { process RunBamQC { tag {idPatient + "-" + idSample} - publishDir directoryMap.bamQC, mode: 'link' + publishDir directoryMap.bamQC, mode: params.publishDirMode input: set idPatient, status, idSample, file(bam), file(bai) from bamForBamQC @@ -357,7 +357,7 @@ if (params.verbose) vcfsToMerge = vcfsToMerge.view { process ConcatVCF { tag {variantCaller + "_" + idSampleTumor + "_vs_" + idSampleNormal} - publishDir "${directoryMap."$variantCaller"}", mode: 'link' + publishDir "${directoryMap."$variantCaller"}", mode: params.publishDirMode input: set variantCaller, idPatient, idSampleNormal, idSampleTumor, file(vcFiles) from vcfsToMerge @@ -392,7 +392,7 @@ if (params.verbose) vcfConcatenated = vcfConcatenated.view { process RunStrelka { tag {idSampleTumor + "_vs_" + idSampleNormal} - publishDir directoryMap.strelka, mode: 'link' + publishDir directoryMap.strelka, mode: params.publishDirMode input: set idPatient, idSampleNormal, file(bamNormal), file(baiNormal), idSampleTumor, file(bamTumor), file(baiTumor) from bamsForStrelka @@ -449,7 +449,7 @@ if (params.verbose) strelkaOutput = strelkaOutput.view { process RunManta { tag {idSampleTumor + "_vs_" + idSampleNormal} - publishDir directoryMap.manta, mode: 'link' + publishDir directoryMap.manta, mode: params.publishDirMode input: set idPatient, idSampleNormal, file(bamNormal), file(baiNormal), idSampleTumor, file(bamTumor), file(baiTumor) from bamsForManta @@ -503,7 +503,7 @@ if (params.verbose) mantaOutput = mantaOutput.view { process RunSingleManta { tag {idSample + " - Tumor-Only"} - publishDir directoryMap.manta, mode: 'link' + publishDir directoryMap.manta, mode: params.publishDirMode input: set idPatient, status, idSample, file(bam), file(bai) from bamsForSingleManta @@ -562,7 +562,7 @@ bamsForStrelkaBP = bamsForStrelkaBP.map { process RunStrelkaBP { tag {idSampleTumor + "_vs_" + idSampleNormal} - publishDir directoryMap.strelkabp, mode: 'link' + publishDir directoryMap.strelkabp, mode: params.publishDirMode input: set idPatient, idSampleNormal, file(bamNormal), file(baiNormal), idSampleTumor, file(bamTumor), file(baiTumor), file(mantaCSI), file(mantaCSIi) from bamsForStrelkaBP @@ -654,7 +654,7 @@ alleleCountOutput = alleleCountOutput.map { process RunConvertAlleleCounts { tag {idSampleTumor + "_vs_" + idSampleNormal} - publishDir directoryMap.ascat, mode: 'link' + publishDir directoryMap.ascat, mode: params.publishDirMode input: set idPatient, idSampleNormal, idSampleTumor, file(alleleCountNormal), file(alleleCountTumor) from alleleCountOutput @@ -676,7 +676,7 @@ process RunConvertAlleleCounts { process RunAscat { tag {idSampleTumor + "_vs_" + idSampleNormal} - publishDir directoryMap.ascat, mode: 'link' + publishDir directoryMap.ascat, mode: params.publishDirMode input: set idPatient, idSampleNormal, idSampleTumor, file(bafNormal), file(logrNormal), file(bafTumor), file(logrTumor) from convertAlleleCountsOutput @@ -734,7 +734,7 @@ vcfForQC = Channel.empty().mix( process RunBcftoolsStats { tag {vcf} - publishDir directoryMap.bcftoolsStats, mode: 'link' + publishDir directoryMap.bcftoolsStats, mode: params.publishDirMode input: set variantCaller, file(vcf) from vcfForBCFtools @@ -757,7 +757,7 @@ bcfReport.close() process RunVcftools { tag {vcf} - publishDir directoryMap.vcftools, mode: 'link' + publishDir directoryMap.vcftools, mode: params.publishDirMode input: set variantCaller, file(vcf) from vcfForVCFtools @@ -778,7 +778,7 @@ if (params.verbose) vcfReport = vcfReport.view { vcfReport.close() process GetVersionAlleleCount { - publishDir directoryMap.version, mode: 'link' + publishDir directoryMap.version, mode: params.publishDirMode output: file("v_*.txt") when: 'ascat' in tools && !params.onlyQC @@ -789,7 +789,7 @@ process GetVersionAlleleCount { } process GetVersionASCAT { - publishDir directoryMap.version, mode: 'link' + publishDir directoryMap.version, mode: params.publishDirMode output: file("v_*.txt") when: 'ascat' in tools && !params.onlyQC From 49fdda889e9df36bb5dd1f155d07aa52f805860f Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Tue, 23 Oct 2018 14:45:26 +0200 Subject: [PATCH 2/2] update CHANGELOG [skip ci] --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51aa88fd9d..a3ab31d757 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### `Added` + +- [#671](https://github.com/SciLifeLab/Sarek/pull/671) - New `publishDirMode` param and docs + ### `Changed` - [#663](https://github.com/SciLifeLab/Sarek/pull/663) - Update `do_release.sh` script +- [#671](https://github.com/SciLifeLab/Sarek/pull/671) - publishDir modes are now params ### `Fixed`