1.2.3

.gitignore

@@ -1,12 +1,13 @@
-data/chr17_testdata
-Annotation
+Annotation/
+data/chr17_testdata/
 Preprocessing/
-Reports/
 References/
+Reports/
 VariantCalling/
 work/
 .*swp
 .nextflow*
+*.img
+*.tar.gz
 timeline.html*
 trace.txt*
-*.tar.gz

.travis.yml

@@ -11,11 +11,11 @@ env:
   - NXF_VER=0.25.6 SGT_VER=2.3.1 PROFILE=singularity TEST=ANNOTATESNPEFF TOOL_INSTALL=all
   - NXF_VER=0.25.6 PROFILE=docker TEST=ANNOTATESNPEFF TOOL_INSTALL=nextflow
   - NXF_VER=0.25.6 PROFILE=docker TEST=ANNOTATEVEP TOOL_INSTALL=nextflow
-  - NXF_VER=0.25.6 PROFILE=docker TEST=BUILDCONTAINERS TOOL_INSTALL=nextflow
   - NXF_VER=0.25.6 SGT_VER=2.3.1 PROFILE=singularity TEST=RECALIBRATE TOOL_INSTALL=all
   - NXF_VER=0.25.6 PROFILE=docker TEST=RECALIBRATE TOOL_INSTALL=nextflow
   - NXF_VER=0.25.6 SGT_VER=2.3.1 PROFILE=singularity TEST=REALIGN TOOL_INSTALL=all
   - NXF_VER=0.25.6 PROFILE=docker TEST=REALIGN TOOL_INSTALL=nextflow
+  - NXF_VER=0.25.6 PROFILE=docker TEST=BUILDCONTAINERS TOOL_INSTALL=nextflow
   - NXF_VER=0.25.6 SGT_VER=2.3.1 PROFILE=singularity TEST=MAPPING TOOL_INSTALL=all
   - NXF_VER=0.25.6 PROFILE=docker TEST=MAPPING TOOL_INSTALL=nextflow
 

README.md

@@ -4,7 +4,6 @@
 
 [![caw version][version-badge]][version-link] [![License][license-badge]][license-link] [![nextflow version][nextflow-badge]][nextflow-link] [![Join the chat at https://gitter.im/SciLifeLab/CAW][gitter-badge]][gitter-link] [![Travis status][travis-badge]][travis-link]
 
-
 CAW is a complete open source pipeline to detect somatic variants from WGS data developed at the [National Genomics Infastructure][ngi-link] at [SciLifeLab Stockholm][scilifelab-stockholm-link], Sweden and [National Bioinformatics Infastructure Sweden][nbis-link] at [SciLifeLab][scilifelab-link].
 
 The pipeline uses [Nextflow][nextflow-link], a bioinformatics domain specific language for workflow building and [Singularity](http://singularity.lbl.gov/), a container technology specific for high-performance computing.

buildContainers.nf

@@ -36,7 +36,7 @@ kate: syntax groovy; space-indent on; indent-width 2;
 ================================================================================
 */
 
-version = '1.2.2'
+version = '1.2.3'
 
 // Check that Nextflow version is up to date enough
 // try / throw / catch works for NF versions < 0.25 when this was implemented
@@ -58,6 +58,22 @@ if (params.version) exit 0, versionMessage()
 if (!isAllowedParams(params)) exit 1, "params is unknown, see --help for more information"
 if (!checkUppmaxProject()) exit 1, "No UPPMAX project ID found! Use --project <UPPMAX Project ID>"
 
+// Default params:
+// Such params are overridden by command line or configuration definitions
+
+// containerPath is empty
+params.containerPath = ''
+// all containers to be build
+params.containers = 'all'
+// Docker will not be used
+params.docker = false
+// Containers will not be pushed on DockerHub
+params.push = false
+// DockerHub repository is maxulysse
+params.repository = 'maxulysse'
+// Singularity will not be used
+params.singularity = false
+
 verbose = params.verbose
 containersList = defineContainersList()
 containers = params.containers.split(',').collect {it.trim()}
@@ -67,7 +83,9 @@ push = params.docker && params.push ? true : false
 repository = params.repository
 tag = params.tag ? params.tag : version
 singularity = params.singularity ? true : false
-singularityPublishDir = params.singularity && params.singularityPublishDir ? params.singularityPublishDir : "."
+containerPath = params.singularity && params.containerPath ? params.containerPath : "."
+
+if (!docker && !singularity) exit 1, 'No builder choose, specify --docker or --singularity, see --help for more information'
 
 if (!checkContainers(containers,containersList)) exit 1, 'Unknown container(s), see --help for more information'
 
@@ -106,7 +124,7 @@ if (verbose) dockerContainersBuilt = dockerContainersBuilt.view {
 process PullSingularityContainers {
   tag {repository + "/" + container + ":" + tag}
 
-  publishDir singularityPublishDir, mode: 'move'
+  publishDir containerPath, mode: 'move'
 
   input:
     val container from singularityContainers
@@ -188,6 +206,8 @@ def checkParams(it) {
     'callName',
     'contact-mail',
     'contactMail',
+    'container-path',
+    'containerPath',
     'containers',
     'docker',
     'genome',
@@ -205,9 +225,7 @@ def checkParams(it) {
     'sampleDir',
     'single-CPUMem',
     'singleCPUMem',
-    'singularity-publish-dir',
     'singularity',
-    'singularityPublishDir',
     'step',
     'tag',
     'test',
@@ -241,7 +259,6 @@ def defineContainersList(){
     'snpeff',
     'snpeffgrch37',
     'snpeffgrch38',
-    'vep',
     'vepgrch37',
     'vepgrch38'
     ]
@@ -258,7 +275,7 @@ def helpMessage() {
   log.info "    Usage:"
   log.info "       nextflow run SciLifeLab/buildContainers.nf [--docker] [--push]"
   log.info "          [--containers <container1...>] [--singularity]"
-  log.info "          [--singularityPublishDir <path>]"
+  log.info "          [--containerPath <path>]"
   log.info "          [--tag <tag>] [--repository <repository>]"
   log.info "    Example:"
   log.info "      nextflow run . --docker --containers multiqc,fastqc"
@@ -275,7 +292,7 @@ def helpMessage() {
   log.info "    --repository: Build containers under given repository"
   log.info "       Default: maxulysse"
   log.info "    --singularity: Build containers using Singularity"
-  log.info "    --singularityPublishDir: Select where to download containers"
+  log.info "    --containerPath: Select where to download containers"
   log.info "       Default: $PWD"
   log.info "    --tag`: Build containers using given tag"
   log.info "       Default (version number): " + version

buildReferences.nf

@@ -39,7 +39,7 @@ kate: syntax groovy; space-indent on; indent-width 2;
 ================================================================================
 */
 
-version = '1.2.2'
+version = '1.2.3'
 
 // Check that Nextflow version is up to date enough
 // try / throw / catch works for NF versions < 0.25 when this was implemented
@@ -61,8 +61,14 @@ if (params.version) exit 0, versionMessage()
 if (!isAllowedParams(params)) exit 1, "params is unknown, see --help for more information"
 if (!checkUppmaxProject()) exit 1, "No UPPMAX project ID found! Use --project <UPPMAX Project ID>"
 
+// Default params:
+// Such params are overridden by command line or configuration definitions
+
+// No download
 params.download = false
-params.refDir = ""
+// refDir is empty
+params.refDir = ''
+
 verbose = params.verbose
 download = params.download ? true : false
 
@@ -309,6 +315,8 @@ def checkParams(it) {
     'callName',
     'contact-mail',
     'contactMail',
+    'container-path',
+    'containerPath',
     'containers',
     'docker',
     'download',
@@ -331,9 +339,7 @@ def checkParams(it) {
     'sampleDir',
     'single-CPUMem',
     'singleCPUMem',
-    'singularity-publish-dir',
     'singularity',
-    'singularityPublishDir',
     'step',
     'tag',
     'test',

configuration/containers.config

@@ -0,0 +1,44 @@
+/*
+vim: syntax=groovy
+-*- mode: groovy;-*-
+ * -------------------------------------------------
+ * Nextflow config file for CAW project
+ * -------------------------------------------------
+ * Images for every process
+ * All images will be pulled automatically
+ * -------------------------------------------------
+ */
+
+process {
+  $BuildBWAindexes.container          = "${params.repository}/caw:${params.tag}"
+  $BuildPicardIndex.container         = "${params.repository}/picard:${params.tag}"
+  $BuildSAMToolsIndex.container       = "${params.repository}/caw:${params.tag}"
+  $BuildVCFIndex.container            = "${params.repository}/igvtools:${params.tag}"
+  $ConcatVCF.container                = "${params.repository}/caw:${params.tag}"
+  $CreateRecalibrationTable.container = "${params.repository}/gatk:${params.tag}"
+  $IndelRealigner.container           = "${params.repository}/gatk:${params.tag}"
+  $MapReads.container                 = "${params.repository}/caw:${params.tag}"
+  $MarkDuplicates.container           = "${params.repository}/picard:${params.tag}"
+  $MergeBams.container                = "${params.repository}/caw:${params.tag}"
+  $RealignerTargetCreator.container   = "${params.repository}/gatk:${params.tag}"
+  $RecalibrateBam.container           = "${params.repository}/gatk:${params.tag}"
+  $RunAlleleCount.container           = "${params.repository}/runallelecount:${params.tag}"
+  $RunAscat.container                 = "${params.repository}/r-base:${params.tag}"
+  $RunBamQC.container                 = "${params.repository}/qualimap:${params.tag}"
+  $RunBcftoolsStats.container         = "${params.repository}/caw:${params.tag}"
+  $RunConvertAlleleCounts.container   = "${params.repository}/r-base:${params.tag}"
+  $RunFastQC.container                = "${params.repository}/fastqc:${params.tag}"
+  $RunFreeBayes.container             = "${params.repository}/freebayes:${params.tag}"
+  $RunGenotypeGVCFs.container         = "${params.repository}/gatk:${params.tag}"
+  $RunHaplotypecaller.container       = "${params.repository}/gatk:${params.tag}"
+  $RunManta.container                 = "${params.repository}/caw:${params.tag}"
+  $RunMultiQC.container               = "${params.repository}/multiqc:${params.tag}"
+  $RunMutect1.container               = "${params.repository}/mutect1:${params.tag}"
+  $RunMutect2.container               = "${params.repository}/gatk:${params.tag}"
+  $RunSamtoolsStats.container         = "${params.repository}/caw:${params.tag}"
+  $RunSingleManta.container           = "${params.repository}/caw:${params.tag}"
+  $RunSingleStrelka.container         = "${params.repository}/caw:${params.tag}"
+  $RunSnpeff.container = {params.genome == "GRCh38" ? "${params.repository}/snpeffgrch38:${params.tag}" : "${params.repository}/snpeffgrch37:${params.tag}"}
+  $RunStrelka.container               = "${params.repository}/caw:${params.tag}"
+  $RunVEP.container = {params.genome == "GRCh38" ? "${params.repository}/vepgrch38:${params.tag}" : "${params.repository}/vepgrch37:${params.tag}"}
+}

configuration/docker.config

@@ -14,36 +14,5 @@ docker {
   fixOwnership = true
 }
 
-process {
-  $BuildBWAindexes.container          = 'maxulysse/caw:1.2.1'
-  $BuildPicardIndex.container         = 'maxulysse/picard:1.2.1'
-  $BuildSAMToolsIndex.container       = 'maxulysse/caw:1.2.1'
-  $BuildVCFIndex.container            = 'maxulysse/igvtools:1.2.1'
-  $ConcatVCF.container                = 'maxulysse/caw:1.2.1'
-  $CreateRecalibrationTable.container = 'maxulysse/gatk:1.2.1'
-  $IndelRealigner.container           = 'maxulysse/gatk:1.2.1'
-  $MapReads.container                 = 'maxulysse/caw:1.2.1'
-  $MarkDuplicates.container           = 'maxulysse/picard:1.2.1'
-  $MergeBams.container                = 'maxulysse/caw:1.2.1'
-  $RealignerTargetCreator.container   = 'maxulysse/gatk:1.2.1'
-  $RecalibrateBam.container           = 'maxulysse/gatk:1.2.1'
-  $RunAlleleCount.container           = 'maxulysse/runallelecount:1.2.1'
-  $RunAscat.container                 = 'maxulysse/r-base:1.2.1'
-  $RunBamQC.container                 = 'maxulysse/qualimap:1.2.1'
-  $RunBcftoolsStats.container         = 'maxulysse/caw:1.2.1'
-  $RunConvertAlleleCounts.container   = 'maxulysse/r-base:1.2.1'
-  $RunFastQC.container                = 'maxulysse/fastqc:1.2.1'
-  $RunFreeBayes.container             = 'maxulysse/freebayes:1.2.1'
-  $RunGenotypeGVCFs.container         = 'maxulysse/gatk:1.2.1'
-  $RunHaplotypecaller.container       = 'maxulysse/gatk:1.2.1'
-  $RunManta.container                 = 'maxulysse/caw:1.2.1'
-  $RunMultiQC.container               = 'maxulysse/multiqc:1.2.1'
-  $RunMutect1.container               = 'maxulysse/mutect1:1.2.1'
-  $RunMutect2.container               = 'maxulysse/gatk:1.2.1'
-  $RunSamtoolsStats.container         = 'maxulysse/caw:1.2.1'
-  $RunSingleManta.container           = 'maxulysse/caw:1.2.1'
-  $RunSingleStrelka.container         = 'maxulysse/caw:1.2.1'
-  $RunSnpeff.container = {params.genome == "GRCh38" ? 'maxulysse/snpeffgrch38:1.2.1' : 'maxulysse/snpeffgrch37:1.2.1'}
-  $RunStrelka.container               = 'maxulysse/caw:1.2.1'
-  $RunVEP.container = {params.genome == "GRCh38" ? 'maxulysse/vepgrch38:1.2.1' : 'maxulysse/vepgrch37:1.2.1'}
-}
+params.repository='maxulysse'
+params.tag='1.2.3'

configuration/singularity-path.config

@@ -12,39 +12,43 @@ vim: syntax=groovy
 
 singularity {
   enabled = true
+  runOptions = "--bind /scratch"
 }
 
+params.containerPath='containers'
+params.tag='1.2.3'
+
 process {
-  $BuildBWAindexes.container          = 'containers/caw-1.2.1.img'
-  $BuildPicardIndex.container         = 'containers/picard-1.2.1.img'
-  $BuildSAMToolsIndex.container       = 'containers/caw-1.2.1.img'
-  $BuildVCFIndex.container            = 'containers/igvtools-1.2.1.img'
-  $ConcatVCF.container                = 'containers/caw-1.2.1.img'
-  $CreateRecalibrationTable.container = 'containers/gatk-1.2.1.img'
-  $GenerateMultiQCconfig.container    = 'containers/multiqc-1.2.1.img'
-  $IndelRealigner.container           = 'containers/gatk-1.2.1.img'
-  $MapReads.container                 = 'containers/caw-1.2.1.img'
-  $MarkDuplicates.container           = 'containers/picard-1.2.1.img'
-  $MergeBams.container                = 'containers/caw-1.2.1.img'
-  $RealignerTargetCreator.container   = 'containers/gatk-1.2.1.img'
-  $RecalibrateBam.container           = 'containers/gatk-1.2.1.img'
-  $RunAlleleCount.container           = 'containers/runallelecount-1.2.1.img'
-  $RunAscat.container                 = 'containers/r-base-1.2.1.img'
-  $RunBamQC.container                 = 'containers/qualimap-1.2.1.img'
-  $RunBcftoolsStats.container         = 'containers/caw-1.2.1.img'
-  $RunConvertAlleleCounts.container   = 'containers/r-base-1.2.1.img'
-  $RunFastQC.container                = 'containers/fastqc-1.2.1.img'
-  $RunFreeBayes.container             = 'containers/freebayes-1.2.1.img'
-  $RunGenotypeGVCFs.container         = 'containers/gatk-1.2.1.img'
-  $RunHaplotypecaller.container       = 'containers/gatk-1.2.1.img'
-  $RunManta.container                 = 'containers/caw-1.2.1.img'
-  $RunMultiQC.container               = 'containers/multiqc-1.2.1.img'
-  $RunMutect1.container               = 'containers/mutect1-1.2.1.img'
-  $RunMutect2.container               = 'containers/gatk-1.2.1.img'
-  $RunSamtoolsStats.container         = 'containers/caw-1.2.1.img'
-  $RunSingleManta.container           = 'containers/caw-1.2.1.img'
-  $RunSingleStrelka.container         = 'containers/caw-1.2.1.img'
-  $RunSnpeff.container                = {params.genome == "GRCh38" ? 'containers/snpeffgrch38-1.2.1.img' : 'containers/snpeffgrch37-1.2.1.img'}
-  $RunStrelka.container               = 'containers/caw-1.2.1.img'
-  $RunVEP.container                   = {params.genome == "GRCh38" ? 'containers/vepgrch38-1.2.1.img' : 'containers/vepgrch37-1.2.1.img'}
+  $BuildBWAindexes.container          = "${params.containerPath}/caw-${params.tag}.img"
+  $BuildPicardIndex.container         = "${params.containerPath}/picard-${params.tag}.img"
+  $BuildSAMToolsIndex.container       = "${params.containerPath}/caw-${params.tag}.img"
+  $BuildVCFIndex.container            = "${params.containerPath}/igvtools-${params.tag}.img"
+  $ConcatVCF.container                = "${params.containerPath}/caw-${params.tag}.img"
+  $CreateRecalibrationTable.container = "${params.containerPath}/gatk-${params.tag}.img"
+  $GenerateMultiQCconfig.container    = "${params.containerPath}/multiqc-${params.tag}.img"
+  $IndelRealigner.container           = "${params.containerPath}/gatk-${params.tag}.img"
+  $MapReads.container                 = "${params.containerPath}/caw-${params.tag}.img"
+  $MarkDuplicates.container           = "${params.containerPath}/picard-${params.tag}.img"
+  $MergeBams.container                = "${params.containerPath}/caw-${params.tag}.img"
+  $RealignerTargetCreator.container   = "${params.containerPath}/gatk-${params.tag}.img"
+  $RecalibrateBam.container           = "${params.containerPath}/gatk-${params.tag}.img"
+  $RunAlleleCount.container           = "${params.containerPath}/runallelecount-${params.tag}.img"
+  $RunAscat.container                 = "${params.containerPath}/r-params.base-${params.tag}.img"
+  $RunBamQC.container                 = "${params.containerPath}/qualimap-${params.tag}.img"
+  $RunBcftoolsStats.container         = "${params.containerPath}/caw-${params.tag}.img"
+  $RunConvertAlleleCounts.container   = "${params.containerPath}/r-params.base-${params.tag}.img"
+  $RunFastQC.container                = "${params.containerPath}/fastqc-${params.tag}.img"
+  $RunFreeBayes.container             = "${params.containerPath}/freebayes-${params.tag}.img"
+  $RunGenotypeGVCFs.container         = "${params.containerPath}/gatk-${params.tag}.img"
+  $RunHaplotypecaller.container       = "${params.containerPath}/gatk-${params.tag}.img"
+  $RunManta.container                 = "${params.containerPath}/caw-${params.tag}.img"
+  $RunMultiQC.container               = "${params.containerPath}/multiqc-${params.tag}.img"
+  $RunMutect1.container               = "${params.containerPath}/mutect1-${params.tag}.img"
+  $RunMutect2.container               = "${params.containerPath}/gatk-${params.tag}.img"
+  $RunSamtoolsStats.container         = "${params.containerPath}/caw-${params.tag}.img"
+  $RunSingleManta.container           = "${params.containerPath}/caw-${params.tag}.img"
+  $RunSingleStrelka.container         = "${params.containerPath}/caw-${params.tag}.img"
+  $RunSnpeff.container                = {params.genome == "GRCh38" ? "${params.containerPath}/snpeffgrch38-${params.tag}.img" : "${params.containerPath}/snpeffgrch37-${params.tag}.img"}
+  $RunStrelka.container               = "${params.containerPath}/caw-${params.tag}.img"
+  $RunVEP.container                   = {params.genome == "GRCh38" ? "${params.containerPath}/vepgrch38-${params.tag}.img" : "${params.containerPath}/vepgrch37-${params.tag}.img"}
 }