From d755288c6797c1739dd3134dd1a9bf4c4174d1bd Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Sun, 4 Nov 2018 23:10:11 +0100 Subject: [PATCH 01/14] Changelog update --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc60f786c..c0b7dd508 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## unpublished +* [#77](https://github.com/nf-core/eager/pull/77) - Lots of documentation updates by [@jfy133](https://github.com/jfy133) ## [2.0.2] - 2018-11-03 From 7663b0b23d6a39f20a40938efc7942a3629c62db Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Sun, 4 Nov 2018 23:15:48 +0100 Subject: [PATCH 02/14] Started work for index file handling --- main.nf | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/main.nf b/main.nf index 59643ed3d..b49c6c750 100644 --- a/main.nf +++ b/main.nf @@ -211,9 +211,23 @@ wherearemyfiles = file("$baseDir/assets/where_are_my_files.txt") // Validate inputs Channel.fromPath("${params.fasta}") - .ifEmpty { exit 1, "No genome specified! Please specify one with --fasta or --bwa_index"} + .ifEmpty { exit 1, "No genome specified! Please specify one with --fasta"} .into {ch_fasta_for_bwa_indexing;ch_fasta_for_faidx_indexing;ch_fasta_for_dict_indexing; ch_fasta_for_bwa_mapping; ch_fasta_for_damageprofiler; ch_fasta_for_qualimap; ch_fasta_for_pmdtools; ch_fasta_for_circularmapper; ch_fasta_for_circularmapper_index;ch_fasta_for_bwamem_mapping} +//Index files provided? Then check whether they are correct and complete +if (params.aligner != 'bwa' && !params.circularmapper && !params.bwamem){ + exit 1, "Invalid aligner option. Default is bwa, but specify --circularmapper or --bwamem to use these." +} +if( params.bwa_index && (params.aligner == 'bwa' | params.bwamem)){ + bwa_index = Channel + .fromPath(params.bwa_index) + .ifEmpty { exit 1, "BWA index not found: ${params.bwa_index}" } +} +else { + exit 1, "No reference genome specified!" +} + + //Validate that either pairedEnd or singleEnd has been specified by the user! if( params.singleEnd || params.pairedEnd ){ } else { From 7b09fe48d49e87583b76be36f645202005ab0236 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Sun, 4 Nov 2018 23:16:29 +0100 Subject: [PATCH 03/14] Changelog ++ --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0b7dd508..3ed14b60d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## unpublished +* Index file handling for bwa_index, fasta_index, seq_dict TODO * [#77](https://github.com/nf-core/eager/pull/77) - Lots of documentation updates by [@jfy133](https://github.com/jfy133) ## [2.0.2] - 2018-11-03 From e3510afc930dd5f813c25d2bea56de6ae19a798d Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Sun, 4 Nov 2018 23:18:00 +0100 Subject: [PATCH 04/14] Catching fasta issues differently --- main.nf | 3 --- 1 file changed, 3 deletions(-) diff --git a/main.nf b/main.nf index b49c6c750..d54c0a405 100644 --- a/main.nf +++ b/main.nf @@ -223,9 +223,6 @@ if( params.bwa_index && (params.aligner == 'bwa' | params.bwamem)){ .fromPath(params.bwa_index) .ifEmpty { exit 1, "BWA index not found: ${params.bwa_index}" } } -else { - exit 1, "No reference genome specified!" -} //Validate that either pairedEnd or singleEnd has been specified by the user! From fc3407e7eac59efb725a61cd04b2a9849f3387ce Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 5 Nov 2018 13:16:40 +0100 Subject: [PATCH 05/14] Explicit naming of tools in conda --- environment.yml | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/environment.yml b/environment.yml index 6ea84afc3..014fa9364 100644 --- a/environment.yml +++ b/environment.yml @@ -4,26 +4,26 @@ channels: - bioconda - conda-forge dependencies: - - openjdk=8.0.152 - - fastqc=0.11.8 - - adapterremoval=2.2.2 - - adapterremovalfixprefix=0.0.4 - - bwa=0.7.17 - - picard=2.18.15 - - samtools=1.9 - - dedup=0.12.3 - - angsd=0.923 - - circularmapper=1.93.4 - - gatk4=4.0.11.0 - - qualimap=2.2.2b - - vcf2genome=0.91 - - damageprofiler=0.3.11 - - multiqc=1.6 - - pmdtools=0.60 - - r-rmarkdown=1.10 - - libiconv=1.15 - - sequencetools=1.2.2 - - preseq=2.0.3 - - fastp=0.19.4 - - bamutil=1.0.14 + - anaconda::openjdk=8.0.152 + - bioconda::fastqc=0.11.8 + - bioconda::adapterremoval=2.2.2 + - bioconda::adapterremovalfixprefix=0.0.4 + - bioconda::bwa=0.7.17 + - bioconda::picard=2.18.15 + - bioconda::samtools=1.9 + - bioconda::dedup=0.12.3 + - bioconda::angsd=0.923 + - bioconda::circularmapper=1.93.4 + - bioconda::gatk4=4.0.11.0 + - bioconda::qualimap=2.2.2b + - bioconda::vcf2genome=0.91 + - bioconda::damageprofiler=0.3.11 + - bioconda::multiqc=1.6 + - bioconda::pmdtools=0.60 + - conda-forge::r-rmarkdown=1.10 + - conda-forge::libiconv=1.15 + - bioconda::sequencetools=1.2.2 + - bioconda::preseq=2.0.3 + - bioconda::fastp=0.19.4 + - bioconda::bamutil=1.0.14 #Missing Schmutzi,snpAD From ba3f3f2bb4960c95076ab22c01350b1062e51c21 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 5 Nov 2018 13:31:59 +0100 Subject: [PATCH 06/14] Summary addition --- main.nf | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/main.nf b/main.nf index d54c0a405..62d9c2290 100644 --- a/main.nf +++ b/main.nf @@ -220,8 +220,10 @@ if (params.aligner != 'bwa' && !params.circularmapper && !params.bwamem){ } if( params.bwa_index && (params.aligner == 'bwa' | params.bwamem)){ bwa_index = Channel - .fromPath(params.bwa_index) + .fromPath("${params.bwa_index}/**.*") + .flatten() .ifEmpty { exit 1, "BWA index not found: ${params.bwa_index}" } + .into{ch_bwa_index_existing;ch_bwa_index_bwamem_existing} } @@ -291,6 +293,7 @@ summary['Pipeline Version'] = workflow.manifest.version summary['Run Name'] = custom_runName ?: workflow.runName summary['Reads'] = params.reads summary['Fasta Ref'] = params.fasta +if(params.bwa_index) summary['BWA Index'] = params.bwa_index summary['Data Type'] = params.singleEnd ? 'Single-End' : 'Paired-End' summary['Max Memory'] = params.max_memory summary['Max CPUs'] = params.max_cpus @@ -375,7 +378,7 @@ process makeBWAIndex { else null } - when: !params.bwa_index && params.fasta && params.aligner == 'bwa' + when: !params.bwa_index && params.fasta && (params.aligner == 'bwa' || params.bwamem) input: file fasta from ch_fasta_for_bwa_indexing @@ -572,7 +575,7 @@ process bwa { input: file(reads) from ch_clipped_reads - file "*" from ch_bwa_index + file "*" from ch_bwa_index.mix(ch_bwa_index_existing).first() file fasta from ch_fasta_for_bwa_mapping output: @@ -649,7 +652,7 @@ process bwamem { input: file(reads) from ch_clipped_reads_bwamem - file "*" from ch_bwa_index_bwamem + file "*" from ch_bwa_index_bwamem.mix(ch_bwa_index_bwamem_existing).first() file fasta from ch_fasta_for_bwamem_mapping output: From aa44e2e5755228dd7d50cb4068ad9e13339fcc9c Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 5 Nov 2018 13:40:59 +0100 Subject: [PATCH 07/14] BWA indices are handled appropriately now --- main.nf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/main.nf b/main.nf index 62d9c2290..f8199ff5f 100644 --- a/main.nf +++ b/main.nf @@ -221,7 +221,6 @@ if (params.aligner != 'bwa' && !params.circularmapper && !params.bwamem){ if( params.bwa_index && (params.aligner == 'bwa' | params.bwamem)){ bwa_index = Channel .fromPath("${params.bwa_index}/**.*") - .flatten() .ifEmpty { exit 1, "BWA index not found: ${params.bwa_index}" } .into{ch_bwa_index_existing;ch_bwa_index_bwamem_existing} } @@ -575,7 +574,7 @@ process bwa { input: file(reads) from ch_clipped_reads - file "*" from ch_bwa_index.mix(ch_bwa_index_existing).first() + file "*" from ch_bwa_index.mix(ch_bwa_index_existing).collect() file fasta from ch_fasta_for_bwa_mapping output: From dd19040bab75494ab09739587eba418e69da18ab Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 5 Nov 2018 13:54:29 +0100 Subject: [PATCH 08/14] Minor fix for collect() --- main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.nf b/main.nf index f8199ff5f..f4987bee2 100644 --- a/main.nf +++ b/main.nf @@ -651,7 +651,7 @@ process bwamem { input: file(reads) from ch_clipped_reads_bwamem - file "*" from ch_bwa_index_bwamem.mix(ch_bwa_index_bwamem_existing).first() + file "*" from ch_bwa_index_bwamem.mix(ch_bwa_index_bwamem_existing).collect() file fasta from ch_fasta_for_bwamem_mapping output: From c86a50a6e8ff4de4dee5c6604240a4844329a69c Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 5 Nov 2018 14:08:46 +0100 Subject: [PATCH 09/14] Indexing should be fine --- main.nf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/main.nf b/main.nf index f4987bee2..cbe92b1ac 100644 --- a/main.nf +++ b/main.nf @@ -223,8 +223,7 @@ if( params.bwa_index && (params.aligner == 'bwa' | params.bwamem)){ .fromPath("${params.bwa_index}/**.*") .ifEmpty { exit 1, "BWA index not found: ${params.bwa_index}" } .into{ch_bwa_index_existing;ch_bwa_index_bwamem_existing} -} - +} //Validate that either pairedEnd or singleEnd has been specified by the user! if( params.singleEnd || params.pairedEnd ){ @@ -384,7 +383,7 @@ process makeBWAIndex { file wherearemyfiles output: - file "*.{amb,ann,bwt,pac,sa,fasta,fa}" into (ch_bwa_index,ch_bwa_index_bwamem) + file "*.{amb,ann,bwt,pac,sa,fasta,fa}" into (ch_bwa_index,ch_bwa_index_bwamem_existing) file "where_are_my_files.txt" script: From ea4d42ce5dea1598a4d45acac62871b85045859b Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 5 Nov 2018 14:33:50 +0100 Subject: [PATCH 10/14] Dependent channels for bwa index --- main.nf | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/main.nf b/main.nf index cbe92b1ac..b5c8c71e0 100644 --- a/main.nf +++ b/main.nf @@ -223,7 +223,12 @@ if( params.bwa_index && (params.aligner == 'bwa' | params.bwamem)){ .fromPath("${params.bwa_index}/**.*") .ifEmpty { exit 1, "BWA index not found: ${params.bwa_index}" } .into{ch_bwa_index_existing;ch_bwa_index_bwamem_existing} -} + ch_bwa_index = Channel.create() + ch_bwa_index_bwamem = Channel.create() +} else { + ch_bwa_index_existing = Channel.create() + ch_bwa_index_bwamem_existing = Channel.create() +} //Validate that either pairedEnd or singleEnd has been specified by the user! if( params.singleEnd || params.pairedEnd ){ @@ -383,7 +388,7 @@ process makeBWAIndex { file wherearemyfiles output: - file "*.{amb,ann,bwt,pac,sa,fasta,fa}" into (ch_bwa_index,ch_bwa_index_bwamem_existing) + file "*.{amb,ann,bwt,pac,sa,fasta,fa}" into (ch_bwa_index,ch_bwa_index_bwamem) file "where_are_my_files.txt" script: From 0725bf984de0700cf0da20096b3d02f8e9170df4 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 5 Nov 2018 14:45:21 +0100 Subject: [PATCH 11/14] Thanks @micans --- main.nf | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/main.nf b/main.nf index b5c8c71e0..82e085c91 100644 --- a/main.nf +++ b/main.nf @@ -223,11 +223,10 @@ if( params.bwa_index && (params.aligner == 'bwa' | params.bwamem)){ .fromPath("${params.bwa_index}/**.*") .ifEmpty { exit 1, "BWA index not found: ${params.bwa_index}" } .into{ch_bwa_index_existing;ch_bwa_index_bwamem_existing} - ch_bwa_index = Channel.create() - ch_bwa_index_bwamem = Channel.create() } else { - ch_bwa_index_existing = Channel.create() - ch_bwa_index_bwamem_existing = Channel.create() + //Create empty channels to make sure later mix() does not fail + ch_bwa_index_existing = Channel.empty() + ch_bwa_index_bwamem_existing = Channel.empty() } //Validate that either pairedEnd or singleEnd has been specified by the user! @@ -885,10 +884,10 @@ process markDup{ } //If no deduplication runs, the input is mixed directly from samtools filter, if it runs either markdup or dedup is used thus mixed from these two channels -ch_dedup_for_pmdtools = Channel.create() +ch_dedup_for_pmdtools = Channel.empty() //Bamutils TrimBam Channel -ch_for_bamutils = Channel.create() +ch_for_bamutils = Channel.empty() if(!params.skip_deduplication){ ch_dedup_for_pmdtools.mix(ch_markdup_bam,ch_dedup_bam).into {ch_for_pmdtools;ch_for_bamutils} From f12c7199272c366bf1119fd5cfa85adbcb2ebf72 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 5 Nov 2018 14:48:15 +0100 Subject: [PATCH 12/14] Use the initially create reference genome properly --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3b29c207b..6003abcb4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,16 +37,16 @@ script: # Lint the pipeline code - nf-core lint ${TRAVIS_BUILD_DIR} # Run the basic pipeline with the test profile - - nextflow run ${TRAVIS_BUILD_DIR} -profile test,docker --pairedEnd + - nextflow run ${TRAVIS_BUILD_DIR} -profile test,docker --pairedEnd --saveReference # Run the basic pipeline with single end data (pretending its single end actually) - - nextflow run ${TRAVIS_BUILD_DIR} -profile test,docker --singleEnd + - nextflow run ${TRAVIS_BUILD_DIR} -profile test,docker --singleEnd --bwa_index results/reference_genome/bwa_index/ # Run the same pipeline testing optional step: fastp, complexity - - nextflow run ${TRAVIS_BUILD_DIR} -profile test,docker --pairedEnd --complexity_filter + - nextflow run ${TRAVIS_BUILD_DIR} -profile test,docker --pairedEnd --complexity_filter --bwa_index results/reference_genome/bwa_index/ # Test BAM Trimming - - nextflow run ${TRAVIS_BUILD_DIR} -profile test,docker --pairedEnd --trim_bam + - nextflow run ${TRAVIS_BUILD_DIR} -profile test,docker --pairedEnd --trim_bam --bwa_index results/reference_genome/bwa_index/ # Test running with CircularMapper - nextflow run ${TRAVIS_BUILD_DIR} -profile test,docker --pairedEnd --circularmapper --circulartarget 'NC_007596.2' # Test running with BWA Mem - - nextflow run ${TRAVIS_BUILD_DIR} -profile test,docker --pairedEnd --bwamem + - nextflow run ${TRAVIS_BUILD_DIR} -profile test,docker --pairedEnd --bwamem --bwa_index results/reference_genome/bwa_index/ # Test basic pipeline with Conda too - - nextflow run ${TRAVIS_BUILD_DIR} -profile test,conda --pairedEnd \ No newline at end of file + - nextflow run ${TRAVIS_BUILD_DIR} -profile test,conda --pairedEnd --bwa_index results/reference_genome/bwa_index/ \ No newline at end of file From 6b463cf7a548a527be6c5ec299a64796ddac0f1f Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 5 Nov 2018 16:11:57 +0100 Subject: [PATCH 13/14] Conda environment takes quite long unfortunately to build --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6003abcb4..65e95c56f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,4 +49,4 @@ script: # Test running with BWA Mem - nextflow run ${TRAVIS_BUILD_DIR} -profile test,docker --pairedEnd --bwamem --bwa_index results/reference_genome/bwa_index/ # Test basic pipeline with Conda too - - nextflow run ${TRAVIS_BUILD_DIR} -profile test,conda --pairedEnd --bwa_index results/reference_genome/bwa_index/ \ No newline at end of file + - travis_wait 25 nextflow run ${TRAVIS_BUILD_DIR} -profile test,conda --pairedEnd --bwa_index results/reference_genome/bwa_index/ \ No newline at end of file From 02b528afb9f429f2aac965139a950e725c504cbb Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 5 Nov 2018 19:13:45 +0100 Subject: [PATCH 14/14] Add some stuff to changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ed14b60d..b0f862f28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## unpublished -* Index file handling for bwa_index, fasta_index, seq_dict TODO +### `Added` +* [#80](https://github.com/nf-core/eager/pull/80) - BWA Index file handling * [#77](https://github.com/nf-core/eager/pull/77) - Lots of documentation updates by [@jfy133](https://github.com/jfy133) ## [2.0.2] - 2018-11-03