Merge pull request #763 from maxplanck-ie/max_thread

Max thread & spike-in bamCompare

.ci_stuff/test_dag.sh

@@ -168,10 +168,10 @@ WC=`ChIP-seq -d outdir --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeO
 if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 717 ]; then exit 1 ; fi
 # spikein
 WC=`ChIP-seq -d BAM_input --useSpikeInForNorm --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp"  .ci_stuff/spikein_organism.yaml .ci_stuff/ChIP.sample_config.yaml | tee >(cat 1>&2) | grep -v "Conda environment" | sed '/^\s*$/d' | wc -l`
-if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 435 ]; then exit 1 ; fi
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 635 ]; then exit 1 ; fi
 # fromBAM and spikein
 WC=`ChIP-seq -d outdir --useSpikeInForNorm --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" --fromBAM BAM_input/filtered_bam/ .ci_stuff/spikein_organism.yaml .ci_stuff/ChIP.sample_config.yaml | tee >(cat 1>&2) | grep -v "Conda environment" | sed '/^\s*$/d' | wc -l`
-if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 593 ]; then exit 1 ; fi
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 793 ]; then exit 1 ; fi
 WC=`ChIP-seq -d outdir --useSpikeInForNorm --getSizeFactorsFrom TSS --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" --fromBAM BAM_input/filtered_bam/ .ci_stuff/spikein_organism.yaml .ci_stuff/ChIP.sample_config.yaml | tee >(cat 1>&2) | grep -v "Conda environment" | sed '/^\s*$/d' | wc -l`
 if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 593 ]; then exit 1 ; fi
 WC=`ChIP-seq -d outdir --useSpikeInForNorm --getSizeFactorsFrom input --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" --fromBAM BAM_input/filtered_bam/ .ci_stuff/spikein_organism.yaml .ci_stuff/ChIP.sample_config.yaml | tee >(cat 1>&2) | grep -v "Conda environment" | sed '/^\s*$/d' | wc -l`

snakePipes/shared/defaults.yaml

@@ -18,6 +18,8 @@ onlySSL: False
 emailSender:
 smtpUsername:
 smtpPassword:
+# maximum number of threads to be used
+max_thread: 25
 #print tools versions
 toolsVersion: True
 oldConfig:

snakePipes/shared/rules/ATAC_qc.snakefile

@@ -20,7 +20,7 @@ rule plotFingerprint:
         err = os.path.join(deeptools_ATAC, "logs/plotFingerprint.err")
     benchmark:
         os.path.join(deeptools_ATAC, ".benchmark/plotFingerprint.benchmark")
-    threads: 24
+    threads: lambda wildcards: 24 if 24<max_thread else max_thread
     conda: CONDA_SHARED_ENV
     shell: plotFingerprint_cmd
 
@@ -44,7 +44,7 @@ rule plotFingerprint_allelic:
         err = os.path.join(deeptools_ATAC, "logs/plotFingerprint_allelic.err")
     benchmark:
         os.path.join(deeptools_ATAC, ".benchmark/plotFingerprint_allelic.benchmark")
-    threads: 24
+    threads: lambda wildcards: 24 if 24<max_thread else max_thread
     conda: CONDA_SHARED_ENV
     shell: plotFingerprint_cmd
 

snakePipes/shared/rules/Bowtie2.snakefile

@@ -17,7 +17,7 @@ if pairedEnd:
             tempDir = tempDir
         benchmark:
             "Bowtie2/.benchmark/Bowtie2.{sample}.benchmark"
-        threads: 24  # 1G per core
+        threads: lambda wildcards: 24 if 24<max_thread else max_thread  # 1G per core
         conda: CONDA_DNA_MAPPING_ENV
         shell: """
             TMPDIR={params.tempDir}
@@ -48,7 +48,7 @@ else:
             tempDir = tempDir
         benchmark:
             "Bowtie2/.benchmark/Bowtie2.{sample}.benchmark"
-        threads: 24  # 1G per core
+        threads: lambda wildcards: 24 if 24<max_thread else max_thread  # 1G per core
         conda: CONDA_DNA_MAPPING_ENV
         shell: """
             TMPDIR={params.tempDir}

snakePipes/shared/rules/Bowtie2_allelic.snakefile

@@ -27,7 +27,7 @@ if aligner == "Bowtie2":
                 tempDir = tempDir
             benchmark:
                 aligner+"/.benchmark/Bowtie2.{sample}.benchmark"
-            threads: 24  # 1G per core
+            threads: lambda wildcards: 24 if 24<max_thread else max_thread  # 1G per core
             conda: CONDA_DNA_MAPPING_ENV
             shell: """
                 TMPDIR={params.tempDir}
@@ -59,7 +59,7 @@ if aligner == "Bowtie2":
                 tempDir = tempDir
             benchmark:
                 aligner+"/.benchmark/Bowtie2.{sample}.benchmark"
-            threads: 24  # 1G per core
+            threads: lambda wildcards: 24 if 24<max_thread else max_thread  # 1G per core
             conda: CONDA_DNA_MAPPING_ENV
             shell: """
                 TMPDIR={params.tempDir}

snakePipes/shared/rules/FASTQ.snakefile

@@ -35,12 +35,12 @@ if downsample:
                 num_reads = downsample
             benchmark:
                 "FASTQ/.benchmark/FASTQ_downsample.{sample}.benchmark"
-            threads: 10
+            threads: lambda wildcards: 10 if 10<max_thread else max_thread
             conda: CONDA_SHARED_ENV
             shell: """
                 seqtk sample -s 100 {input.r1} {params.num_reads} | pigz -p {threads} -9 > {output.r1} 2> {log}
                 seqtk sample -s 100 {input.r2} {params.num_reads} | pigz -p {threads} -9 > {output.r2} 2>> {log}
-                
+
                 """
     else:
         rule FASTQdownsample:
@@ -49,7 +49,7 @@ if downsample:
             output:
                 fq = "originalFASTQ/downsample_{sample}.fastq.gz"
             log: "originalFASTQ/logs/{sample}.FASTQdownsample.log"
-            threads: 12
+            threads: lambda wildcards: 12 if 12<max_thread else max_thread
             params:
                 num_reads = downsample
             conda: CONDA_SHARED_ENV

snakePipes/shared/rules/Qualimap_bamqc.snakefile

@@ -14,7 +14,7 @@ rule Qualimap_bamqc:
         err = "Qualimap_qc/logs/Qualimap_bamqc.{sample}.filtered.err"
     benchmark:
         "Qualimap_qc/.benchmark/Qualimap_bamqc.{sample}.filtered.benchmark"
-    threads: 16
+    threads: lambda wildcards: 16 if 16<max_thread else max_thread
     conda: CONDA_DNA_MAPPING_ENV
     shell:
         "unset DISPLAY && "

snakePipes/shared/rules/RNA_mapping.snakefile

@@ -42,7 +42,7 @@ if aligner.upper().find("HISAT2") >=0:
                 tempDir = tempDir
             benchmark:
                 aligner+"/.benchmark/HISAT2.{sample}.benchmark"
-            threads: 10
+            threads: lambda wildcards: 10 if 10<max_thread else max_thread
             conda: CONDA_RNASEQ_ENV
             shell: """
                 TMPDIR={params.tempDir}
@@ -75,7 +75,7 @@ if aligner.upper().find("HISAT2") >=0:
                 tempDir = tempDir
             benchmark:
                 aligner+"/.benchmark/HISAT2.{sample}.benchmark"
-            threads: 10
+            threads: lambda wildcards: 10 if 10<max_thread else max_thread
             conda: CONDA_RNASEQ_ENV
             shell: """
                 TMPDIR={params.tempDir}
@@ -109,7 +109,7 @@ elif aligner.upper().find("STAR") >=0:
                 tempDir = tempDir
             benchmark:
                 aligner+"/.benchmark/STAR.{sample}.benchmark"
-            threads: 20  # 3.2G per core
+            threads: lambda wildcards: 20 if 20<max_thread else max_thread  # 3.2G per core
             conda: CONDA_RNASEQ_ENV
             shell: """
                 TMPDIR={params.tempDir}
@@ -146,7 +146,7 @@ elif aligner.upper().find("STAR") >=0:
                 tempDir = tempDir
             benchmark:
                 aligner+"/.benchmark/STAR.{sample}.benchmark"
-            threads: 20  # 3.2G per core
+            threads: lambda wildcards: 20 if 20<max_thread else max_thread  # 3.2G per core
             conda: CONDA_RNASEQ_ENV
             shell: """
                 TMPDIR={params.tempDir}

snakePipes/shared/rules/RNA_mapping_allelic.snakefile

@@ -19,7 +19,7 @@ if aligner == "STAR":
                 tempDir = tempDir
             benchmark:
                 aligner+"/.benchmark/STAR.{sample}.benchmark"
-            threads: 12
+            threads: lambda wildcards: 12 if 12<max_thread else max_thread
             conda: CONDA_RNASEQ_ENV
             shell: """
                 TMPDIR={params.tempDir}
@@ -65,7 +65,7 @@ if aligner == "STAR":
                 tempDir = tempDir
             benchmark:
                 aligner+"/.benchmark/STAR.{sample}.benchmark"
-            threads: 12
+            threads: lambda wildcards: 12 if 12<max_thread else max_thread
             conda: CONDA_RNASEQ_ENV
             shell: """
                 TMPDIR={params.tempDir}

snakePipes/shared/rules/Salmon.snakefile

@@ -15,7 +15,7 @@ rule SalmonIndex:
     log:
         out = "Salmon/SalmonIndex/SalmonIndex.out",
         err = "Salmon/SalmonIndex/SalmonIndex.err",
-    threads: 16
+    threads: lambda wildcards: 16 if 16<max_thread else max_thread
     conda: CONDA_RNASEQ_ENV
     shell: """
         grep "^>" {input[1]} | cut -d " " -f 1 | tr -d ">" > {output[0]}

snakePipes/shared/rules/WGBS.snakefile

@@ -32,7 +32,7 @@ if pairedEnd and not fromBAM:
         params:
             bwameth_index=bwameth_index,
             tempDir = tempDir
-        threads: 20
+        threads: lambda wildcards: 20 if 20<max_thread else max_thread
         conda: CONDA_WGBS_ENV
         shell: """
             TMPDIR={params.tempDir}
@@ -54,7 +54,7 @@ elif not pairedEnd and not fromBAM:
         params:
             bwameth_index=bwameth_index,
             tempDir = tempDir
-        threads: 20
+        threads: lambda wildcards: 20 if 20<max_thread else max_thread
         conda: CONDA_WGBS_ENV
         shell: """
             TMPDIR={params.tempDir}
@@ -88,7 +88,7 @@ if not skipBamQC:
         log:
             err="Sambamba/logs/{sample}.rm_dupes.err",
             out="Sambamba/logs/{sample}.rm_dupes.out"
-        threads: 10
+        threads: lambda wildcards: 10 if 10<max_thread else max_thread
         params:
             tempDir = tempDir
         conda: CONDA_SAMBAMBA_ENV
@@ -189,7 +189,7 @@ rule calc_Mbias:
         genome=genome_fasta
     log:
         out="QC_metrics/logs/{sample}.calc_Mbias.out"
-    threads: 10
+    threads: lambda wildcards: 10 if 10<max_thread else max_thread
     conda: CONDA_WGBS_ENV
     shell: """
         MethylDackel mbias -@ {threads} {params.genome} {input[0]} QC_metrics/{wildcards.sample} 2> {output} > {log.out}
@@ -206,7 +206,7 @@ rule calcCHHbias:
         genome=genome_fasta
     log:
         err="QC_metrics/logs/{sample}.calcCHHbias.err"
-    threads: 10
+    threads: lambda wildcards: 10 if 10<max_thread else max_thread
     conda: CONDA_WGBS_ENV
     shell: """
         MethylDackel mbias -@ {threads} --CHH --noCpG --noSVG {params.genome} {input[0]} QC_metrics/{wildcards.sample} > {output} 2> {log.err}
@@ -225,7 +225,7 @@ rule calc_GCbias:
         twobitpath=genome_2bit
     log:
         out="QC_metrics/logs/calc_GCbias.out"
-    threads: 20
+    threads: lambda wildcards: 20 if 20<max_thread else max_thread
     conda: CONDA_SHARED_ENV
     shell: """
         computeGCBias -b {input.BAMS} --effectiveGenomeSize {params.genomeSize} -g {params.twobitpath} -l 300 --GCbiasFrequenciesFile {output[0]} -p {threads} --biasPlot {output[1]}
@@ -244,7 +244,7 @@ rule DepthOfCov:
     params:
         options="--minMappingQuality 10 --smartLabels --samFlagExclude 256",
         thresholds="-ct 0 -ct 1 -ct 2 -ct 5 -ct 10 -ct 15 -ct 20 -ct 30 -ct 50"
-    threads: 20
+    threads: lambda wildcards: 20 if 20<max_thread else max_thread
     log:
         err="QC_metrics/logs/DepthOfCov.err"
     conda: CONDA_SHARED_ENV
@@ -265,7 +265,7 @@ rule DepthOfCovGenome:
     params:
         options="--minMappingQuality 10 --smartLabels --samFlagExclude 256",
         thresholds="-ct 0 -ct 1 -ct 2 -ct 5 -ct 10 -ct 15 -ct 20 -ct 30 -ct 50"
-    threads: 20
+    threads: lambda wildcards: 20 if 20<max_thread else max_thread
     log:
         err="QC_metrics/logs/DepthOfCovGenome.err"
     conda: CONDA_SHARED_ENV
@@ -316,7 +316,7 @@ if not noAutoMethylationBias:
         log:
             err="MethylDackel/logs/{sample}.methyl_extract.err",
             out="MethylDackel/logs/{sample}.methyl_extract.out"
-        threads: 10
+        threads: lambda wildcards: 10 if 10<max_thread else max_thread
         conda: CONDA_WGBS_ENV
         shell: """
             mi=$(cat {input[2]} | sed 's/Suggested inclusion options: //' )
@@ -335,7 +335,7 @@ else:
         log:
             err="MethylDackel/logs/{sample}.methyl_extract.err",
             out="MethylDackel/logs/{sample}.methyl_extract.out"
-        threads: 10
+        threads: lambda wildcards: 10 if 10<max_thread else max_thread
         conda: CONDA_WGBS_ENV
         shell: """
             MethylDackel extract -o MethylDackel/{wildcards.sample} {params.MethylDackelOptions} -@ {threads} {params.genome} {input[0]} 1> {log.out} 2> {log.err}
@@ -354,7 +354,7 @@ rule prepForMetilene:
         blacklist=blacklist
     log:
         err='{}/logs/prep_for_stats.err'.format(get_outdir("metilene", targetRegions, minCoverage)),
-    threads: 10
+    threads: lambda wildcards: 10 if 10<max_thread else max_thread
     conda: CONDA_WGBS_ENV
     script: "../rscripts/WGBS_mergeStats.R"
 
@@ -374,7 +374,7 @@ rule DSS:
         minMethDiff=minMethDiff,
         minCoverage=minCoverage,
         FDR=FDR
-    threads: 10
+    threads: lambda wildcards: 10 if 10<max_thread else max_thread
     benchmark: '{}/.benchmark/DSS.benchmark'.format(get_outdir("DSS", None, minCoverage))
     conda: CONDA_WGBS_ENV
     script: "../rscripts/WGBS_DSS.Rmd"
@@ -395,7 +395,7 @@ rule dmrseq:
         minMethDiff=minMethDiff,
         minCoverage=minCoverage,
         FDR=FDR
-    threads: 10
+    threads: lambda wildcards: 10 if 10<max_thread else max_thread
     benchmark: '{}/.benchmark/dmrseq.benchmark'.format(get_outdir("dmrseq", None, minCoverage))
     conda: CONDA_WGBS_ENV
     script: "../rscripts/WGBS_dmrseq.Rmd"
@@ -418,7 +418,7 @@ rule run_metilene:
         opts = metileneOptions if metileneOptions else ''
     log:
         err="{}/logs/run_metilene.err".format(get_outdir("metilene", targetRegions, minCoverage))
-    threads: 10
+    threads: lambda wildcards: 10 if 10<max_thread else max_thread
     benchmark: '{}/.benchmark/run_metilene.benchmark'.format(get_outdir("metilene", targetRegions, minCoverage))
     conda: CONDA_WGBS_ENV
     shell: """
@@ -456,7 +456,7 @@ rule metileneReport:
 
 
 rule bedGraphToBigWig:
-    input: 
+    input:
         "MethylDackel/{sample}_CpG.bedGraph",
         genome_index
     output:

snakePipes/shared/rules/bam_filtering.snakefile

@@ -11,13 +11,13 @@ rule samtools_filter:
     output:
         bam = temp("filtered_bam/{sample}.filtered.tmp.bam")
     params:
-        shell = lambda wildcards,input,output: "samtools view -@ {} -b {} -o {} {} ".format(str(8 if not local else 2), bam_filter_string,output.bam,input[0]) if bam_filter_string.strip() !="" else "ln -s ../{} {}".format(input[0],output.bam) 
+        shell = lambda wildcards,input,output: "samtools view -@ {} -b {} -o {} {} ".format(str(8 if not local else 2), bam_filter_string,output.bam,input[0]) if bam_filter_string.strip() !="" else "ln -s ../{} {}".format(input[0],output.bam)
     log:
         out = "filtered_bam/logs/samtools_filter.{sample}.out",
         err = "filtered_bam/logs/samtools_filter.{sample}.err"
     benchmark:
         "filtered_bam/.benchmark/samtools_filter.{sample}.benchmark"
-    threads: 8
+    threads: lambda wildcards: 8 if 8<max_thread else max_thread
     conda: CONDA_SHARED_ENV
     shell: """
         {params.shell} 2> {log.err}

snakePipes/shared/rules/bwa.snakefile

@@ -11,8 +11,8 @@ if pairedEnd:
             bwa_index = bwa_index,
             alignerOpts = str(alignerOpts or ''),
             tempDir = tempDir
-        threads: 24
-        conda: CONDA_DNA_MAPPING_ENV 
+        threads: lambda wildcards: 24 if 24<max_thread else max_thread
+        conda: CONDA_DNA_MAPPING_ENV
         shell:"""
             TMPDIR={params.tempDir}
             MYTEMP=$(mktemp -d ${{TMPDIR:-/tmp}}/snakepipes.XXXXXXXXXX);
@@ -37,7 +37,7 @@ else:
             bwa_index = bwa_index,
             alignerOpts = str(alignerOpts or ''),
             tempDir = tempDir
-        threads : 24 
+        threads : lambda wildcards: 24 if 24<max_thread else max_thread
         conda: CONDA_DNA_MAPPING_ENV
         shell: """
             TMPDIR={params.tempDir}

snakePipes/shared/rules/createIndices.snakefile

@@ -189,7 +189,7 @@ rule bowtie2Index:
     params:
       basedir = os.path.join(outdir, "BowtieIndex")
     conda: CONDA_CREATE_INDEX_ENV
-    threads: 10
+    threads: lambda wildcards: 10 if 10<max_thread else max_thread
     shell: """
         ln -s {input} {params.basedir}/genome.fa
         bowtie2-build -t {threads} {params.basedir}/genome.fa {params.basedir}/genome
@@ -204,7 +204,7 @@ rule hisat2Index:
     log: "logs/hisat2Index.log"
     params:
       basedir = os.path.join(outdir, "HISAT2Index")
-    threads: 10
+    threads: lambda wildcards: 10 if 10<max_thread else max_thread
     conda: CONDA_CREATE_INDEX_ENV
     shell: """
         ln -s {input} {params.basedir}/genome.fa
@@ -219,7 +219,7 @@ rule makeKnownSpliceSites:
     output: known_splicesites
     log: "logs/makeKnownSpliceSites.log"
     conda: CONDA_CREATE_INDEX_ENV
-    threads: 10
+    threads: lambda wildcards: 10 if 10<max_thread else max_thread
     shell: """
         hisat2_extract_splice_sites.py {input} > {output} 2> {log}
         """
@@ -233,7 +233,7 @@ rule starIndex:
     params:
       basedir = os.path.join(outdir, "STARIndex")
     conda: CONDA_CREATE_INDEX_ENV
-    threads: 10
+    threads: lambda wildcards: 10 if 10<max_thread else max_thread
     shell: """
         STAR --runThreadN {threads} --runMode genomeGenerate --genomeDir {params.basedir} --genomeFastaFiles {input} 2> {log}
         if [[ -w Log.out ]]; then rm -v Log.out; elif [[ -w {params.basedir}/Log.out ]]; then rm -v {params.basedir}/Log.out; fi

snakePipes/shared/rules/deepTools_ATAC.snakefile

@@ -17,6 +17,6 @@ rule bamCompare_subtract:
         err = "deepTools_ATAC/logs/bamCompare.{sample}.filtered.out"
     benchmark:
         "deepTools_ATAC/.benchmark/deepTools_ATAC/logs/bamCompare.{sample}.filtered.benchmark"
-    threads: 16
+    threads: lambda wildcards: 16 if 16<max_thread else max_thread
     conda: CONDA_SHARED_ENV
     shell: bamcov_cmd