Merge pull request #796 from maxplanck-ie/develop

release 2.5.2

.azure-pipelines/setup.yml

@@ -7,7 +7,7 @@ steps:
 #    installOptions: -c conda-forge -c bioconda --quiet
 #    createOptions: -c conda-forge -c bioconda --quiet --yes
 - bash: |
-    conda create -n foo -q --yes --quiet -c conda-forge -c bioconda snakemake=5.18.0 fuzzywuzzy mock sphinx sphinx-argparse sphinx_rtd_theme flake8 coreutils python=3.7
+    conda create -n foo -q --yes --quiet -c conda-forge -c bioconda snakemake=6.2.1 fuzzywuzzy mock sphinx sphinx-argparse sphinx_rtd_theme flake8 coreutils python=3.7
   displayName: Installing dependencies
 - bash: |
     source activate foo

.ci_stuff/test_dag.sh

@@ -275,15 +275,15 @@ if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 686 ]; then exit 1 ; fi
 
 # HiC
 WC=`HiC -i PE_input -o output --snakemakeOptions " --dryrun --conda-prefix /tmp" --correctionMethod ICE .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v "Conda environment" | sed '/^\s*$/d' | wc -l`
-if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 487 ]; then exit 1 ; fi
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 495 ]; then exit 1 ; fi
 WC=`HiC -i PE_input -o output --snakemakeOptions " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v "Conda environment" | sed '/^\s*$/d' | wc -l`
-if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 455 ]; then exit 1 ; fi
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 463 ]; then exit 1 ; fi
 WC=`HiC -i PE_input -o output --snakemakeOptions " --dryrun --conda-prefix /tmp" --trim .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v "Conda environment" | sed '/^\s*$/d' | wc -l`
-if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 511 ]; then exit 1 ; fi
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 519 ]; then exit 1 ; fi
 WC=`HiC -i PE_input -o output --snakemakeOptions " --dryrun --conda-prefix /tmp" --enzyme DpnII .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v "Conda environment" | sed '/^\s*$/d' | wc -l`
-if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 455 ]; then exit 1 ; fi
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 463 ]; then exit 1 ; fi
 WC=`HiC -i PE_input -o output --snakemakeOptions " --dryrun --conda-prefix /tmp" --noTAD .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v "Conda environment" | sed '/^\s*$/d' | wc -l`
-if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 405 ]; then exit 1 ; fi
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 413 ]; then exit 1 ; fi
 
 # preprocessing
 WC=`preprocessing -i PE_input -o output --snakemakeOptions " --dryrun --conda-prefix /tmp"  --fastqc --optDedupDist 2500 | tee >(cat 1>&2) | grep -v "Conda environment" | sed '/^\s*$/d' | wc -l`

.snakemake-workflow-catalog.yml

@@ -0,0 +1,4 @@
+usage:
+  software-stack-deployment:
+    conda: true 
+  report: true 

README.rst

@@ -15,7 +15,7 @@ snakePipes
     :alt: Citation
 
 
-snakePipes are flexible and powerful workflows built using `snakemake <snakemake.readthedocs.io>`__ that simplify the analysis of NGS data.
+snakePipes are flexible and powerful workflows built using `Snakemake <https://snakemake.readthedocs.io>`__ that simplify the analysis of NGS data.
 
 .. image:: ./docs/content/images/snakePipes_small.png
    :scale: 20 %
@@ -40,7 +40,7 @@ Workflows available
 Installation
 -------------
 
-Snakepipes uses conda for installation and dependency resolution, so you will need to `install conda <https://conda.io/docs/user-guide/install/index.html>`__ first.
+Snakepipes is a set of Snakemake workflows which use conda for installation and dependency resolution, so you will need to `install conda <https://conda.io/docs/user-guide/install/index.html>`__ first.
 
 Afterward, simply run the following:
 

conda-recipe/meta.yaml

@@ -1,6 +1,6 @@
 package:
   name: snakepipes
-  version: 2.5.1
+  version: 2.5.2
 
 source:
   path: ../

docs/content/News.rst

@@ -1,6 +1,17 @@
 snakePipes News
 ===============
 
+snakePipes 2.5.2
+----------------
+
+* snakemake version >= 6.2.1
+* python version >= 3.7
+* added snakeMake in readme
+* bug fix 777, 781
+* multiqc version = 1.10.1 to be able to report qc fir hic pipeline
+* added snakemake catalog yaml
+
+
 snakePipes 2.5.1
 ----------------
 

requirements.txt

@@ -1,4 +1,4 @@
-snakemake==5.18.0
+snakemake >= 6.2.1
 psutil
 pandas
 fuzzywuzzy

setup.py

@@ -18,6 +18,7 @@
     scripts=scripts,
     packages=find_packages(),
     include_package_data=True,
+    python_requires='>=3.7',
     install_requires=requires,
     url='https://github.com/maxplanck-ie/snakepipes',
     license='MIT',

snakePipes/__init__.py

@@ -1 +1 @@
-__version__ = '2.5.1'
+__version__ = '2.5.2'

snakePipes/shared/rules/ChIP_peak_calling.snakefile

@@ -28,7 +28,7 @@ if pairedEnd:
                 else "",
             genome_size = str(genome_size),
             ext_size =
-                lambda wildcards: " --nomodel --extsize "+get_pe_frag_length(wildcards.chip_sample,
+                lambda wildcards: " --nomodel --extsize "+get_pe_frag_length("filtered_bam/"+wildcards.chip_sample+".filtered.bam",
                                                                             "deepTools_qc/bamPEFragmentSize/fragmentSize.metric.tsv") \
                                                                             if not cutntag else " ",
             peakCaller_options = lambda wildcards: str(peakCallerOptions or '') if not cutntag else " -p 1e-5 ",

snakePipes/shared/rules/ChIP_peak_calling_spikein.snakefile

@@ -30,7 +30,7 @@ if pairedEnd:
                 lambda wildcards: "-c split_bam/"+get_control(wildcards.chip_sample)+"_host.bam" if get_control(wildcards.chip_sample)
                 else "",
             ext_size =
-                lambda wildcards: " --nomodel --extsize "+get_pe_frag_length(wildcards.chip_sample,
+                lambda wildcards: " --nomodel --extsize "+get_pe_frag_length("split_bam/"+wildcards.chip_sample+"_host.bam",
                                                                             "split_deepTools_qc/bamPEFragmentSize/host.fragmentSize.metric.tsv") \
                                                                             if not cutntag else " ",
             peakCaller_options = lambda wildcards: str(peakCallerOptions or '') if not cutntag else " -p 1e-5 ",

snakePipes/shared/rules/envs/shared.yaml

@@ -3,14 +3,14 @@ channels:
  - conda-forge
  - bioconda
 dependencies:
- - deeptools = 3.3.2
+ - deeptools = 3.5.0
  - seqtk = 1.3
  - pigz = 2.3.4
  - snpsplit = 0.3.4
  - samtools = 1.10
  - fastqc = 0.11.9
  - cutadapt = 2.8
  - trim-galore = 0.6.5
- - multiqc = 1.8
+ - multiqc = 1.10.1
  - fastp = 0.20.0
  - umi_tools = 1.0.1

snakePipes/shared/rules/multiQC.snakefile

@@ -30,7 +30,8 @@ def multiqc_input_check(return_value):
     if pipeline=="dna-mapping":
         # pipeline is DNA-mapping
         if aligner=="Bowtie2":
-            infiles.append( expand("Bowtie2/{sample}.Bowtie2_summary.txt", sample = samples) +
+            infiles.append("deepTools_qc/bamPEFragmentSize/fragmentSize.metric.tsv")
+            infiles.append(expand("Bowtie2/{sample}.Bowtie2_summary.txt", sample = samples) +
                     expand("Sambamba/{sample}.markdup.txt", sample = samples) +
                     expand("deepTools_qc/estimateReadFiltering/{sample}_filtering_estimation.txt",sample=samples))
             indir += " Sambamba "

snakePipes/workflows/ChIP-seq/internals.snakefile

@@ -64,7 +64,7 @@ def is_allelic(workingdir):
 def get_pe_frag_length(sample, frag_len_file):
     try:
         df = pd.read_csv(frag_len_file, header = None, skiprows = 1, sep = "\t")
-        df = df.loc[df[0] == "filtered_bam/"+sample+".filtered.bam"]
+        df = df.loc[df[0] == sample]
         frag_len = int(df[5].values[0])
         return str(frag_len)
     except:

snakePipes/workflows/HiC/Snakefile

@@ -147,7 +147,7 @@ rule all:
         run_build_matrices(),
         expand("HiC_matrices/QCplots/{sample}_QC/QC.log", sample = samples),
         run_dist_vs_count(),
-        #"multiQC/multiqc_report.html"
+        "multiQC/multiqc_report.html"
 
 ### execute after workflow finished ############################################
 ################################################################################