Merge pull request #899 from maxplanck-ie/dev_wd

Dev wd

.github/snakePipesEnvCI.yml

@@ -5,7 +5,7 @@ channels:
 dependencies:
   - python=3.9
   - snakemake=6.2.1
-  - fuzzywuzzy
+  - thefuzz[speedup]
   - mock
   - sphinx=4.2.0
   - sphinx-argparse

.github/workflows/linux.yml

@@ -75,6 +75,7 @@ jobs:
           'CONDA_ATAC_ENV',
           'CONDA_HIC_ENV',
           'CONDA_WGBS_ENV',
+          'CONDA_DSS_ENV',
           'CONDA_RMD_ENV',
           'CONDA_PREPROCESSING_ENV',
           'CONDA_NONCODING_RNASEQ_ENV',

.github/workflows/osx.yml

@@ -26,6 +26,7 @@ jobs:
           'CONDA_ATAC_ENV',
           'CONDA_HIC_ENV',
           'CONDA_WGBS_ENV',
+          'CONDA_DSS_ENV',
           'CONDA_RMD_ENV',
           'CONDA_PREPROCESSING_ENV',
           'CONDA_NONCODING_RNASEQ_ENV',

conda-recipe/meta.yaml

@@ -1,6 +1,6 @@
 package:
   name: snakepipes
-  version: 2.7.2
+  version: 2.7.3
 
 source:
   path: ../
@@ -18,7 +18,7 @@ requirements:
     - tabulate <0.9
     - pandas
     - graphviz
-    - fuzzywuzzy
+    - thefuzz[speedup]
     - pyyaml >=5.1
     - coreutils
 

docs/conf.py

@@ -17,7 +17,7 @@
 
 # to allow readthedocs to compile without installing some dependencies
 import mock
-MOCK_MODULES = ['yaml', 'fuzzywuzzy']
+MOCK_MODULES = ['yaml', 'thefuzz[speedup]']
 
 for mod_name in MOCK_MODULES:
     sys.modules[mod_name] = mock.Mock()

docs/content/News.rst

@@ -5,11 +5,10 @@ snakePipes x.x.x
 ----------------
 * Changed the behaviour of snakePipes createEnvs - it is no longer possible to set condaEnvDir with this function. It is required to set it with snakePipes config beforhand, instead. To ingore what's in the defaults.yaml and overwrite the condaEnvDir value with default system conda prefix, use '--autodetectCondaEnvDir'.
 * Snakemake options in the defaults.yaml are now an empty string. The required arguments '--use-conda --conda-prefix' have been directly added to the command string. condaEnvDir is parsed from defaults.yaml, requiring running snakePipes config first.
-
 * Fixes #819
-
-
-
+* fastq files are checked for validity
+* an 'on success' file is touched in the output directory when a workflow is finished successfully
+* fuzzywuzzy deprecated in favor for thefuzz
 
 snakePipes 2.7.2
 ----------------

requirements.txt

@@ -1,6 +1,6 @@
 snakemake >= 6.2.1, <7.19.0
 psutil
 pandas
-fuzzywuzzy
+thefuzz[speedup]
 pyyaml >= 5.1
 tabulate < 0.9

snakePipes/__init__.py

@@ -1 +1 @@
-__version__ = '2.7.2'
+__version__ = '2.7.3'

snakePipes/common_functions.py

@@ -9,7 +9,8 @@
 import glob
 import sys
 import shutil
-from fuzzywuzzy import fuzz
+from pathlib import Path
+from thefuzz import fuzz
 import smtplib
 from email.message import EmailMessage
 from snakePipes import __version__
@@ -34,6 +35,7 @@ def set_env_yamls():
             'CONDA_ATAC_ENV': 'envs/atac_seq.yaml',
             'CONDA_HIC_ENV': 'envs/hic.yaml',
             'CONDA_WGBS_ENV': 'envs/wgbs.yaml',
+            'CONDA_DSS_ENV': 'envs/wgbs_dss.yaml',
             'CONDA_RMD_ENV': 'envs/rmarkdown.yaml',
             'CONDA_PREPROCESSING_ENV': 'envs/preprocessing.yaml',
             'CONDA_NONCODING_RNASEQ_ENV': 'envs/noncoding.yaml',
@@ -720,8 +722,10 @@ def runAndCleanup(args, cmd, logfile_name):
             sendEmail(args, p.returncode)
         sys.exit(p.returncode)
     else:
+        Path(
+            os.path.join(args.outdir, "snakePipes.done")
+        ).touch()
         if os.path.exists(os.path.join(args.outdir, ".snakemake")):
-            import shutil
             shutil.rmtree(os.path.join(args.outdir, ".snakemake"), ignore_errors=True)
 
     # Send email if desired

snakePipes/shared/rules/FASTQ.snakefile

@@ -1,6 +1,20 @@
+rule validateFQ:
+    input:
+        r1 = indir+"/{sample}"+reads[0]+ext
+    output:
+        temp("originalFASTQ/{sample}.valid")
+    params:
+        sdir = lambda wildcards: "{}/{}*".format(indir, wildcards.sample)
+    conda: CONDA_SHARED_ENV
+    shell:"""
+        fq lint {params.sdir}*
+        touch {output}
+        """
+
 rule origFASTQ1:
       input:
-          indir+"/{sample}"+reads[0]+ext
+          r1 = indir+"/{sample}"+reads[0]+ext,
+          valid = 'originalFASTQ/{sample}.valid'
       output:
           "originalFASTQ/{sample}"+reads[0]+".fastq.gz"
       params:
@@ -12,7 +26,8 @@ rule origFASTQ1:
 if pairedEnd or pipeline=="scrna-seq":
     rule origFASTQ2:
         input:
-            indir+"/{sample}"+reads[1]+ext
+            r2 = indir+"/{sample}"+reads[1]+ext,
+            valid = 'originalFASTQ/{sample}.valid'
         output:
             "originalFASTQ/{sample}"+reads[1]+".fastq.gz"
         params:
@@ -21,6 +36,7 @@ if pairedEnd or pipeline=="scrna-seq":
                {params.cmd}
             """
 
+
 if downsample:
     if pairedEnd:
         rule FASTQdownsample:

snakePipes/shared/rules/WGBS.snakefile

@@ -376,7 +376,7 @@ rule DSS:
         FDR=FDR
     threads: lambda wildcards: 10 if 10<max_thread else max_thread
     benchmark: '{}/.benchmark/DSS.benchmark'.format(get_outdir("DSS", None, minCoverage))
-    conda: CONDA_WGBS_ENV
+    conda: CONDA_DSS_ENV
     script: "../rscripts/WGBS_DSS.Rmd"
 
 

snakePipes/shared/rules/envs/shared.yaml

@@ -15,5 +15,6 @@ dependencies:
  - multiqc = 1.12
  - fastp = 0.23.2
  - umi_tools = 1.1.2
+ - fq = 0.10.0
  - pybigwig = 0.3.18
-
+

snakePipes/shared/rules/envs/wgbs.yaml

@@ -18,12 +18,8 @@ dependencies:
   - r-dplyr
   - r-tidyr
   - r-pander
-  - r-factominer
-  - bioconductor-bsseq
   - bioconductor-genomicranges
   - bioconductor-genomicfeatures
-  - bioconductor-biomart
-  - bioconductor-dss
   - bioconductor-dmrseq
   - bioconductor-rtracklayer
   - bioconductor-biocparallel

snakePipes/shared/rules/envs/wgbs_dss.yaml

@@ -0,0 +1,16 @@
+name: WGBS_DSS-env-2.0
+channels:
+  - mpi-ie
+  - conda-forge
+  - bioconda
+dependencies:
+  - r-base
+  - r-car
+  - r-factominer
+  - r-ggplot2
+  - r-rmarkdown
+  - r-dplyr
+  - r-tidyr
+  - r-pander
+  - bioconductor-dss
+  - bioconductor-biocparallel

snakePipes/shared/rules/scRNAseq_Alevin.snakefile

@@ -1,7 +1,10 @@
 ### cut columns 1 and 3 from t2g tsv #########
 import gzip
+import os
 
 def get_flank_length(file,read_length_frx):
+    if not os.path.exists(file):
+        return (0)
     with gzip.open(file,"r") as rf:
         head = [next(rf) for x in range(4)]
     read_length = len(head[1])