Merge branch 'master' into wgs-cnv-calling-add-gcnv

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
           - "3.7"
           - "3.8"
           - "3.9"
-          # - "3.10"  # `pysam` is incompatible with python v3.10
+          - "3.10"
 
     steps:
       - name: Install Python via conda.
@@ -34,6 +34,11 @@ jobs:
           /tmp/environment.yml
       - name: Update environment using mamba.
         run: mamba env update --name root --file /tmp/environment.yml
+      - name: Fail early on htslib etc. problems
+        run: |
+          bgzip -c /dev/null >/dev/null
+          samtools version
+          bcftools version
       - name: Install some more dependencies via pip
         run: pip install -r requirements/test.txt
       - name: Run tests
@@ -43,22 +48,16 @@ jobs:
           DIFF_LAST_COMMIT: ${{ github.ref == 'refs/heads/master' }}
       - name: Create text report
         run: coverage report
-      - name: Create XML report for codacy
-        run: coverage xml
       - name: Coveralls
         uses: AndreMiras/coveralls-python-action@develop
-      - name: Run codacy-coverage-reporter
-        uses: codacy/codacy-coverage-reporter-action@master
-        with:
-          project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
-          coverage-reports: coverage.xml
-        if: ${{ matrix.python-version == '3.7' && (github.ref == 'refs/heads/master') }}
       - name: Check import order with isort
-        run: isort --force-sort-within-sections -profile
-        if: ${{ matrix.python-version < '3.9' }}
+        run: isort --force-sort-within-sections --profile=black --check .
+        if: ${{ matrix.python-version == '3.10' }}
       - name: Check style with black
         run: black --check --line-length 100 .
-        if: ${{ matrix.python-version < '3.9' }}
+        if: ${{ matrix.python-version == '3.10' }}
       - name: Check style with snakefmt
         run: snakefmt -l 100 --check --include '(\.smk$|\.rules$|^Snakefile)' .
-        if: ${{ matrix.python-version < '3.9' }}
+        if: ${{ matrix.python-version == '3.10' }}
+      - name: Run linting
+        run: flake8

Makefile

@@ -9,6 +9,7 @@ help:
 	@echo srcfmt    -- apply black and snakefmt formatters
 	@echo lint      -- run linters
 	@echo test      -- run tests through pytest
+	@echo isort     -- run isort
 
 .PHONY: black
 black:
@@ -22,10 +23,15 @@ snakefmt:
 srcfmt: black snakefmt
 
 .PHONY: lint
-lint: prospector
+lint: flake8
 
-.PHONY: prospector
-prospector:
+.PHONY: isort
+isort:
+	isort --force-sort-within-sections --profile=black .
+
+.PHONY: flake8
+flake8:
+	flake8
 
 test:
 	py.test

README.rst

@@ -1,12 +1,8 @@
 .. image:: https://github.com/bihealth/cubi-tk/workflows/CI/badge.svg
     :target: https://github.com/bihealth/snappy-pipeline/actions
     :alt: Continuous Integration Status
-.. image:: https://app.codacy.com/project/badge/Grade/d93e08bfbb554e4fbf45eed8862beb90
-    :target: https://www.codacy.com/gh/bihealth/snappy-pipeline/dashboard?utm_source=github.com&utm_medium=referral&utm_content=bihealth/snappy-pipeline&utm_campaign=Badge_Grade
-.. image:: https://app.codacy.com/project/badge/Coverage/d93e08bfbb554e4fbf45eed8862beb90
-    :target: https://www.codacy.com/gh/bihealth/snappy-pipeline/dashboard?utm_source=github.com&utm_medium=referral&utm_content=bihealth/snappy-pipeline&utm_campaign=Badge_Coverage
-.. image:: https://coveralls.io/repos/github/bihealth/snappy-pipeline/badge.svg?branch=coveralls
-    :target: https://coveralls.io/github/bihealth/snappy-pipeline?branch=coveralls
+.. image:: https://coveralls.io/repos/github/bihealth/snappy-pipeline/badge.svg?branch=master
+    :target: https://coveralls.io/github/bihealth/snappy-pipeline?branch=master
 .. image:: https://readthedocs.org/projects/snappy-pipeline/badge/?version=latest
     :target: https://snappy-pipeline.readthedocs.io/en/latest/?badge=latest
     :alt: Documentation Status

docs/conf.py

@@ -57,8 +57,8 @@
 master_doc = "index"
 
 # General information about the project.
-project = u"SNAPPY Pipeline"
-copyright = u"2015-2021, CUBI, Berlin Institute of Health"
+project = "SNAPPY Pipeline"
+copyright = "2015-2021, CUBI, Berlin Institute of Health"
 
 # The version info for the project you're documenting, acts as replacement
 # for |version| and |release|, also used in various other places throughout
@@ -211,8 +211,8 @@
     (
         "index",
         "snappy_pipeline.tex",
-        u"SNAPPY Pipeline Documentation",
-        u"Manuel Holtgrewe",
+        "SNAPPY Pipeline Documentation",
+        "Manuel Holtgrewe",
         "manual",
     )
 ]
@@ -242,9 +242,7 @@
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    ("index", "snappy_pipeline", u"SNAPPY Pipeline Documentation", [u"Manuel Holtgrewe"], 1)
-]
+man_pages = [("index", "snappy_pipeline", "SNAPPY Pipeline Documentation", ["Manuel Holtgrewe"], 1)]
 
 # If true, show URL addresses after external links.
 # man_show_urls = False
@@ -259,8 +257,8 @@
     (
         "index",
         "snappy_pipeline",
-        u"SNAPPY Pipeline Documentation",
-        u"Manuel Holtgrewe",
+        "SNAPPY Pipeline Documentation",
+        "Manuel Holtgrewe",
         "snappy_pipeline",
         "One line description of project.",
         "Miscellaneous",

environment.yml

@@ -23,8 +23,9 @@ dependencies:
   - jinja2                  # Jinja 2 template rendering
 
   # Bioinformatics-related libraries used by snappy
-  - htslib                  # From previous environment.yml
-  - bcftools                # From previous environment.yml
+  - htslib==1.15            # From previous environment.yml
+  - bcftools==1.15          # From previous environment.yml
+  - samtools==1.15
   - vcfpy                   # Library for working with VCF files
   - pysam                   # Support for vcfpy
   - pytabix                 # Support for vcfpy

requirements/base.txt

@@ -2,7 +2,7 @@
 ruamel.yaml>=0.15.35
 
 # File-based locks
-fasteners==0.14.1
+fasteners==0.17.3
 
 # We're trying to keep the PyPi package up to date, you might have to install
 # from source, though.
@@ -14,19 +14,19 @@ termcolor==1.1.0
 # Snakemake is used for providing the actual wrapper calling functionality
 snakemake >=7.8.0
 # Snakemake needs manual install of PyYAML to make YAML configuration loading work
-PyYAML>=3.12
+PyYAML>=6.0
 
 # Required for plotting
 matplotlib>=2.1.2
 
 # Library for working with VCF files.
 vcfpy >=0.13.2
 # Support for vcfpy
-pysam
-pytabix
+pysam >=0.19.1
+pytabix >=0.1
 
 # Jinja 2 template rendering
-jinja2 >=2.10
+jinja2 >=3.1.2
 
 # Parsing of ISA-tab.
 altamisa >=0.2.6

requirements/dev.txt

@@ -6,12 +6,6 @@ ipython
 # Prettier CLI for py.test
 pytest-sugar
 
-# PDB support for py.test
-# not available from anaconda, must be installed through pip
-# remove this requirement?
-# current version: 0.3.1
-# pytest-pdb==0.2.0
-
 # Sphinx
 sphinx
 

requirements/test.txt

@@ -1,30 +1,25 @@
 -r base.txt
 
 # Wonderful py.test library, style checker etc.
-pytest
-coverage
-pytest-cov
-pytest-mock
-
-# Better linting with flake8
-# discontinued since flake8 version 3.7.0
-# flake8-per-file-ignores==0.4
+pytest ==7.1.3
+coverage ==6.4.4
+pytest-cov ==3.0.0
+pytest-mock ==3.8.2
 
 # Fake file system for testing
-pyfakefs
+pyfakefs ==4.6.3
 
 # "Black" code formatter and checker.
-black ==20.8b1
+black ==22.8.0
 # "snakefmt" code formatter and checker.
-snakefmt ==0.3.1
+snakefmt ==0.6.1
 # isort code formatter - import order
-isort ==4.3.21
+isort ==5.10.1
 
 # Linting etc. through prospector.
-prospector ==1.3.1
-flake8
-flake8-import-order
-pytest-flake8
+flake8 ==4.0.1
+flake8-import-order ==0.18.1
+pytest-flake8 ==1.1.1
 
 # coveralls.io tooling
 coveralls

setup.cfg

@@ -35,6 +35,9 @@ exclude =
     splitMNPsAndComplex.py
     wrapper.py
     snappy_pipeline/__init__.py
+    versioneer.py
+max-complexity = 18
+select = B,C,E,F,W,T4,B9
 ignore = E203, E266, E501, W503
 max-line-length = 100
 per-file-ignore =

snappy_pipeline/apps/impl/yaml_utils.py

@@ -9,22 +9,39 @@
 
 
 def remove_yaml_comment_lines(yaml_str):
-    """Get default configuration YAML string without comment-online lines"""
+    """
+    :param yaml_str: YAML configuration as string. Expected use case: clean DEFAULT_CONFIG from
+    workflows.
+    :type yaml_str: str
+
+    :return: Returns default configuration YAML string without commented nor empty lines.
+    """
     result = []
     for line in yaml_str.splitlines(True):
-        if not re.match(r"^\s*#", line):
+        if not (re.match(r"^\s*#", line) or line == "\n"):
             result.append(line)
     return "".join(result)
 
 
 def remove_non_required(yaml_obj):
-    """Remove items that are not marked as "required" with a comment (case-insensitive)"""
+    """Remove non required arguments
+
+    Remove items that are not marked as 'required' nor 'optional' with a comment (case-insensitive).
+
+    :param yaml_obj: YAML configuration.
+    :type yaml_obj: ruamel.yaml.comments.CommentedMap
+
+    :return: Returns ordered dictionary-like with arguments to be included in the snappy
+    directory config. Output is either ``CommentedMap`` or ``CommentedSeq``.
+    """
     if isinstance(yaml_obj, (dict, MutableMapping)):
         result = CommentedMap()
         for key, value in yaml_obj.items():
-            required = (
-                key in yaml_obj.ca.items and "required" in yaml_obj.ca.items[key][2].value.lower()
+            required = key in yaml_obj.ca.items and (
+                "required" in yaml_obj.ca.items[key][2].value.lower()
+                or "optional" in yaml_obj.ca.items[key][2].value.lower()
             )
+
             if isinstance(value, (dict, MutableMapping, list, MutableSequence)):
                 value = remove_non_required(value)
                 required = required or value
@@ -39,7 +56,9 @@ def remove_non_required(yaml_obj):
             comment = ""
             if yaml_obj.ca.items and key in yaml_obj.ca.items and yaml_obj.ca.items[key][2]:
                 comment = yaml_obj.ca.items[key][2].value
-            required = key in yaml_obj.ca.items and "required" in comment.lower()
+            required = key in yaml_obj.ca.items and (
+                "required" in comment.lower() or "optional" in comment.lower()
+            )
             if isinstance(value, (dict, MutableMapping, list, MutableSequence)):
                 value = remove_non_required(value)
                 required = required or value
@@ -49,4 +68,4 @@ def remove_non_required(yaml_obj):
                     result.yaml_add_eol_comment(yaml_obj.ca.items[key][2].value)
         return result
     else:
-        assert False, "Must be dict/list"
+        assert False, "Input must be either dict or list."

snappy_pipeline/apps/snappy_pull_sheet.py

@@ -47,7 +47,7 @@ def strip(x):
         return x
 
 
-def run(args):
+def run(args):  # noqa: C901
     # Query investigation JSON from API.
     url = URL_TPL % vars(args)
     print("Fetching %s" % url, file=sys.stderr)
@@ -63,7 +63,6 @@ def run(args):
     study_top = study_infos["top_header"]
     n_source = study_top[0]["colspan"]
     n_extraction = study_top[1]["colspan"]
-    n_sample = study_top[2]["colspan"]
     cols_source = study_infos["field_header"][:n_source]
     cols_extraction = study_infos["field_header"][n_source : n_source + n_extraction]
     cols_sample = study_infos["field_header"][n_source + n_extraction :]
@@ -129,7 +128,7 @@ def run(args):
     # Generate the resulting sample sheet.
     print("\n".join(HEADER_TPL) % vars(args), file=args.output)
     for source, info in study_map.items():
-        if not source in assay_map:
+        if source not in assay_map:
             print("INFO: source %s does not have an assay." % source, file=sys.stderr)
             dict_lib = {"Name": "-.1", "Folder Name": ".", "Batch": "."}  # HAAACKY
             proc_lib = {}

snappy_pipeline/apps/snappy_snake.py

@@ -151,7 +151,7 @@ def binary_available(name):
     return retcode == 0
 
 
-def run(wrapper_args):
+def run(wrapper_args):  # noqa: C901
     """Launch the CUBI Pipeline wrapper for the given arguments"""
     # Setup logging
     setup_logging(wrapper_args)

snappy_pipeline/apps/snappy_start_step.py

@@ -172,9 +172,11 @@ def _setup_configuration(self, config_yaml):
         yaml = ruamel_yaml.YAML()
         buf = io.StringIO()
         yaml.dump(config_yaml, stream=buf)
+        buf.seek(0)
+        contents = buf.read()
         update_file(
             path=config_filename,
-            contents=str(buf),
+            contents=contents,
             message="Updating project config with required default config for step {step}",
             message_args={"step": self.step},
         )

snappy_pipeline/find_file.py

@@ -177,7 +177,7 @@ def run(self, root_dir, dir_name, pattern_sets, allow_empty_right):
                 else:
                     returned = len(set_matches)
                 # Must have the same number of matches for each pattern
-                lst_lens = [len(l) for l in set_matches.values()]
+                lst_lens = [len(i) for i in set_matches.values()]
                 if len(set(lst_lens)) != 1:
                     raise ValueError(  # pragma: no cover
                         "Must have the same number of matches per pattern, but found {}".format(