Fix SequanaConfig class (regression bug).

.github/workflows/main.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       max-parallel: 5
       matrix:
-        python: [3.7,3.8]
+        python: [3.7,3.8,3.9]
 
     steps:
     - name: checkout git repo

README.rst

@@ -170,6 +170,9 @@ Changelog
 ========= ====================================================================
 Version   Description
 ========= ====================================================================
+0.6.3     * Fix SequanaConfig file
+0.6.2     * Fix script creation to include wrapper and take new snakemake 
+            syntax into account
 0.6.1     * update schema handling
 0.6.0     * Move all modules related to pipelines rom sequana into 
             sequana_pipetools; This release should now be the entry point for 

sequana_pipetools/__init__.py

@@ -15,5 +15,6 @@
 
 logger = colorlog.getLogger(logger.name)
 
-from .snaketools import Module, SequanaConfig, PipelineManagerGeneric, PipelineManager
+from .snaketools import Module, SequanaConfig, PipelineManagerGeneric, PipelineManager, PipelineManagerDirectory
 from .sequana_manager import SequanaManager, get_pipeline_location
+

sequana_pipetools/completion.py

@@ -14,6 +14,7 @@
 import os
 import argparse
 import pkgutil
+from pkg_resources import DistributionNotFound
 import importlib
 
 
@@ -217,9 +218,12 @@ def main(args=None):
     if choice == "y":
         print("Please source the files using:: \n")
         for name in names:
-            c = Complete(name)
-            c.save_completion_script()
-            print("source ~/.config/sequana/pipelines/{}.sh".format(name))
+            try:
+                c = Complete(name)
+                c.save_completion_script()
+                print("source ~/.config/sequana/pipelines/{}.sh".format(name))
+            except DistributionNotFound:
+                print(f"# Warning {name} could not be imported. Nothing done")
         print("\nto activate the completion")
     else:  # pragma: no cover
         print("Stopping creation of completion scripts")

sequana_pipetools/snaketools/__init__.py

@@ -2,7 +2,7 @@
 from .dot_parser import DOTParser
 from .module import Module, modules, pipeline_names
 from .module_finder import ModuleFinder
-from .pipeline_manager import PipelineManager, PipelineManagerGeneric
+from .pipeline_manager import PipelineManager, PipelineManagerGeneric, PipelineManagerDirectory
 from .pipeline_utils import (message, OnSuccessCleaner, OnSuccess, build_dynamic_rule, get_pipeline_statistics,
                              create_cleanup, Makefile)
 from .file_factory import FileFactory, FastQFactory

sequana_pipetools/snaketools/pipeline_manager.py

@@ -156,6 +156,7 @@ def clean_multiqc(self, filename):
     def teardown(self, extra_dirs_to_remove=[], extra_files_to_remove=[], plot_stats=True):
         # create and save the stats plot
         if plot_stats:
+            logger.warning("deprecated no stats to be provided in the future")
             N = len(self.samples.keys())
             self.plot_stats(N=N)
 

sequana_pipetools/snaketools/sequana_config.py

@@ -20,9 +20,10 @@
 from pykwalify.core import Core, CoreError, SchemaError
 import ruamel.yaml
 
+
 try:
     import importlib.resources as pkg_resources
-except ImportError:
+except ImportError:  # pragma: no cover
     # Try backported to PY<37 `importlib_resources`.
     import importlib_resources as pkg_resources
 
@@ -75,39 +76,45 @@ def __init__(self, data=None):
             except TypeError:
                 if hasattr(data, "config"):
                     self.config = AttrDict(**data.config)
-                    self._yaml_code = ruamel.yaml.comments.CommentedMap(self.config.copy())
+                    self._yaml_code = ruamel.yaml.comments.CommentedMap(
+                        self.config.copy()
+                    )
                 else:
-                    self.config = self._read_file(data)
+                    # populate self._yaml_code
+                    config = self._read_file(data)
+                    self.config = AttrDict(**config)
         else:
             self.config = AttrDict()
             self._yaml_code = ruamel.yaml.comments.CommentedMap()
-        self.cleanup_config()
+
+        # remove templates and None->""
+        self._recursive_cleanup(self.config)
 
     def _read_file(self, data):
-        """Read yaml or json file"""
-        try:
-            is_yml = data.endswith((".yaml", ".yml"))
-        except AttributeError:
-            raise IOError("Data param does not have the good format. It's a filename or a dict")
+        """Read yaml"""
+        if not data.endswith((".yaml", ".yml")):
+            logger.warning("You should use a YAML file with .yaml or .yml extension")
 
         if os.path.exists(data):
             yaml = ruamel.yaml.YAML()
-            if is_yml:
-                with open(data, "r") as fh:
-                    self._yaml_code = yaml.load(fh.read())
-            else:
-                # read a JSON
-                with open(data, "r") as fh:
-                    self._yaml_code = yaml.load(json.dumps(json.loads(fh.read())))
+            with open(data, "r") as fh:
+                self._yaml_code = yaml.load(fh.read())
+
+            # import the Python yaml module to avoid ruamel.yaml ordereddict and
+            # other structures. We only want the data
+            with open(data, "r") as fh:
+                import yaml as _yaml
+
+                config = _yaml.load(fh, Loader=_yaml.FullLoader)
+            return config
         else:
             raise FileNotFoundError(f"input string must be an existing file {data}")
-        return AttrDict(**self._yaml_code.copy())
 
-    def save(self, filename="config.yaml", cleanup=True):
+    def save(self, filename="config.yaml"):
         """Save the yaml code in _yaml_code with comments"""
-        # This works only if the input data was a yaml
-        if cleanup:
-            self.cleanup()  # changes the config and yaml_code to remove %()s
+        # make sure that the changes made in the config are saved into the yaml
+        # before saving it
+        self._update_yaml()
 
         # get the YAML formatted code and save it
         yaml = ruamel.yaml.YAML()
@@ -124,8 +131,8 @@ def _recursive_update(self, target, data):
 
         # !! essential to use the update() method of the dictionary otherwise
         # comments are lost
-
         for key, value in data.items():
+
             if isinstance(value, dict):
                 target.update({key: self._recursive_update(target[key], value)})
             elif isinstance(value, list):
@@ -141,10 +148,12 @@ def _recursive_update(self, target, data):
                     value = data[key]
                     target.update({key: value})
                 else:
-                    logger.warning("This %s key was not in the original config" " but added" % key)
+                    logger.warning(
+                        "This %s key was not in the original config" " but added" % key
+                    )
                     value = data[key]
                     target.update({key: value})
-            else:
+            else:  # pragma: no cover
                 raise NotImplementedError(
                     "Only dictionaries and list are authorised in the input configuration file."
                     f" Key/value that cause error are {key}/{value}"
@@ -154,11 +163,6 @@ def _recursive_update(self, target, data):
     def _update_yaml(self):
         self._recursive_update(self._yaml_code, self.config)
 
-    def _update_config(self):
-        # probably useless function now since we do not use json as input
-        # anymore
-        self._recursive_update(self.config, self._yaml_code)
-
     def _recursive_cleanup(self, d):
         # expand the tilde (see https://github.com/sequana/sequana/issues/486)
         # remove the %() templates
@@ -169,38 +173,30 @@ def _recursive_cleanup(self, d):
                 if value is None:
                     d[key] = ""
                 elif isinstance(value, str):
-                    if value.startswith("%("):
-                        d[key] = None
-                    else:
-                        d[key] = value.strip()
+                    d[key] = value.strip()
+
                     # https://github.com/sequana/sequana/issues/486
                     if key.endswith("_directory") and value.startswith("~/"):
                         d[key] = os.path.expanduser(value)
                     if key.endswith("_file") and value.startswith("~/"):
                         d[key] = os.path.expanduser(value)
 
-    def cleanup_config(self):
-        self._recursive_cleanup(self.config)
-        # self._update_yaml()
-
-    def cleanup(self):
-        """Remove template elements and change None to empty string."""
-        self._recursive_cleanup(self._yaml_code)
-        self._update_config()
+        self._update_yaml()
 
     def copy_requirements(self, target):
         """Copy files to run the pipeline
 
         If a requirement file exists, it is copied in the target directory.
         If not, it can be either an http resources or a sequana resources.
         """
-
+        # make sure that if the config changed, the yaml is up-to-date
+        self._update_yaml()
         if "requirements" in self._yaml_code.keys():
             for requirement in self._yaml_code["requirements"]:
                 if os.path.exists(requirement):
                     try:
                         shutil.copy(requirement, target)
-                    except shutil.SameFileError:
+                    except shutil.SameFileError: #pragma: no cover
                         pass  # the target and input may be the same
                 elif requirement.startswith("http"):
                     logger.info(f"This file {requirement} will be needed. Downloading")
@@ -214,15 +210,19 @@ def check_config_with_schema(self, schemafile):
 
         """
         # add custom extensions
-        with pkg_resources.path('sequana_pipetools.resources', 'ext.py') as ext_name:
+        with pkg_resources.path("sequana_pipetools.resources", "ext.py") as ext_name:
             extensions = [str(ext_name)]
         # causes issue with ruamel.yaml 0.12.13. Works for 0.15
         warnings.simplefilter("ignore", ruamel.yaml.error.UnsafeLoaderWarning)
         try:
             # open the config and the schema file
             with TempFile(suffix=".yaml") as fh:
                 self.save(fh.name)
-                c = Core(source_file=fh.name, schema_files=[schemafile], extensions=extensions)
+                c = Core(
+                    source_file=fh.name,
+                    schema_files=[schemafile],
+                    extensions=extensions,
+                )
                 c.validate()
                 return True
         except (SchemaError, CoreError) as err:

setup.py

@@ -3,7 +3,7 @@
 
 _MAJOR = 0
 _MINOR = 6
-_MICRO = 2
+_MICRO = 3
 version = f"{_MAJOR}.{_MINOR}.{_MICRO}"
 release = f"{_MAJOR}.{_MINOR}"
 
@@ -22,9 +22,9 @@
         "Intended Audience :: Science/Research",
         "License :: OSI Approved :: BSD License",
         "Operating System :: OS Independent",
-        "Programming Language :: Python :: 3.5",
-        "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
         "Topic :: Software Development :: Libraries :: Python Modules",
         "Topic :: Scientific/Engineering :: Bio-Informatics",
         "Topic :: Scientific/Engineering :: Information Analysis",

tests/snaketools/test_pipeline_manager.py

@@ -17,20 +17,17 @@ def test_pipeline_manager(tmpdir):
     # normal behaviour but no input provided:
     config = Module("fastqc")._get_config()
     cfg = SequanaConfig(config)
-    cfg.cleanup()  # remove templates
     with pytest.raises(ValueError):
         pm = snaketools.PipelineManager("custom", cfg)
 
     # normal behaviour
     cfg = SequanaConfig(config)
-    cfg.cleanup()  # remove templates
     file1 = os.path.join(test_dir, "data", "Hm2_GTGAAA_L005_R1_001.fastq.gz")
     cfg.config.input_directory, cfg.config.input_pattern = os.path.split(file1)
     pm = snaketools.PipelineManager("custom", cfg)
     assert not pm.paired
 
     cfg = SequanaConfig(config)
-    cfg.cleanup()  # remove templates
     cfg.config.input_directory, cfg.config.input_pattern = os.path.split(file1)
     cfg.config.input_pattern = "Hm*gz"
     pm = snaketools.PipelineManager("custom", cfg)
@@ -130,7 +127,12 @@ class WF:
     wf = WF()
     gg["workflow"] = wf
     pm.setup(gg)
-    pm.teardown()
+    try:
+        pm.teardown()
+    except Exception:
+        assert False
+    finally:
+        os.remove("Makefile")
 
     multiqc = tmpdir.join('multiqc.html')
     with open(multiqc, 'w') as fh: