feat: make it possible to exclude rules that will be imported when us…

…ing 'use rule' statement (#1717)

* feat: make it possible to exclude rules that will be imported when using 'use rule' statement

* docs: update documentation

* style: make black formatting happy

docs/snakefiles/modularization.rst

@@ -130,11 +130,11 @@ With Snakemake 6.0 and later, it is possible to define external workflows as mod
             # here, plain paths, URLs and the special markers for code hosting providers (see below) are possible.
             "other_workflow/Snakefile"
     
-    use rule * from other_workflow as other_*
+    use rule * from other_workflow exclude ruleC as other_*
 
 The ``module other_workflow:`` statement registers the external workflow as a module, by defining the path to the main snakefile of ``other_workflow``.
 Here, plain paths, HTTP/HTTPS URLs and special markers for code hosting providers like Github or Gitlab are possible (see :ref:`snakefile-code-hosting-providers`).
-The second statement, ``use rule * from other_workflow as other_*``, declares all rules of that module to be used in the current one.
+The second statement, ``use rule * from other_workflow exclude ruleC as other_*``, declares all rules of that module to be used in the current one, except for ruleC.
 Thereby, the ``as other_*`` at the end renames all those rules with a common prefix.
 This can be handy to avoid rule name conflicts (note that rules from modules can otherwise overwrite rules from your current workflow or other modules).
 

snakemake/modules.py

@@ -73,6 +73,7 @@ def use_rules(
         self,
         rules=None,
         name_modifier=None,
+        exclude_rules=None,
         ruleinfo=None,
         skip_global_report_caption=False,
     ):
@@ -84,6 +85,7 @@ def use_rules(
             skip_configfile=self.config is not None,
             skip_validation=self.skip_validation,
             skip_global_report_caption=skip_global_report_caption,
+            rule_exclude_list=exclude_rules,
             rule_whitelist=self.get_rule_whitelist(rules),
             rulename_modifier=get_name_modifier_func(rules, name_modifier),
             ruleinfo_overwrite=ruleinfo,
@@ -140,6 +142,7 @@ def __init__(
         skip_global_report_caption=False,
         rulename_modifier=None,
         rule_whitelist=None,
+        rule_exclude_list=None,
         ruleinfo_overwrite=None,
         allow_rule_overwrite=False,
         replace_prefix=None,
@@ -156,6 +159,7 @@ def __init__(
             self.skip_validation = parent_modifier.skip_validation
             self.skip_global_report_caption = parent_modifier.skip_global_report_caption
             self.rule_whitelist = parent_modifier.rule_whitelist
+            self.rule_exclude_list = parent_modifier.rule_exclude_list
             self.ruleinfo_overwrite = parent_modifier.ruleinfo_overwrite
             self.allow_rule_overwrite = parent_modifier.allow_rule_overwrite
             self.path_modifier = parent_modifier.path_modifier
@@ -178,14 +182,17 @@ def __init__(
         self.skip_validation = skip_validation
         self.skip_global_report_caption = skip_global_report_caption
         self.rule_whitelist = rule_whitelist
+        self.rule_exclude_list = rule_exclude_list
         self.ruleinfo_overwrite = ruleinfo_overwrite
         self.allow_rule_overwrite = allow_rule_overwrite
         self.path_modifier = PathModifier(replace_prefix, prefix, workflow)
         self.replace_wrapper_tag = replace_wrapper_tag
         self.namespace = namespace
 
     def skip_rule(self, rulename):
-        return self.rule_whitelist is not None and rulename not in self.rule_whitelist
+        return (
+            self.rule_whitelist is not None and rulename not in self.rule_whitelist
+        ) or (self.rule_exclude_list is not None and rulename in self.rule_exclude_list)
 
     def modify_rulename(self, rulename):
         if self.rulename_modifier is not None:

snakemake/parser.py

@@ -947,6 +947,7 @@ def __init__(self, snakefile, base_indent=0, dedent=0, root=True):
         super().__init__(snakefile, base_indent=base_indent, dedent=dedent, root=root)
         self.state = self.state_keyword_rule
         self.rules = []
+        self.exclude_rules = []
         self.has_with = False
         self.name_modifier = []
         self.from_module = None
@@ -955,8 +956,8 @@ def __init__(self, snakefile, base_indent=0, dedent=0, root=True):
 
     def end(self):
         name_modifier = "".join(self.name_modifier) if self.name_modifier else None
-        yield "@workflow.userule(rules={!r}, from_module={!r}, name_modifier={!r}, lineno={})".format(
-            self.rules, self.from_module, name_modifier, self.lineno
+        yield "@workflow.userule(rules={!r}, from_module={!r}, exclude_rules={!r}, name_modifier={!r}, lineno={})".format(
+            self.rules, self.from_module, self.exclude_rules, name_modifier, self.lineno
         )
         yield "\n"
 
@@ -1060,6 +1061,9 @@ def state_modifier(self, token):
             if token.string == "as" and not self.name_modifier:
                 self.state = self.state_as
                 yield from ()
+            elif token.string == "exclude":
+                self.state = self.state_exclude
+                yield from ()
             elif token.string == "with":
                 yield from self.handle_with(token)
             else:
@@ -1111,9 +1115,43 @@ def state_with(self, token):
             yield from ()
         else:
             self.error(
-                "Expecting colon after 'with' keyword in 'use rule' statement.", token
+                "Expecting colon after 'with' keyword in 'use rule' statement.",
+                token,
+            )
+
+    def state_exclude(self, token):
+        if is_name(token):
+            self.exclude_rules.append(token.string)
+            self.state = self.state_exclude_comma_or_end
+            yield from ()
+        else:
+            self.error(
+                "Expecting rule name(s) after 'exclude' keyword in 'use rule' statement.",
+                token,
             )
 
+    def state_exclude_comma_or_end(self, token):
+        if is_name(token):
+            if token.string == "from" or token.string == "as":
+                if not self.exclude_rules:
+                    self.error(
+                        "Expecting rule names after 'exclude' statement.",
+                        token,
+                    )
+                if token.string == "from":
+                    self.state = self.state_from
+                else:
+                    self.state = self.state_as
+                yield from ()
+            else:
+                yield from ()
+        elif is_comma(token):
+            self.state = self.state_exclude
+            yield from ()
+        else:
+            self.state = self.state_modifier
+            yield from ()
+
     def block_content(self, token):
         if is_comment(token):
             yield "\n", token

snakemake/workflow.py

@@ -1941,7 +1941,14 @@ def module(
             prefix=prefix,
         )
 
-    def userule(self, rules=None, from_module=None, name_modifier=None, lineno=None):
+    def userule(
+        self,
+        rules=None,
+        from_module=None,
+        exclude_rules=None,
+        name_modifier=None,
+        lineno=None,
+    ):
         def decorate(maybe_ruleinfo):
             if from_module is not None:
                 try:
@@ -1955,6 +1962,7 @@ def decorate(maybe_ruleinfo):
                 module.use_rules(
                     rules,
                     name_modifier,
+                    exclude_rules=exclude_rules,
                     ruleinfo=None if callable(maybe_ruleinfo) else maybe_ruleinfo,
                     skip_global_report_caption=self.report_text
                     is not None,  # do not overwrite existing report text via module

tests/test_modules_all_exclude/Snakefile

@@ -0,0 +1,17 @@
+shell.executable("bash")
+
+configfile: "config/config.yaml"
+
+
+module test:
+    snakefile:
+        "module-test/Snakefile"
+    config:
+        config
+    replace_prefix:
+        {"results/": "results/testmodule/"}
+
+
+use rule * from test as test_*
+
+assert test.some_func() == 15

tests/test_modules_all_exclude/Snakefile_exclude

@@ -0,0 +1,17 @@
+shell.executable("bash")
+
+configfile: "config/config.yaml"
+
+
+module test:
+    snakefile:
+        "module-test/Snakefile"
+    config:
+        config
+    replace_prefix:
+        {"results/": "results/testmodule/"}
+
+
+use rule * from test exclude b, d
+
+assert test.some_func() == 15

tests/test_modules_all_exclude/config/config.yaml

@@ -0,0 +1,4 @@
+test: 1
+testb: 2
+testc: 3
+testd: 4

tests/test_modules_all_exclude/expected-results/results/testmodule/test.out

@@ -0,0 +1 @@
+1

tests/test_modules_all_exclude/expected-results/results/testmodule/test2.out

@@ -0,0 +1 @@
+3

tests/test_modules_all_exclude/module-test/Snakefile

@@ -0,0 +1,35 @@
+configfile: "config.yaml" # does not exist, but this statement should be ignored on module import
+
+
+def some_func():
+    return 15
+
+
+rule all:
+    input:
+        "results/test.out", "results/test2.out"
+
+
+rule a:
+    output:
+        "results/test.out"
+    shell:
+        "echo {config[test]} > {output}"
+
+rule b:
+    output:
+        "results/test2.out"
+    shell:
+        "echo {config[testb]} > {output}"
+
+rule c:
+    output:
+        "results/test2.out"
+    shell:
+        "echo {config[testc]} > {output}"
+
+rule d:
+    output:
+        "results/test2.out"
+    shell:
+        "echo {config[testc]} > {output}"

tests/tests.py

@@ -1378,6 +1378,20 @@ def test_modules_all():
     run(dpath("test_modules_all"), targets=["a"])
 
 
+def test_modules_all_exclude_1():
+    # Fail due to conflicting rules
+    run(dpath("test_modules_all_exclude"), shouldfail=True)
+
+
+def test_modules_all_exclude_2():
+    # Successed since the conflicting rules have been excluded
+    run(
+        dpath("test_modules_all_exclude"),
+        snakefile="Snakefile_exclude",
+        shouldfail=False,
+    )
+
+
 @skip_on_windows
 def test_modules_prefix():
     run(dpath("test_modules_prefix"), targets=["a"])