fix: use base64 encoding when passing resources and threads to remote…

… jobs (this solves issues with complex quoted resources) (#2778)

fixes #2631.

### Description

<!--Add a description of your PR here-->

### QC
<!-- Make sure that you can tick the boxes below. -->

* [x] The PR contains a test case for the changes or the changes are
already covered by an existing test case.
* [x] The documentation (`docs/`) is updated to reflect the changes or
this is not necessary (e.g. if the change does neither modify the
language nor the behavior or functionalities of Snakemake).

setup.cfg

@@ -51,7 +51,7 @@ install_requires =
     requests >=2.8.1,<3.0
     reretry
     smart-open >=3.0,<8.0
-    snakemake-interface-executor-plugins >=9.0.2,<10.0
+    snakemake-interface-executor-plugins >=9.1.0,<10.0
     snakemake-interface-common >=1.17.0,<2.0
     snakemake-interface-storage-plugins >=3.1.0,<4.0
     snakemake-interface-report-plugins >=1.0.0,<2.0.0

snakemake/cli.py

@@ -13,7 +13,7 @@
 from typing import Set
 
 from snakemake_interface_executor_plugins.settings import ExecMode
-from snakemake_interface_executor_plugins.utils import is_quoted
+from snakemake_interface_executor_plugins.utils import is_quoted, maybe_base64
 from snakemake_interface_storage_plugins.registry import StoragePluginRegistry
 
 import snakemake.common.argparse
@@ -521,7 +521,7 @@ def get_argument_parser(profiles=None):
         metavar="RULE=THREADS",
         nargs="+",
         default=dict(),
-        parse_func=parse_set_threads,
+        parse_func=maybe_base64(parse_set_threads),
         help="Overwrite thread usage of rules. This allows to fine-tune workflow "
         "parallelization. In particular, this is helpful to target certain cluster nodes "
         "by e.g. shifting a rule to use more, or less threads than defined in the workflow. "
@@ -541,7 +541,7 @@ def get_argument_parser(profiles=None):
         metavar="RULE:RESOURCE=VALUE",
         nargs="+",
         default=dict(),
-        parse_func=parse_set_resources,
+        parse_func=maybe_base64(parse_set_resources),
         help="Overwrite resource usage of rules. This allows to fine-tune workflow "
         "resources. In particular, this is helpful to target certain cluster nodes "
         "by e.g. defining a certain partition for a rule, or overriding a temporary directory. "

snakemake/spawn_jobs.py

@@ -104,6 +104,7 @@ def get_orig_arg(value):
                     for name, value in res.items()
                 ],
                 skip=not self.workflow.resource_settings.overwrite_resources,
+                base64_encode=True,
             ),
             format_cli_arg(
                 "--set-threads",
@@ -112,6 +113,7 @@ def get_orig_arg(value):
                     for rule, value in self.workflow.resource_settings.overwrite_threads.items()
                 ],
                 skip=not self.workflow.resource_settings.overwrite_threads,
+                base64_encode=True,
             ),
         ]
 
@@ -149,7 +151,13 @@ def get_group_args(self):
         return join_cli_args([groups, group_components])
 
     def workflow_property_to_arg(
-        self, property, flag=None, quote=True, skip=False, invert=False, attr=None
+        self,
+        property,
+        flag=None,
+        base64_encode=False,
+        skip=False,
+        invert=False,
+        attr=None,
     ):
         if skip:
             return ""
@@ -171,7 +179,7 @@ def workflow_property_to_arg(
         if invert and isinstance(value, bool):
             value = not value
 
-        return format_cli_arg(flag, value, quote=quote)
+        return format_cli_arg(flag, value, base64_encode=base64_encode)
 
     def envvars(self) -> Mapping[str, str]:
         envvars = {

test-environment.yml

@@ -63,7 +63,7 @@ dependencies:
   - pip:
       - cwltool
       - cwl-utils
-      - snakemake-interface-executor-plugins >=9.0.2
+      - snakemake-interface-executor-plugins >=9.1.0
       - snakemake-executor-plugin-cluster-generic >=1.0.9
       - snakemake-storage-plugin-http
       - snakemake-storage-plugin-s3

tests/test_set_resources_complex/Snakefile

@@ -4,6 +4,6 @@ rule a:
     output:
         "test.out"
     run:
-        with open("test.in") as f:
-            with open("test.out", "w") as out:
-                print(resources.slurm_extra, file=out)
+        with open(input[0]) as f:
+            with open(output[0], "w") as out:
+                print(resources.slurm_extra, file=out)