Skip to content
Permalink
Browse files
fix: more robust handling of incompletely evaluated parameters (any i…
…nteraction with them will result in a string <TBD> now). (#1525)

* fix: catch any errors when formatting job information

* Detect incomplete params evaluation and avoid printing the shell command in such a case

* More robust TBD handling, working for any interaction with the <TBD> object.
  • Loading branch information
johanneskoester committed Mar 28, 2022
1 parent f9cbc1e commit 3d4c768aafbdca67a9032ad9e3b73449a1fadb0d
Show file tree
Hide file tree
Showing 6 changed files with 146 additions and 10 deletions.
@@ -16,13 +16,14 @@
from pathlib import Path

from snakemake._version import get_versions
from snakemake.common.tbdstring import TBDString

__version__ = get_versions()["version"]
del get_versions


MIN_PY_VERSION = (3, 7)
DYNAMIC_FILL = "__snakemake_dynamic__"
DYNAMIC_FILL = "__othernakemake_dynamic__"
SNAKEMAKE_SEARCHPATH = str(Path(__file__).parent.parent.parent)
UUID_NAMESPACE = uuid.uuid5(uuid.NAMESPACE_URL, "https://snakemake.readthedocs.io")
NOTHING_TO_BE_DONE_MSG = (
@@ -47,13 +48,6 @@ def async_run(coroutine):
asyncio.create_task(coroutine)


# A string that prints as TBD
class TBDString(str):
# the second arg is necessary to avoid problems when pickling
def __new__(cls, _=None):
return str.__new__(cls, "<TBD>")


APPDIRS = None


@@ -178,7 +172,7 @@ class Mode:


class lazy_property(property):
__slots__ = ["method", "cached", "__doc__"]
__otherlots__ = ["method", "cached", "__doc__"]

@staticmethod
def clean(instance, method):
@@ -0,0 +1,99 @@
# A string that prints as TBD
# whatever interaction happens on this class, <TBD> shall be returned
class TBDString(str):
# the second arg is necessary to avoid problems when pickling
def __new__(cls, _=None):
return str.__new__(cls, "<TBD>")

def __getitem__(self, __item):
return self

def __getattribute__(self, __name):
return self

def __bool__(self):
return False

def __add__(self, __other):
return self

def __sub__(self, __other):
return self

def __mul__(self, __other):
return self

def __matmul__(self, __other):
return self

def __truediv__(self, __other):
return self

def __floordiv__(self, __other):
return self

def __mod__(self, __other):
return self

def __divmod__(self, __other):
return self

def __pow__(self, __other):
return self

def __lshift__(self, __other):
return self

def __rshift__(self, __other):
return self

def __and__(self, __other):
return self

def __xor__(self, __other):
return self

def __or__(self, __other):
return self

def __neg__(self):
return self

def __pos__(self):
return self

def __abs__(self):
return self

def __invert__(self):
return self

def __complex__(self):
return self

def __int__(self):
return self

def __float__(self):
return self

def __index__(self):
return self

def __round__(self, ndigits=0):
return self

def __trunc__(self):
return self

def __floor__(self):
return self

def __ceil__(self):
return self

def __enter__(self):
return self

def __exit__(self, __exc_type, __exc_value, __traceback):
return self
@@ -24,7 +24,11 @@
wait_for_files,
)
from snakemake.utils import format, listfiles
from snakemake.exceptions import RuleException, ProtectedOutputException, WorkflowError
from snakemake.exceptions import (
RuleException,
ProtectedOutputException,
WorkflowError,
)
from snakemake.logging import logger
from snakemake.common import (
DYNAMIC_FILL,
@@ -904,6 +908,10 @@ def format_wildcards(self, string, **variables):
raise RuleException("NameError: " + str(ex), rule=self.rule)
except IndexError as ex:
raise RuleException("IndexError: " + str(ex), rule=self.rule)
except Exception as ex:
raise WorkflowError(
f"Error when formatting '{string}' for rule {self.rule.name}. {ex}"
)

def properties(self, omit_resources=["_cores", "_nodes"], **aux_properties):
resources = {
@@ -1469,6 +1477,10 @@ def format_wildcards(self, string, **variables):
raise WorkflowError(
"IndexError with group job {}: {}".format(self.jobid, str(ex))
)
except Exception as ex:
raise WorkflowError(
f"Error when formatting {string} for group job {self.jobid}: {ex}"
)

@property
def threads(self):
@@ -0,0 +1,27 @@
def get_x(wildcards, input):
with open(input[0]) as infile:
return {"foo": infile.read()}


def get_mem_mb(wildcards, input):
return os.path.getsize(input[0]) / 1024.0


rule a:
input:
"test.in",
output:
"test.out",
params:
x=get_x,
resources:
mem_mb=get_mem_mb,
shell:
"echo {params.x[foo]} > {output}"


rule b:
output:
"test.in",
shell:
"touch {output}"
Empty file.
@@ -1561,3 +1561,7 @@ def test_groupid_expand_cluster():
@skip_on_windows
def test_service_jobs():
run(dpath("test_service_jobs"), check_md5=False)


def test_incomplete_params():
run(dpath("test_incomplete_params"), dryrun=True, printshellcmds=True)

0 comments on commit 3d4c768

Please sign in to comment.