Skip to content
Permalink
Browse files
fix: env file usage after changes to source file handling (inspired by
…#1233 and #1211). (#1236)

* fix: more informative nothing to be done message

* fmt

* fmt, fixes and additional tests

* implement fspath
  • Loading branch information
johanneskoester committed Oct 26, 2021
1 parent 368d265 commit 3ac8e858a7b908326922c8f68cae512b1250e906
Show file tree
Hide file tree
Showing 5 changed files with 62 additions and 42 deletions.
@@ -7,6 +7,7 @@
from snakemake.exceptions import WorkflowError
from snakemake.deployment import conda
from snakemake.logging import logger
from snakemake.sourcecache import LocalSourceFile


CONDA_ENV_PATH = "/conda-envs"
@@ -22,7 +23,11 @@ def containerize(workflow):
"Containerization of conda based workflows is not allowed if any conda env definition contains a wildcard."
)

relfile = lambda env: os.path.relpath(env.file, os.getcwd())
def relfile(env):
if isinstance(env.file, LocalSourceFile):
return os.path.relpath(env.file.get_path_or_uri(), os.getcwd())
else:
return env.file.get_path_or_uri()

envs = sorted(
set(
@@ -59,11 +64,12 @@ def containerize(workflow):
"\n".join(map("# {}".format, env.content.decode().strip().split("\n")))
)
get_env_cmds.append("RUN mkdir -p {}".format(prefix))
if env.file.startswith("https://"):
# get_env_cmds.append("RUN curl -sSL {} > {}".format(env.file, env_target_path))
get_env_cmds.append("ADD {} {}".format(env.file, env_target_path))
else:
if isinstance(env.file, LocalSourceFile):
get_env_cmds.append("COPY {} {}".format(env_source_path, env_target_path))
else:
get_env_cmds.append(
"ADD {} {}".format(env.file.get_path_or_uri(), env_target_path)
)

generate_env_cmds.append(
"mamba env create --prefix {} --file {} &&".format(prefix, env_target_path)
@@ -105,6 +105,9 @@ def isabs(self):
def simplify_path(self):
return utils.simplify_path(self.path)

def __fspath__(self):
return self.path


class LocalGitFile(SourceFile):
def __init__(
@@ -958,7 +958,7 @@ def files(items):
for env in set(job.conda_env for job in dag.jobs):
if env:
print(
simplify_path(env.file),
env.file.simplify_path(),
env.container_img_url or "",
simplify_path(env.path),
sep="\t",
@@ -88,6 +88,7 @@ def run(
subpath=None,
no_tmpdir=False,
check_md5=True,
check_results=True,
cores=3,
set_pythonpath=True,
cleanup=True,
@@ -181,42 +182,43 @@ def run(
assert not success, "expected error on execution"
else:
assert success, "expected successful execution"
for resultfile in get_expected_files(results_dir):
if resultfile in [".gitignore", ".gitkeep"] or not os.path.isfile(
os.path.join(results_dir, resultfile)
):
# this means tests cannot use directories as output files
continue
targetfile = join(tmpdir, resultfile)
expectedfile = join(results_dir, resultfile)

if ON_WINDOWS:
if os.path.exists(join(results_dir, resultfile + "_WIN")):
continue # Skip test if a Windows specific file exists
if resultfile.endswith("_WIN"):
targetfile = join(tmpdir, resultfile[:-4])
elif resultfile.endswith("_WIN"):
# Skip win specific result files on Posix platforms
continue

assert os.path.exists(targetfile), 'expected file "{}" not produced'.format(
resultfile
)
if check_md5:
md5expected = md5sum(expectedfile, ignore_newlines=ON_WINDOWS)
md5target = md5sum(targetfile, ignore_newlines=ON_WINDOWS)
if md5target != md5expected:
with open(expectedfile) as expected:
expected_content = expected.read()
with open(targetfile) as target:
content = target.read()
assert (
False
), "wrong result produced for file '{resultfile}':\n------found------\n{content}\n-----expected-----\n{expected_content}\n-----------------".format(
resultfile=resultfile,
content=content,
expected_content=expected_content,
)
if check_results:
for resultfile in get_expected_files(results_dir):
if resultfile in [".gitignore", ".gitkeep"] or not os.path.isfile(
os.path.join(results_dir, resultfile)
):
# this means tests cannot use directories as output files
continue
targetfile = join(tmpdir, resultfile)
expectedfile = join(results_dir, resultfile)

if ON_WINDOWS:
if os.path.exists(join(results_dir, resultfile + "_WIN")):
continue # Skip test if a Windows specific file exists
if resultfile.endswith("_WIN"):
targetfile = join(tmpdir, resultfile[:-4])
elif resultfile.endswith("_WIN"):
# Skip win specific result files on Posix platforms
continue

assert os.path.exists(
targetfile
), 'expected file "{}" not produced'.format(resultfile)
if check_md5:
md5expected = md5sum(expectedfile, ignore_newlines=ON_WINDOWS)
md5target = md5sum(targetfile, ignore_newlines=ON_WINDOWS)
if md5target != md5expected:
with open(expectedfile) as expected:
expected_content = expected.read()
with open(targetfile) as target:
content = target.read()
assert (
False
), "wrong result produced for file '{resultfile}':\n------found------\n{content}\n-----expected-----\n{expected_content}\n-----------------".format(
resultfile=resultfile,
content=content,
expected_content=expected_content,
)

if not cleanup:
return tmpdir
@@ -409,6 +409,10 @@ def test_conda():
run(dpath("test_conda"), use_conda=True)


def test_conda_list_envs():
run(dpath("test_conda"), list_conda_envs=True, check_results=False)


def test_upstream_conda():
run(dpath("test_conda"), use_conda=True, conda_frontend="conda")

@@ -1230,6 +1234,11 @@ def test_containerized():
run(dpath("test_containerized"), use_conda=True, use_singularity=True)


@skip_on_windows
def test_containerize():
run(dpath("test_conda"), containerize=True, check_results=False)


def test_long_shell():
run(dpath("test_long_shell"))

0 comments on commit 3ac8e85

Please sign in to comment.