Skip to content
Permalink
Browse files
feat: support for post deploy scripts (#1325)
* feat: support for post deploy scripts (skeleton)

* fixes

* fmt with black

* Add info message

* Add documentation

* move code

* Moved to method

* Fix variable

* Add test

* Add result

* Add missing folder

* Fix name

* Fixed typo

* Update docs

* write to tmp

* fix patho

* transform to path

Co-authored-by: Felix Mölder <felix.moelder@uni-due.de>
  • Loading branch information
johanneskoester and FelixMoelder committed Jan 14, 2022
1 parent 9fe8060 commit e5dac4ff297b7aeeb1e1a0bbdd03cb967cee3011
Show file tree
Hide file tree
Showing 7 changed files with 122 additions and 4 deletions.
@@ -262,6 +262,36 @@ Conda deployment also works well for offline or air-gapped environments. Running

.. _singularity:


-------------------------
Providing post-deployment scripts
-------------------------

From Snakemake 6.14 onwards post-deployment shell-scripts can be provided to perform additional adjustments of a conda environment.
This might be helpful in case a conda package is missing components or requires further configuration for execution.
Post-deployment scripts must be placed next to their corresponding environment-file and require the suffix ``.post-deploy.sh``, e.g.:

.. code-block:: python
rule NAME:
input:
"seqs.fastq"
output:
"results.tsv"
conda:
"envs/interproscan.yaml"
shell:
"interproscan.sh -i {input} -f tsv -o {output}"
.. code-block:: none
├── Snakefile
└── envs
├── interproscan.yaml
└── interproscan.post-deploy.sh
The path of the conda environment can be accessed within the script via ``$CONDA_PREFIX``.

--------------------------
Running jobs in containers
--------------------------
@@ -4,6 +4,7 @@
__license__ = "MIT"

import os
from pathlib import Path
import re
from snakemake.sourcecache import LocalGitFile, LocalSourceFile, infer_source_file
import subprocess
@@ -60,6 +61,7 @@ def __init__(
self._hash = None
self._content_hash = None
self._content = None
self._content_deploy = None
self._path = None
self._archive_file = None
self._cleanup = cleanup
@@ -68,6 +70,10 @@ def __init__(
def _get_content(self):
return self.workflow.sourcecache.open(self.file, "rb").read()

def _get_content_deploy(self):
deploy_file = Path(self.file).with_suffix(".post-deploy.sh")
return self.workflow.sourcecache.open(deploy_file, "rb").read()

@property
def _env_archive_dir(self):
return self.workflow.persistence.conda_env_archive_path
@@ -82,6 +88,12 @@ def content(self):
self._content = self._get_content()
return self._content

@property
def content_deploy(self):
if self._content_deploy is None:
self._content_deploy = self._get_content_deploy()
return self._content_deploy

@property
def hash(self):
if self._hash is None:
@@ -211,13 +223,36 @@ def create_archive(self):
raise e
return env_archive

def execute_deployment_script(self, env_file, deploy_file):
"""Execute post-deployment script if present"""
from snakemake.shell import shell

if ON_WINDOWS:
raise WorkflowError(
"Post deploy script {} provided for conda env {} but unsupported on windows.".format(
deploy_file, env_file
)
)
logger.info(
"Running post-deploy script {}...".format(
Path(deploy_file).relative_to(os.getcwd())
)
)
conda = Conda(self._container_img)
shell.check_output(
conda.shellcmd(self.path, "sh {}".format(deploy_file)),
stderr=subprocess.STDOUT,
)

def create(self, dryrun=False):
"""Create the conda enviroment."""
from snakemake.shell import shell

# Read env file and create hash.
env_file = self.file
tmp_file = None
deploy_file = None
tmp_env_file = None
tmp_deploy_file = None

if not isinstance(env_file, LocalSourceFile) or isinstance(
env_file, LocalGitFile
@@ -226,9 +261,23 @@ def create(self, dryrun=False):
# write to temp file such that conda can open it
tmp.write(self.content)
env_file = tmp.name
tmp_file = tmp.name
tmp_env_file = tmp.name
if (
Path(self.file.get_path_or_uri())
.with_suffix(".post-deploy.sh")
.exists()
):
with tempfile.NamedTemporaryFile(
delete=False, suffix=".post-deploy.sh"
) as tmp:
# write to temp file such that conda can open it
tmp.write(self.content_deploy)
deploy_file = tmp.name
tmp_deploy_file = tmp.name
else:
env_file = env_file.get_path_or_uri()
if Path(env_file).with_suffix(".post-deploy.sh").exists():
deploy_file = Path(env_file).with_suffix(".post-deploy.sh")

env_hash = self.hash
env_path = self.path
@@ -374,6 +423,13 @@ def create(self, dryrun=False):
"Cleaning up conda package tarballs and package cache."
)
shell.check_output("conda clean -y --tarballs --packages")

# Execute post-deplay script if present
if deploy_file:
target_deploy_file = env_path + ".post-deploy.sh"
shutil.copy(deploy_file, target_deploy_file)
self.execute_deployment_script(env_file, target_deploy_file)

# Touch "done" flag file
with open(os.path.join(env_path, "env_setup_done"), "a") as f:
pass
@@ -392,9 +448,11 @@ def create(self, dryrun=False):
+ e.output
)

if tmp_file:
if tmp_env_file:
# temporary file was created
os.remove(tmp_file)
os.remove(tmp_env_file)
if tmp_deploy_file:
os.remove(tmp_deploy_file)

return env_path

@@ -0,0 +1,17 @@
shell.executable("bash")

rule all:
input:
"test.out"

rule a:
output:
"test.out"
conda:
"test-env.yaml"
shell:
"""
if [ -f $CONDA_PREFIX/test.txt ] ;then
touch {output}
fi
"""
Empty file.
@@ -0,0 +1,3 @@
#!/bin/bash

touch $CONDA_PREFIX/test.txt
@@ -0,0 +1,5 @@
channels:
- bioconda
- conda-forge
dependencies:
- python <3.10
@@ -429,6 +429,11 @@ def test_upstream_conda():
run(dpath("test_conda"), use_conda=True, conda_frontend="conda")


@skip_on_windows
def test_deploy_script():
run(dpath("test_deploy_script"), use_conda=True)


def test_conda_custom_prefix():
run(
dpath("test_conda_custom_prefix"),

0 comments on commit e5dac4f

Please sign in to comment.