Skip to content

Commit

Permalink
fix: fixed conda frontend detection and checking to also work with la…
Browse files Browse the repository at this point in the history 
…test mambaforge (#1781)

* fix: fixed conda frontend detection and checking to also work with latest mambaforge

* use conda for info retrieval
  • Loading branch information
@johanneskoester
johanneskoester committed Jul 27, 2022
1 parent 19cc2f4 commit 225e68c
Show file tree
Hide file tree
Showing 3 changed files with 71 additions and 62 deletions.
16 changes: 0 additions & 16 deletions snakemake/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2627,22 +2627,6 @@ def parse_cores(cores):
)
sys.exit(1)

if args.use_conda and args.conda_frontend == "mamba":
from snakemake.deployment.conda import is_mamba_available

if not is_mamba_available():
print(
"Error: mamba package manager is not available. "
"The mamba package manager (https://github.com/mamba-org/mamba) is an "
"extremely fast and robust conda replacement. "
"It is the recommended way of using Snakemake's conda integration. "
"It can be installed with `conda install -n base -c conda-forge mamba`. "
"If you still prefer to use conda, you can enforce that by setting "
"`--conda-frontend conda`.",
file=sys.stderr,
)
sys.exit(1)

if args.singularity_prefix and not args.use_singularity:
print(
"Error: --use_singularity must be set if --singularity-prefix " "is set.",
Expand Down
105 changes: 65 additions & 40 deletions snakemake/deployment/conda.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,9 @@ def __init__(

@lazy_property
def conda(self):
return Conda(self._container_img)
return Conda(
container_img=self._container_img, frontend=self.frontend, check=True
)

@lazy_property
def pin_file(self):
Expand Down Expand Up @@ -626,7 +628,7 @@ class Conda:
instances = dict()
lock = threading.Lock()

def __new__(cls, container_img=None, prefix_path=None):
def __new__(cls, container_img=None, prefix_path=None, frontend=None, check=False):
with cls.lock:
if container_img not in cls.instances:
inst = super().__new__(cls)
Expand All @@ -635,18 +637,22 @@ def __new__(cls, container_img=None, prefix_path=None):
else:
return cls.instances[container_img]

def __init__(self, container_img=None, prefix_path=None):
def __init__(
self, container_img=None, prefix_path=None, frontend=None, check=False
):
if not self.is_initialized: # avoid superfluous init calls
from snakemake.deployment import singularity
from snakemake.shell import shell

if isinstance(container_img, singularity.Image):
container_img = container_img.path
self.container_img = container_img
self.frontend = frontend

self.info = json.loads(
shell.check_output(
self._get_cmd("conda info --json"), universal_newlines=True
self._get_cmd(f"conda info --json"),
universal_newlines=True,
)
)

Expand All @@ -658,7 +664,10 @@ def __init__(self, container_img=None, prefix_path=None):
self.platform = self.info["platform"]

# check conda installation
self._check()
if check:
if frontend is None:
raise ValueError("Frontend must be specified if check is True.")
self._check()

@property
def is_initialized(self):
Expand All @@ -672,48 +681,64 @@ def _get_cmd(self, cmd):
def _check(self):
from snakemake.shell import shell

# Use type here since conda now is a function.
# type allows to check for both functions and regular commands.
if not ON_WINDOWS or shell.get_executable():
locate_cmd = "type conda"
else:
locate_cmd = "where conda"
frontends = ["conda"]
if self.frontend == "mamba":
frontends = ["mamba", "conda"]

try:
shell.check_output(self._get_cmd(locate_cmd), stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
if self.container_img:
raise CreateCondaEnvironmentException(
"The 'conda' command is not "
"available inside "
"your singularity container "
"image. Snakemake mounts "
"your conda installation "
"into singularity. "
"Sometimes, this can fail "
"because of shell restrictions. "
"It has been tested to work "
"with docker://ubuntu, but "
"it e.g. fails with "
"docker://bash "
)
for frontend in frontends:
# Use type here since conda now is a function.
# type allows to check for both functions and regular commands.
if not ON_WINDOWS or shell.get_executable():
locate_cmd = f"type {frontend}"
else:
raise CreateCondaEnvironmentException(
"The 'conda' command is not "
"available in the "
"shell {} that will be "
"used by Snakemake. You have "
"to ensure that it is in your "
"PATH, e.g., first activating "
"the conda base environment "
"with `conda activate base`.".format(shell.get_executable())
)
locate_cmd = f"where {frontend}"

try:
shell.check_output(self._get_cmd(locate_cmd), stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
if self.container_img:
msg = (
f"The '{frontend}' command is not "
"available inside "
"your singularity container "
"image. Snakemake mounts "
"your conda installation "
"into singularity. "
"Sometimes, this can fail "
"because of shell restrictions. "
"It has been tested to work "
"with docker://ubuntu, but "
"it e.g. fails with "
"docker://bash "
)
else:
msg = (
f"The '{frontend}' command is not "
"available in the "
f"shell {shell.get_executable()} that will be "
"used by Snakemake. You have "
"to ensure that it is in your "
"PATH, e.g., first activating "
"the conda base environment "
"with `conda activate base`."
)
if frontend == "mamba":
msg += (
"The mamba package manager (https://github.com/mamba-org/mamba) is a "
"fast and robust conda replacement. "
"It is the recommended way of using Snakemake's conda integration. "
"It can be installed with `conda install -n base -c conda-forge mamba`. "
"If you still prefer to use conda, you can enforce that by setting "
"`--conda-frontend conda`.",
)
raise CreateCondaEnvironmentException(msg)

try:
self._check_version()
self._check_condarc()
except subprocess.CalledProcessError as e:
raise CreateCondaEnvironmentException(
"Unable to check conda installation:\n" + e.stderr.decode()
f"Unable to check conda installation:" "\n" + e.stderr.decode()
)

def _check_version(self):
Expand Down
12 changes: 6 additions & 6 deletions snakemake/shell.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,13 +176,13 @@ def __new__(
if conda_env:
if ON_WINDOWS and not cls.get_executable():
# If we use cmd.exe directly on winodws we need to prepend batch activation script.
cmd = Conda(container_img, prefix_path=conda_base_path).shellcmd_win(
conda_env, cmd
)
cmd = Conda(
container_img=container_img, prefix_path=conda_base_path
).shellcmd_win(conda_env, cmd)
else:
cmd = Conda(container_img, prefix_path=conda_base_path).shellcmd(
conda_env, cmd
)
cmd = Conda(
container_img=container_img, prefix_path=conda_base_path
).shellcmd(conda_env, cmd)

tmpdir = None
if len(cmd.replace("'", r"'\''")) + 2 > MAX_ARG_LEN:
Expand Down

0 comments on commit 225e68c

Please sign in to comment.