Fix #79 and #92 (#93)

Co-authored-by: Antoine DECHAUME <antoine.dechaume@irt-saintexupery.com>

tests/test_conda.py

@@ -41,6 +41,7 @@ def test_conda_run_command(cmd, initproj):
                 [tox]
                 skipsdist=True
                 [testenv:{}]
+                deps = pip>0,<999
                 commands_pre = python -c "import os; open('commands_pre', 'w').write(os.environ['CONDA_PREFIX'])"
                 commands = python -c "import os; open('commands', 'w').write(os.environ['CONDA_PREFIX'])"
                 commands_post = python -c "import os; open('commands_post', 'w').write(os.environ['CONDA_PREFIX'])"
@@ -49,6 +50,14 @@ def test_conda_run_command(cmd, initproj):
             )
         },
     )
+
+    result = cmd("-v", "-e", env_name)
+    result.assert_success()
+
+    for filename in ("commands_pre", "commands_post", "commands"):
+        assert open(filename).read().endswith(env_name)
+
+    # Run once again when the env creation hooks are not called.
     result = cmd("-v", "-e", env_name)
     result.assert_success()
 

tests/test_conda_env.py

@@ -98,7 +98,11 @@ def test_install_conda_deps(newconfig, mocksession):
 
     pip_cmd = pcalls[-1].args
     assert pip_cmd[6:9] == ["-m", "pip", "install"]
-    assert pip_cmd[9:11] == ["numpy", "astropy"]
+
+    # Get the deps from the requirements file.
+    with open(pip_cmd[9][2:]) as stream:
+        deps = [line.strip() for line in stream.readlines()]
+    assert deps == ["numpy", "astropy"]
 
 
 def test_install_conda_no_pip(newconfig, mocksession):

tox_conda/plugin.py

@@ -2,12 +2,14 @@
 import os
 import re
 import shutil
+import subprocess
+import tempfile
+from contextlib import contextmanager
 
 import pluggy
 import py.path
 import tox
 from tox.config import DepConfig, DepOption, TestenvConfig
-from tox.exception import InvocationError
 from tox.venv import VirtualEnv
 
 hookimpl = pluggy.HookimplMarker("tox")
@@ -57,18 +59,35 @@ class CondaRunWrapper:
     CONDA_RUN_CMD_PREFIX = "{conda_exe} run --no-capture-output -p {envdir}"
 
     def __init__(self, venv, popen):
-        self.__envdir = venv.envconfig.envdir
-        self.__conda_exe = venv.envconfig.conda_exe
+        self.__venv = venv
         self.__popen = popen
 
     def __call__(self, cmd_args, **kwargs):
         conda_run_cmd_prefix = self.CONDA_RUN_CMD_PREFIX.format(
-            conda_exe=self.__conda_exe, envdir=self.__envdir
+            conda_exe=self.__venv.envconfig.conda_exe, envdir=self.__venv.envconfig.envdir
         )
         cmd_args = conda_run_cmd_prefix.split() + cmd_args
         return self.__popen(cmd_args, **kwargs)
 
 
+@contextmanager
+def conda_run(venv, action=None):
+    """Run a command via conda run."""
+    if action is None:
+        initial_popen = venv.popen
+        venv.popen = CondaRunWrapper(venv, initial_popen)
+    else:
+        initial_popen = action.via_popen
+        action.via_popen = CondaRunWrapper(venv, initial_popen)
+
+    yield
+
+    if action is None:
+        venv.popen = initial_popen
+    else:
+        action.via_popen = initial_popen
+
+
 @hookimpl
 def tox_addoption(parser):
     parser.add_testenv_attribute(
@@ -107,7 +126,12 @@ def tox_addoption(parser):
 def tox_configure(config):
     # This is a pretty cheesy workaround. It allows tox to consider changes to
     # the conda dependencies when it decides whether an existing environment
-    # needs to be updated before being used
+    # needs to be updated before being used.
+
+    # Set path to the conda executable because it cannot be determined once
+    # an env has already been created.
+    conda_exe = find_conda()
+
     for envconfig in config.envconfigs.values():
         # Make sure the right environment is activated. This works because we're
         # creating environments using the `-p/--prefix` option in `tox_testenv_create`
@@ -120,8 +144,10 @@ def tox_configure(config):
             conda_deps.append(DepConfig("--file={}".format(envconfig.conda_spec)))
         envconfig.deps.extend(conda_deps)
 
+        envconfig.conda_exe = conda_exe
+
 
-def find_conda(action):
+def find_conda():
     # This should work if we're not already in an environment
     conda_exe = os.environ.get("_CONDA_EXE")
     if conda_exe:
@@ -132,14 +158,14 @@ def find_conda(action):
     if conda_exe:
         return conda_exe
 
+    path = shutil.which("conda")
+
     try:
-        path = shutil.which("conda")
-        action.popen([path, "-h"], report_fail=True, returnout=False)
-        return path
-    except InvocationError:
-        pass
+        subprocess.check_call([str(path), "-h"])
+    except subprocess.CalledProcessError:
+        raise RuntimeError("Can't locate conda executable")
 
-    raise RuntimeError("Can't locate conda executable")
+    return path
 
 
 @hookimpl
@@ -148,19 +174,24 @@ def tox_testenv_create(venv, action):
     basepath = venv.path.dirpath()
 
     # Check for venv.envconfig.sitepackages and venv.config.alwayscopy here
-    conda_exe = find_conda(action)
-    venv.envconfig.conda_exe = conda_exe
-
     envdir = venv.envconfig.envdir
     python = get_py_version(venv.envconfig, action)
 
     if venv.envconfig.conda_env is not None:
         # conda env create does not have a --channel argument nor does it take
         # dependencies specifications (e.g., python=3.8). These must all be specified
         # in the conda-env.yml file
-        args = [conda_exe, "env", "create", "-p", envdir, "--file", venv.envconfig.conda_env]
+        args = [
+            venv.envconfig.conda_exe,
+            "env",
+            "create",
+            "-p",
+            envdir,
+            "--file",
+            venv.envconfig.conda_env,
+        ]
     else:
-        args = [conda_exe, "create", "--yes", "-p", envdir]
+        args = [venv.envconfig.conda_exe, "create", "--yes", "-p", envdir]
         for channel in venv.envconfig.conda_channels:
             args += ["--channel", channel]
 
@@ -180,16 +211,13 @@ def tox_testenv_create(venv, action):
         del venv.envconfig.config.interpreters.name2executable[venv.name]
     except KeyError:
         pass
-    venv.envconfig.config.interpreters.get_executable(venv.envconfig)
 
-    # this will force commands and commands_{pre,post} to be executed via conda run
-    venv.popen = CondaRunWrapper(venv, venv.popen)
+    venv.envconfig.config.interpreters.get_executable(venv.envconfig)
 
     return True
 
 
 def install_conda_deps(venv, action, basepath, envdir):
-    conda_exe = venv.envconfig.conda_exe
     # Account for the fact that we have a list of DepOptions
     conda_deps = [str(dep.name) for dep in venv.envconfig.conda_deps]
     # Add the conda-spec.txt file to the end of the conda deps b/c any deps
@@ -200,7 +228,7 @@ def install_conda_deps(venv, action, basepath, envdir):
     action.setactivity("installcondadeps", ", ".join(conda_deps))
 
     # Install quietly to make the log cleaner
-    args = [conda_exe, "install", "--quiet", "--yes", "-p", envdir]
+    args = [venv.envconfig.conda_exe, "install", "--quiet", "--yes", "-p", envdir]
     for channel in venv.envconfig.conda_channels:
         args += ["--channel", channel]
 
@@ -218,28 +246,37 @@ def install_conda_deps(venv, action, basepath, envdir):
 
 @hookimpl
 def tox_testenv_install_deps(venv, action):
-    basepath = venv.path.dirpath()
-    envdir = venv.envconfig.envdir
-    # Save for later : we will need it for the config file
+    # Save the deps before we make temporary changes.
     saved_deps = copy.deepcopy(venv.envconfig.deps)
 
     num_conda_deps = len(venv.envconfig.conda_deps)
     if venv.envconfig.conda_spec is not None:
         num_conda_deps += 1
 
     if num_conda_deps > 0:
-        install_conda_deps(venv, action, basepath, envdir)
+        install_conda_deps(venv, action, venv.path.dirpath(), venv.envconfig.envdir)
         # Account for the fact that we added the conda_deps to the deps list in
         # tox_configure (see comment there for rationale). We don't want them
-        # to be present when we call pip install
+        # to be present when we call pip install.
         venv.envconfig.deps = venv.envconfig.deps[: -1 * num_conda_deps]
 
-    # Install dependencies from pypi via conda run
-    action.via_popen = CondaRunWrapper(venv, action.via_popen)
-    tox.venv.tox_testenv_install_deps(venv=venv, action=action)
+    if venv.envconfig.deps:
+        # Dump the pypi deps to a requirements file because, as of conda 4.10.1,
+        # the conda run command cannot parse a pip command with conditions on
+        # the dependencies.
+        _, temp_req_filename = tempfile.mkstemp()
+        with open(temp_req_filename, "w") as stream:
+            lines = ["{}\n".format(dep) for dep in venv.envconfig.deps]
+            stream.writelines(lines)
+
+        venv.envconfig.deps = [tox.config.DepConfig("-r{}".format(temp_req_filename))]
 
-    # Restore for the config file
+    with conda_run(venv, action):
+        tox.venv.tox_testenv_install_deps(venv=venv, action=action)
+
+    # Restore the deps.
     venv.envconfig.deps = saved_deps
+
     return True
 
 
@@ -281,3 +318,22 @@ def venv_lookup(self, name):
 
 
 VirtualEnv._venv_lookup = venv_lookup
+
+
+@hookimpl(hookwrapper=True)
+def tox_runtest_pre(venv):
+    with conda_run(venv):
+        yield
+
+
+@hookimpl
+def tox_runtest(venv, redirect):
+    with conda_run(venv):
+        tox.venv.tox_runtest(venv, redirect)
+    return True
+
+
+@hookimpl(hookwrapper=True)
+def tox_runtest_post(venv):
+    with conda_run(venv):
+        yield