From a3b61a508026de5f9bd82dac7060fe5e1200c7bd Mon Sep 17 00:00:00 2001 From: Neal Stewart Date: Thu, 28 Sep 2023 16:11:58 +0200 Subject: [PATCH] Proposed implementation for #515: Install development dependencies incrementally --- conda_lock/conda_lock.py | 30 +++++++- tests/test_conda_lock.py | 133 +++++++++++++++++++++++++++++++++ tests/zlib/environment.dev.yml | 6 ++ 3 files changed, 165 insertions(+), 4 deletions(-) create mode 100644 tests/zlib/environment.dev.yml diff --git a/conda_lock/conda_lock.py b/conda_lock/conda_lock.py index 239e909c7..e500fabfd 100644 --- a/conda_lock/conda_lock.py +++ b/conda_lock/conda_lock.py @@ -197,12 +197,13 @@ def do_validate_platform(lockfile: str) -> None: ) -def do_conda_install( +def do_conda_create_or_update( conda: PathLike, prefix: "str | None", name: "str | None", file: pathlib.Path, copy: bool, + update: bool, ) -> None: _conda = partial(_invoke_conda, conda, prefix, name, check_call=True) @@ -218,24 +219,38 @@ def do_conda_install( else: pip_requirements = [] - env_prefix = ["env"] if kind == "env" and not is_micromamba(conda) else [] + env_prefix = ( + ["env"] if (kind == "env" or update) and not is_micromamba(conda) else [] + ) copy_arg = ["--copy"] if kind != "env" and copy else [] yes_arg = ["--yes"] if kind != "env" else [] + prune_arg = ["--prune"] if update else [] _conda( [ *env_prefix, - "create", + "update" if update else "create", *copy_arg, "--file", str(file), *yes_arg, + *prune_arg, ], ) if not pip_requirements: return + if update: + logger.warning( + ( + "If you have have removed any pip dependencies from your lockfile, " + "they will not be removed from your environment. To remove them, " + "run `conda-lock install` to create a completely fresh conda environment. \n\n" + "You can safely ignore this message if your lockfile is unchanged since you created it." + ) + ) + with temporary_file_with_contents("\n".join(pip_requirements)) as requirements_path: _conda(["run"], ["pip", "install", "--no-deps", "-r", str(requirements_path)]) @@ -1396,6 +1411,7 @@ def lock( default=False, help="don't attempt to use or install micromamba.", ) +@click.option("--update", default=False, help="Update environment if available.") @click.option( "--copy", is_flag=True, @@ -1454,6 +1470,7 @@ def install( validate_platform: bool, log_level: TLogLevel, dev: bool, + update: bool, extras: List[str], ) -> None: # bail out if we do not encounter the lockfile @@ -1469,7 +1486,12 @@ def install( ) _conda_exe = determine_conda_executable(conda, mamba=mamba, micromamba=micromamba) install_func = partial( - do_conda_install, conda=_conda_exe, prefix=prefix, name=name, copy=copy + do_conda_create_or_update, + conda=_conda_exe, + prefix=prefix, + name=name, + copy=copy, + update=update, ) if validate_platform and not lock_file.name.endswith(DEFAULT_LOCKFILE_NAME): lockfile_contents = read_file(lock_file) diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index 265aaa96b..d4d782060 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -162,6 +162,11 @@ def zlib_environment(tmp_path: Path): return clone_test_dir("zlib", tmp_path).joinpath("environment.yml") +@pytest.fixture +def zlib_dev_environment(tmp_path: Path): + return clone_test_dir("zlib", tmp_path).joinpath("environment.dev.yml") + + @pytest.fixture def input_hash_zlib_environment(tmp_path: Path): return clone_test_dir("test-input-hash-zlib", tmp_path).joinpath("environment.yml") @@ -1578,6 +1583,14 @@ def _check_package_installed(package: str, prefix: str): return True +def _check_package_not_installed(package: str, prefix: str): + import glob + + files = list(glob.glob(f"{prefix}/conda-meta/{package}-*.json")) + assert len(files) == 0 + return True + + def conda_supports_env(conda_exe: str): try: subprocess.check_call( @@ -1690,6 +1703,126 @@ def invoke_install(*extra_args: str) -> CliResult: ), f"Package {package} does not exist in {prefix} environment" +@pytest.mark.parametrize("kind", ["explicit", "env"]) +@flaky +def test_install_with_update( + request: "pytest.FixtureRequest", + kind: str, + tmp_path: Path, + conda_exe: str, + zlib_environment: Path, + zlib_dev_environment: Path, + monkeypatch: "pytest.MonkeyPatch", + capsys: "pytest.CaptureFixture[str]", +): + # Test that we can lock, install without dev dependencies, then use the update flag to install the dev dependencies + if is_micromamba(conda_exe): + monkeypatch.setenv("CONDA_FLAGS", "-v") + if kind == "env" and not conda_supports_env(conda_exe): + pytest.skip( + f"Standalone conda @ '{conda_exe}' does not support materializing from environment files." + ) + + root_prefix = tmp_path / "root_prefix" + generated_lockfile_path = tmp_path / "generated_lockfiles" + + root_prefix.mkdir(exist_ok=True) + generated_lockfile_path.mkdir(exist_ok=True) + monkeypatch.chdir(generated_lockfile_path) + + package = "zlib" + dev_dependency = "pydantic" + platform = "linux-64" + + lock_filename_template = ( + request.node.name + "conda-{platform}-{dev-dependencies}.lock" + ) + lock_filename = ( + request.node.name + + "conda-linux-64-true.lock" + + (".yml" if kind == "env" else "") + ) + + with capsys.disabled(): + runner = CliRunner(mix_stderr=False) + result = runner.invoke( + main, + [ + "lock", + "--conda", + conda_exe, + "-p", + platform, + "-f", + str(zlib_environment), + "-f", + str(zlib_dev_environment), + "-k", + kind, + "--filename-template", + lock_filename_template, + ], + catch_exceptions=False, + ) + print(result.stdout, file=sys.stdout) + print(result.stderr, file=sys.stderr) + assert result.exit_code == 0 + + prefix = root_prefix / "test_env" + + def invoke_install(*extra_args: str) -> CliResult: + with capsys.disabled(): + return runner.invoke( + main, + [ + "install", + "--conda", + conda_exe, + "--prefix", + str(prefix), + *extra_args, + lock_filename, + ], + catch_exceptions=False, + ) + + with install_lock(): + result = invoke_install() + print(result.stdout, file=sys.stdout) + print(result.stderr, file=sys.stderr) + if Path(lock_filename).exists(): + logging.debug( + "lockfile contents: \n\n=======\n%s\n\n==========", + Path(lock_filename).read_text(), + ) + + assert _check_package_installed( + package=package, + prefix=str(prefix), + ), f"Package {package} does not exist in {prefix} environment" + + assert _check_package_not_installed( + package=dev_dependency, + prefix=str(prefix), + ), f"Package {dev_dependency} exists in {prefix} environment" + + # Now try to install with the update and dev dependencies flag + with install_lock(): + result = invoke_install("--update", "--dev") + print(result.stdout, file=sys.stdout) + print(result.stderr, file=sys.stderr) + if Path(lock_filename).exists(): + logging.debug( + "lockfile contents: \n\n=======\n%s\n\n==========", + Path(lock_filename).read_text(), + ) + + assert _check_package_installed( + package=dev_dependency, + prefix=str(prefix), + ), f"Package {package} does not exist in {prefix} environment" + + @pytest.mark.parametrize( "line,stripped", ( diff --git a/tests/zlib/environment.dev.yml b/tests/zlib/environment.dev.yml new file mode 100644 index 000000000..850e55085 --- /dev/null +++ b/tests/zlib/environment.dev.yml @@ -0,0 +1,6 @@ +channels: + - conda-forge + - defaults +dependencies: + - pydantic +category: dev \ No newline at end of file