From 1b3a95269e351bcfe1bef09c59cb640652c15eaf Mon Sep 17 00:00:00 2001 From: Florent Lejoly Date: Mon, 12 Feb 2024 09:36:07 +0100 Subject: [PATCH] Fix the reporting of the indexes that were used to install a V2 module or third-party Python dependency if that package could not be found in the case of --no-index (Issue #6096, PR #7148) # Description [Add the used pip indexes to the PackageNotFound exception in run_pip](https://github.com/inmanta/inmanta-core/pull/7064) didn't take --no-index into account. This Pr fixes this. part of https://github.com/inmanta/inmanta-core/issues/6096 # Self Check: Strike through any lines that are not applicable (`~~line~~`) then check the box - [ ] Attached issue to pull request - [x] Changelog entry - [x] Type annotations are present - [x] Code is clear and sufficiently documented - [x] No (preventable) type errors (check using make mypy or make mypy-diff) - [x] Sufficient test cases (reproduces the bug/tests the requested feature) - [x] Correct, in line with design - [ ] End user documentation is included or an issue is created for end-user documentation (add ref to issue here: ) - [ ] If this PR fixes a race condition in the test suite, also push the fix to the relevant stable branche(s) (see [test-fixes](https://internal.inmanta.com/development/core/tasks/build-master.html#test-fixes) for more info) --- changelogs/unreleased/fix-log-pip-index.yml | 6 ++++ src/inmanta/env.py | 11 ++++-- src/inmanta/util/__init__.py | 37 +++++++++++++++++++++ tests/conftest.py | 2 +- tests/test_env.py | 33 +++++++++++++++++- 5 files changed, 85 insertions(+), 4 deletions(-) create mode 100644 changelogs/unreleased/fix-log-pip-index.yml diff --git a/changelogs/unreleased/fix-log-pip-index.yml b/changelogs/unreleased/fix-log-pip-index.yml new file mode 100644 index 0000000000..d7a0fb7aef --- /dev/null +++ b/changelogs/unreleased/fix-log-pip-index.yml @@ -0,0 +1,6 @@ +description: Fix the reporting of the indexes that were used to install a V2 module or third-party Python dependency if that package could not be found in the case of --no-index +issue-nr: 6096 +change-type: patch +destination-branches: [master, iso7, iso6] +sections: + bugfix: "Fix a bug where PIP_NO_INDEX could be used by pip when use_system_config was set to False in the PipConfig" diff --git a/src/inmanta/env.py b/src/inmanta/env.py index 35b4a7c048..89b719af2d 100644 --- a/src/inmanta/env.py +++ b/src/inmanta/env.py @@ -46,6 +46,7 @@ from inmanta.data.model import LEGACY_PIP_DEFAULT, PipConfig from inmanta.server.bootloader import InmantaBootloader from inmanta.stable_api import stable_api +from inmanta.util import strtobool from packaging import version LOGGER = logging.getLogger(__name__) @@ -455,6 +456,8 @@ def run_pip_install_command_from_config( del sub_env["PIP_INDEX_URL"] if "PIP_PRE" in sub_env: del sub_env["PIP_PRE"] + if "PIP_NO_INDEX" in sub_env: + del sub_env["PIP_NO_INDEX"] # setting this env_var to os.devnull disables the loading of all pip configuration file sub_env["PIP_CONFIG_FILE"] = os.devnull @@ -517,13 +520,17 @@ def create_log_content_files(title: str, files: list[str]) -> list[str]: if "versions have conflicting dependencies" in line: conflicts.append(line) # Get the indexes line from full_output + # This is not printed when not using any index or when only using PyPi if "Looking in indexes:" in line: indexes = line if not_found: - if indexes: + no_index: bool = "--no-index" in cmd or strtobool(env.get("PIP_NO_INDEX", "false")) + if no_index: + msg = "Packages %s were not found. No indexes were used." % ", ".join(not_found) + elif indexes: msg = "Packages %s were not found in the given indexes. (%s)" % (", ".join(not_found), indexes) else: - msg = "Packages %s were not found at PyPI." % (", ".join(not_found)) + msg = "Packages %s were not found at PyPI." % ", ".join(not_found) raise PackageNotFound(msg) if conflicts: raise ConflictingRequirements("\n".join(conflicts)) diff --git a/src/inmanta/util/__init__.py b/src/inmanta/util/__init__.py index a53b7ff1f4..d29881d423 100644 --- a/src/inmanta/util/__init__.py +++ b/src/inmanta/util/__init__.py @@ -78,6 +78,43 @@ def is_sub_dict(subdct: dict[PrimitiveTypes, PrimitiveTypes], dct: dict[Primitiv return not any(True for k, v in subdct.items() if k not in dct or dct[k] != v) +def strtobool(val: str) -> bool: + """Convert a string representation of truth to True or False. + + True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values + are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if + 'val' is anything else. + + This function is based on a function in the Python distutils package. Is is subject + to the following license: + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to + deal in the Software without restriction, including without limitation the + rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + IN THE SOFTWARE. + """ + val = val.lower() + if val in ("y", "yes", "t", "true", "on", "1"): + return True + elif val in ("n", "no", "f", "false", "off", "0"): + return False + else: + raise ValueError("invalid truth value %r" % (val,)) + + def hash_file(content: bytes) -> str: """ Create a hash from the given content diff --git a/tests/conftest.py b/tests/conftest.py index c26171351f..b52efad035 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1675,7 +1675,7 @@ def tmpvenv_active_inherit(deactive_venv, tmpdir: py.path.local) -> Iterator[env @pytest.fixture -def create_empty_local_package_index_factory() -> Callable[[], str]: +def create_empty_local_package_index_factory() -> Callable[[str], str]: """ A fixture that acts as a factory to create empty local pip package indexes. Each call creates a new index in a different temporary directory. diff --git a/tests/test_env.py b/tests/test_env.py index 9829899a77..40605ba418 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -238,7 +238,7 @@ def test_process_env_install_from_index( def test_process_env_install_from_index_not_found_env_var( tmpvenv_active: tuple[py.path.local, py.path.local], monkeypatch, - create_empty_local_package_index_factory: Callable[[], str], + create_empty_local_package_index_factory: Callable[[str], str], use_extra_indexes: bool, use_extra_indexes_env: bool, use_system_config: bool, @@ -288,6 +288,37 @@ def test_process_env_install_from_index_not_found_env_var( ) +@pytest.mark.parametrize_any("use_system_config", [True, False]) +def test_process_env_install_no_index(tmpdir: py.path.local, monkeypatch, use_system_config: bool) -> None: + """ + Attempt to install a package that does not exist with --no-index. + To have --no-index set in the pip cmd, the config should not contain an index_url, + we should not be using the system config and a path needs to be specified. + it can also be set in the env_vars + Assert the appropriate error is raised. + """ + if use_system_config: + monkeypatch.setenv("PIP_NO_INDEX", "true") + + setup_py_content = """ +from setuptools import setup +setup(name="test") +""" + # Write the minimal setup.py content to the temporary directory + setup_py_path = os.path.join(tmpdir, "setup.py") + with open(setup_py_path, "w") as setup_file: + setup_file.write(setup_py_content) + + expected = "Packages this-package-does-not-exist were not found. No indexes were used." + + with pytest.raises(env.PackageNotFound, match=re.escape(expected)): + env.process_env.install_for_config( + requirements=[Requirement.parse("this-package-does-not-exist")], + paths=[env.LocalPackagePath(path=str(tmpdir))], + config=PipConfig(use_system_config=use_system_config), + ) + + @pytest.mark.slowtest def test_process_env_install_from_index_conflicting_reqs( tmpdir: str, tmpvenv_active: tuple[py.path.local, py.path.local]