From add2eaae54cdca8d5d188bddcfeb5787bb9007e9 Mon Sep 17 00:00:00 2001 From: Damian Shaw Date: Sun, 17 Mar 2024 11:24:37 -0400 Subject: [PATCH 1/4] Add packse scenarios as tests --- tests/functional/test_resolve_packse.py | 159 ++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 tests/functional/test_resolve_packse.py diff --git a/tests/functional/test_resolve_packse.py b/tests/functional/test_resolve_packse.py new file mode 100644 index 00000000000..c9c18220652 --- /dev/null +++ b/tests/functional/test_resolve_packse.py @@ -0,0 +1,159 @@ +import json +import subprocess +import sys +import time +from typing import Any, Dict, Generator, List + +import pytest + +from tests.lib import PipTestEnvironment + +ambigious_prerelease_resolution = ( + "Spec is ambigious, and uv and pip do not agree: " + "https://github.com/astral-sh/packse/issues/160" +) +requires_different_python_version = ( + "Don't support this test yet with the way pip tests work. Probably " + "need some kind of support from packse, written up a couple of " + "issues that would allow it to be supported: " + "https://github.com/astral-sh/packse/issues/164 " + "https://github.com/astral-sh/packse/issues/163" +) +not_served_as_yanked = ( + "There seems to be an issue with packse serve right now where " + "yanked packages are not showing as yanked in the simple api: " + "https://github.com/astral-sh/packse/issues/165" +) + +EXPECTED_TO_FAIL = { + "example": ( + "Expected solution looks wrong: " + "https://github.com/astral-sh/packse/issues/157" + ), + "local-not-used-with-sdist": ( + "Tests that sdist versions are preferred over local versions. " + "Discussed: https://github.com/astral-sh/packse/issues/158. " + "TODO: Find if this is a known issue on pip and/or packaging" + ), + "local-transitive-confounding": ( + "Expected solution looks wrong: " + "https://github.com/astral-sh/packse/issues/159" + ), + "transitive-prerelease-and-stable-dependency": (ambigious_prerelease_resolution), + "transitive-prerelease-and-stable-dependency-many-versions": ( + ambigious_prerelease_resolution + ), + "transitive-package-only-prereleases-in-range-opt-in": ( + ambigious_prerelease_resolution + ), + "package-only-prereleases-boundary": ( + "Expected solution is probably wrong: " + "https://github.com/astral-sh/packse/issues/161" + ), + "package-prereleases-specifier-boundary": (requires_different_python_version), + "python-greater-than-current": (requires_different_python_version), + "python-greater-than-current-patch": (requires_different_python_version), + "python-greater-than-current-backtrack": (requires_different_python_version), + "python-greater-than-current-excluded": (requires_different_python_version), + "compatible-python-incompatible-override": (requires_different_python_version), + "incompatible-python-compatible-override-unavailable-no-wheels": ( + requires_different_python_version + ), + "incompatible-python-compatible-override-no-compatible-wheels": ( + requires_different_python_version + ), + "incompatible-python-compatible-override-other-wheel": ( + requires_different_python_version + ), + "python-patch-override-no-patch": (requires_different_python_version), + "package-only-yanked": (not_served_as_yanked), + "package-only-yanked-in-range": (not_served_as_yanked), + "requires-package-yanked-and-unyanked-any": (not_served_as_yanked), + "package-yanked-specified-mixed-available": (not_served_as_yanked), + "transitive-package-only-yanked": (not_served_as_yanked), + "transitive-package-only-yanked-in-range": (not_served_as_yanked), + "transitive-yanked-and-unyanked-dependency": (not_served_as_yanked), +} + + +def run_command(command: List[str], cwd: None = None) -> str: + result = subprocess.run(command, cwd=cwd, capture_output=True, text=True) + result.check_returncode() + return result.stdout.strip() + + +@pytest.fixture(scope="session", autouse=True) +def start_packse_server() -> Generator[None, None, None]: + """Starts the packse server before tests run and ensures it's terminated after.""" + proc = subprocess.Popen( + ["packse", "serve", "--host", "127.0.0.1", "--port", "3141"] + ) + time.sleep(1) + yield + proc.terminate() + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + """Dynamically parameterize tests based on scenarios fetched from packse.""" + if "scenario" in metafunc.fixturenames: + # Fetch scenarios using packse + scenarios_json: str = run_command(["packse", "inspect", "scenarios"]) + scenarios = json.loads(scenarios_json) + + # Prepare scenarios for parameterization, marking some as XFAIL + scenarios_for_param = [] + ids_for_param = [] + for scenario in scenarios["scenarios"]: + if scenario["name"] in EXPECTED_TO_FAIL: + mark = pytest.mark.xfail(reason=EXPECTED_TO_FAIL[scenario["name"]]) + scenario_data = pytest.param(scenario, marks=mark) + else: + scenario_data = scenario + scenarios_for_param.append(scenario_data) + ids_for_param.append(scenario["name"]) + + # Parameterize the test function with the prepared scenarios and IDs + metafunc.parametrize("scenario", scenarios_for_param, ids=ids_for_param) + + +@pytest.mark.network +@pytest.mark.skipif(sys.version_info < (3, 12), reason="requires Python 3.12 or higher") +def test_packse_scenario(script: PipTestEnvironment, scenario: Dict[str, Any]) -> None: + """Dynamically generated test for each packse scenario.""" + expected_satisfiable: bool = scenario["expected"]["satisfiable"] + requirements: list[str] = [r["requirement"] for r in scenario["root"]["requires"]] + + resolver_options = [] + if scenario["resolver_options"]["prereleases"]: + resolver_options.append("--pre") + if scenario["resolver_options"]["no_build"]: + resolver_options.append("--only-binary") + resolver_options.append(",".join(scenario["resolver_options"]["no_build"])) + if scenario["resolver_options"]["no_binary"]: + resolver_options.append("--no-binary") + resolver_options.append(",".join(scenario["resolver_options"]["no_binary"])) + + # Install the package as per the scenario setup + result = script.pip( + "install", + "--index-url=http://127.0.0.1:3141/simple", + *resolver_options, + *requirements, + allow_error=True, + ) + + resolution_failure_message = ( + "ERROR: Could not find a version that satisfies the requirement", + "ERROR: ResolutionImpossible", + "ERROR: Cannot install", + "requires a different Python:", + ) + + if expected_satisfiable: + assert "ERROR" not in result.stderr + expected_installed = { + p["name"]: p["version"] for p in scenario["expected"]["packages"] + } + script.assert_installed(**expected_installed) + else: + assert any(error in result.stderr for error in resolution_failure_message) From 4cb553e79d2214f0309cfab34f2e482d96606506 Mon Sep 17 00:00:00 2001 From: Damian Shaw Date: Sun, 17 Mar 2024 11:32:06 -0400 Subject: [PATCH 2/4] NEWS Entry --- news/12580.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/12580.feature.rst diff --git a/news/12580.feature.rst b/news/12580.feature.rst new file mode 100644 index 00000000000..a96279f6794 --- /dev/null +++ b/news/12580.feature.rst @@ -0,0 +1 @@ +Add packse scenarios as pip tests From 40a1b5eda9b16b09f3cef0a33219e9b35a18038d Mon Sep 17 00:00:00 2001 From: Damian Shaw Date: Sun, 17 Mar 2024 11:51:09 -0400 Subject: [PATCH 3/4] Don't attempt to run packse on Python < 3.12 --- tests/functional/test_resolve_packse.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/functional/test_resolve_packse.py b/tests/functional/test_resolve_packse.py index c9c18220652..f2bde263688 100644 --- a/tests/functional/test_resolve_packse.py +++ b/tests/functional/test_resolve_packse.py @@ -95,8 +95,15 @@ def start_packse_server() -> Generator[None, None, None]: def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: """Dynamically parameterize tests based on scenarios fetched from packse.""" + + if sys.version_info < (3, 12): + return + if "scenario" in metafunc.fixturenames: # Fetch scenarios using packse + run_command(["packse", "fetch", "--force"]) + + # Inspect scenarios using packse scenarios_json: str = run_command(["packse", "inspect", "scenarios"]) scenarios = json.loads(scenarios_json) From 8332b548c2eee0b009ab12dce4a752bad6e37d3e Mon Sep 17 00:00:00 2001 From: Damian Shaw Date: Sun, 17 Mar 2024 11:55:40 -0400 Subject: [PATCH 4/4] Add packse serve to Python 3.12+ requirements --- tests/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/requirements.txt b/tests/requirements.txt index 84b7c14d4b4..b86f810c009 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,7 @@ cryptography freezegun installer +packse[serve] ; python_version >= '3.12' pytest pytest-cov pytest-rerunfailures