From 96f3f225f74d3136839bf76e6992e2b31ea69216 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Thu, 28 Apr 2022 01:46:11 +0200 Subject: [PATCH 1/7] repositories: add support for PEP 658 This change allows Poetry to make use of PEP 503 "simple" API repositories that implement PEP 658 for core metadata. Co-authored-by: Bartosz Sokorski --- src/poetry/repositories/http_repository.py | 84 ++++++++++++++++--- src/poetry/repositories/link_sources/html.py | 5 +- .../fixtures/legacy/isort-metadata.html | 12 +++ ...t-metadata-4.3.4-py3-none-any.whl.metadata | 28 +++++++ tests/repositories/test_legacy_repository.py | 35 ++++++++ 5 files changed, 151 insertions(+), 13 deletions(-) create mode 100644 tests/repositories/fixtures/legacy/isort-metadata.html create mode 100644 tests/repositories/fixtures/legacy/metadata/isort-metadata-4.3.4-py3-none-any.whl.metadata diff --git a/src/poetry/repositories/http_repository.py b/src/poetry/repositories/http_repository.py index 0dc30c2a610..9882c321237 100644 --- a/src/poetry/repositories/http_repository.py +++ b/src/poetry/repositories/http_repository.py @@ -10,6 +10,7 @@ from typing import Any from typing import Iterator +import pkginfo import requests import requests.adapters @@ -20,6 +21,7 @@ from poetry.core.version.markers import parse_marker from poetry.config.config import Config +from poetry.inspection.info import PackageInfo from poetry.inspection.lazy_wheel import HTTPRangeRequestUnsupported from poetry.inspection.lazy_wheel import metadata_from_wheel_url from poetry.repositories.cached_repository import CachedRepository @@ -37,7 +39,6 @@ if TYPE_CHECKING: from packaging.utils import NormalizedName - from poetry.inspection.info import PackageInfo from poetry.repositories.link_sources.base import LinkSource from poetry.utils.authenticator import RepositoryCertificateConfig @@ -155,10 +156,29 @@ def _get_info_from_sdist(self, url: str) -> PackageInfo: with self._cached_or_downloaded_file(Link(url)) as filepath: return PackageInfo.from_sdist(filepath) - def _get_info_from_urls(self, urls: dict[str, list[str]]) -> PackageInfo: + @staticmethod + def _get_info_from_metadata( + url: str, metadata: dict[str, pkginfo.Distribution] + ) -> PackageInfo | None: + if url in metadata: + dist = metadata[url] + return PackageInfo( + name=dist.name, + version=dist.version, + summary=dist.summary, + requires_dist=list(dist.requires_dist), + requires_python=dist.requires_python, + ) + return None + + def _get_info_from_urls( + self, + urls: dict[str, list[str]], + metadata: dict[str, pkginfo.Distribution] | None = None, + ) -> PackageInfo: + metadata = metadata or {} # Prefer to read data from wheels: this is faster and more reliable - wheels = urls.get("bdist_wheel") - if wheels: + if wheels := urls.get("bdist_wheel"): # We ought just to be able to look at any of the available wheels to read # metadata, they all should give the same answer. # @@ -194,13 +214,19 @@ def _get_info_from_urls(self, urls: dict[str, list[str]]) -> PackageInfo: platform_specific_wheels.append(wheel) if universal_wheel is not None: - return self._get_info_from_wheel(universal_wheel) + return self._get_info_from_metadata( + universal_wheel, metadata + ) or self._get_info_from_wheel(universal_wheel) info = None if universal_python2_wheel and universal_python3_wheel: - info = self._get_info_from_wheel(universal_python2_wheel) + info = self._get_info_from_metadata( + universal_python2_wheel, metadata + ) or self._get_info_from_wheel(universal_python2_wheel) - py3_info = self._get_info_from_wheel(universal_python3_wheel) + py3_info = self._get_info_from_metadata( + universal_python3_wheel, metadata + ) or self._get_info_from_wheel(universal_python3_wheel) if info.requires_python or py3_info.requires_python: info.requires_python = str( @@ -250,16 +276,24 @@ def _get_info_from_urls(self, urls: dict[str, list[str]]) -> PackageInfo: # Prefer non platform specific wheels if universal_python3_wheel: - return self._get_info_from_wheel(universal_python3_wheel) + return self._get_info_from_metadata( + universal_python3_wheel, metadata + ) or self._get_info_from_wheel(universal_python3_wheel) if universal_python2_wheel: - return self._get_info_from_wheel(universal_python2_wheel) + return self._get_info_from_metadata( + universal_python2_wheel, metadata + ) or self._get_info_from_wheel(universal_python2_wheel) if platform_specific_wheels: first_wheel = platform_specific_wheels[0] - return self._get_info_from_wheel(first_wheel) + return self._get_info_from_metadata( + first_wheel, metadata + ) or self._get_info_from_wheel(first_wheel) - return self._get_info_from_sdist(urls["sdist"][0]) + return self._get_info_from_metadata( + urls["sdist"][0], metadata + ) or self._get_info_from_sdist(urls["sdist"][0]) def _links_to_data(self, links: list[Link], data: PackageInfo) -> dict[str, Any]: if not links: @@ -268,11 +302,37 @@ def _links_to_data(self, links: list[Link], data: PackageInfo) -> dict[str, Any] f' "{data.version}"' ) urls = defaultdict(list) + metadata = {} files: list[dict[str, Any]] = [] for link in links: if link.yanked and not data.yanked: # drop yanked files unless the entire release is yanked continue + if link.has_metadata: + try: + assert link.metadata_url is not None + response = self.session.get(link.metadata_url) + distribution = pkginfo.Distribution() + assert link.metadata_hash_name is not None + metadata_hash = getattr(hashlib, link.metadata_hash_name)( + response.text.encode() + ).hexdigest() + + if metadata_hash != link.metadata_hash: + self._log( + f"Metadata file hash ({metadata_hash}) does not match" + f" expected hash ({link.metadata_hash}).", + level="warning", + ) + + distribution.parse(response.content) + metadata[link.url] = distribution + except requests.HTTPError: + self._log( + f"Failed to retrieve metadata at {link.metadata_url}", + level="debug", + ) + if link.is_wheel: urls["bdist_wheel"].append(link.url) elif link.filename.endswith( @@ -299,7 +359,7 @@ def _links_to_data(self, links: list[Link], data: PackageInfo) -> dict[str, Any] data.files = files - info = self._get_info_from_urls(urls) + info = self._get_info_from_urls(urls, metadata) data.summary = info.summary data.requires_dist = info.requires_dist diff --git a/src/poetry/repositories/link_sources/html.py b/src/poetry/repositories/link_sources/html.py index 74663cd9304..e50983e5f74 100644 --- a/src/poetry/repositories/link_sources/html.py +++ b/src/poetry/repositories/link_sources/html.py @@ -42,7 +42,10 @@ def _link_cache(self) -> LinkCache: yanked = unescape(yanked_value) else: yanked = "data-yanked" in anchor - link = Link(url, requires_python=pyrequire, yanked=yanked) + metadata = anchor.get("data-dist-info-metadata") + link = Link( + url, requires_python=pyrequire, yanked=yanked, metadata=metadata + ) if link.ext not in self.SUPPORTED_FORMATS: continue diff --git a/tests/repositories/fixtures/legacy/isort-metadata.html b/tests/repositories/fixtures/legacy/isort-metadata.html new file mode 100644 index 00000000000..699a13e0192 --- /dev/null +++ b/tests/repositories/fixtures/legacy/isort-metadata.html @@ -0,0 +1,12 @@ + + + + Links for isort + + +

Links for isort

+isort-metadata-4.3.4-py3-none-any.whl
+ + + diff --git a/tests/repositories/fixtures/legacy/metadata/isort-metadata-4.3.4-py3-none-any.whl.metadata b/tests/repositories/fixtures/legacy/metadata/isort-metadata-4.3.4-py3-none-any.whl.metadata new file mode 100644 index 00000000000..bab8d017156 --- /dev/null +++ b/tests/repositories/fixtures/legacy/metadata/isort-metadata-4.3.4-py3-none-any.whl.metadata @@ -0,0 +1,28 @@ +Metadata-Version: 2.0 +Name: isort-metadata +Version: 4.3.4 +Summary: A Python utility / library to sort Python imports. +Home-page: https://github.com/timothycrosley/isort +Author: Timothy Crosley +Author-email: timothy.crosley@gmail.com +License: MIT +Keywords: Refactor,Python,Python2,Python3,Refactoring,Imports,Sort,Clean +Platform: UNKNOWN +Classifier: Development Status :: 6 - Mature +Classifier: Intended Audience :: Developers +Classifier: Natural Language :: English +Classifier: Environment :: Console +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Software Development :: Libraries +Classifier: Topic :: Utilities +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Requires-Dist: futures; python_version=="2.7" diff --git a/tests/repositories/test_legacy_repository.py b/tests/repositories/test_legacy_repository.py index 71b831641fa..ae895507e01 100644 --- a/tests/repositories/test_legacy_repository.py +++ b/tests/repositories/test_legacy_repository.py @@ -1,6 +1,7 @@ from __future__ import annotations import base64 +import posixpath import re import shutil @@ -23,10 +24,13 @@ if TYPE_CHECKING: + from typing import Any + import httpretty from _pytest.monkeypatch import MonkeyPatch from packaging.utils import NormalizedName + from pytest_mock import MockerFixture from poetry.config.config import Config @@ -176,6 +180,37 @@ def test_get_package_information_fallback_read_setup() -> None: ) +def _get_mock(url: str, **__: Any) -> requests.Response: + if url.endswith(".metadata"): + response = requests.Response() + response.encoding = "application/text" + response._content = MockRepository.FIXTURES.joinpath( + "metadata", posixpath.basename(url) + ).read_bytes() + return response + raise requests.HTTPError() + + +def test_get_package_information_pep_658(mocker: MockerFixture) -> None: + repo = MockRepository() + + isort_package = repo.package("isort", Version.parse("4.3.4")) + + mocker.patch.object(repo.session, "get", _get_mock) + + try: + package = repo.package("isort-metadata", Version.parse("4.3.4")) + except FileNotFoundError: + pytest.fail("Metadata was not successfully retrieved") + else: + assert package.source_type == isort_package.source_type == "legacy" + assert package.source_reference == isort_package.source_reference == repo.name + assert package.source_url == isort_package.source_url == repo.url + assert package.name == "isort-metadata" + assert package.version.text == isort_package.version.text == "4.3.4" + assert package.description == isort_package.description + + def test_get_package_information_skips_dependencies_with_invalid_constraints() -> None: repo = MockRepository() From 2e8dd46375465ac743e255cc7e7fbbc1d228140e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Randy=20D=C3=B6ring?= <30527984+radoering@users.noreply.github.com> Date: Sat, 18 Nov 2023 13:42:55 +0100 Subject: [PATCH 2/7] fix parsing of metadata attribute, add test --- src/poetry/repositories/link_sources/html.py | 13 ++++- tests/repositories/link_sources/test_html.py | 54 ++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/src/poetry/repositories/link_sources/html.py b/src/poetry/repositories/link_sources/html.py index e50983e5f74..7dfbd19e061 100644 --- a/src/poetry/repositories/link_sources/html.py +++ b/src/poetry/repositories/link_sources/html.py @@ -42,7 +42,18 @@ def _link_cache(self) -> LinkCache: yanked = unescape(yanked_value) else: yanked = "data-yanked" in anchor - metadata = anchor.get("data-dist-info-metadata") + + # see https://peps.python.org/pep-0714/#clients + # and https://peps.python.org/pep-0658/#specification + metadata: str | bool + for metadata_key in ("data-core-metadata", "data-dist-info-metadata"): + metadata_value = anchor.get(metadata_key) + if metadata_value: + metadata = unescape(metadata_value) + else: + metadata = metadata_key in anchor + if metadata: + break link = Link( url, requires_python=pyrequire, yanked=yanked, metadata=metadata ) diff --git a/tests/repositories/link_sources/test_html.py b/tests/repositories/link_sources/test_html.py index e39edcae034..14451ea78dc 100644 --- a/tests/repositories/link_sources/test_html.py +++ b/tests/repositories/link_sources/test_html.py @@ -91,6 +91,60 @@ def test_yanked( assert page.yanked(canonicalize_name("demo"), Version.parse("0.1")) == expected +@pytest.mark.parametrize( + ("metadata", "expected_has_metadata", "expected_metadata_hashes"), + [ + ("", False, {}), + # new + ("data-core-metadata", True, {}), + ("data-core-metadata=''", True, {}), + ("data-core-metadata='foo'", True, {}), + ("data-core-metadata='sha256=abcd'", True, {"sha256": "abcd"}), + # old + ("data-dist-info-metadata", True, {}), + ("data-dist-info-metadata=''", True, {}), + ("data-dist-info-metadata='foo'", True, {}), + ("data-dist-info-metadata='sha256=abcd'", True, {"sha256": "abcd"}), + # conflicting (new wins) + ("data-core-metadata data-dist-info-metadata='sha256=abcd'", True, {}), + ("data-dist-info-metadata='sha256=abcd' data-core-metadata", True, {}), + ( + "data-core-metadata='sha256=abcd' data-dist-info-metadata", + True, + {"sha256": "abcd"}, + ), + ( + "data-dist-info-metadata data-core-metadata='sha256=abcd'", + True, + {"sha256": "abcd"}, + ), + ( + "data-core-metadata='sha256=abcd' data-dist-info-metadata='sha256=1234'", + True, + {"sha256": "abcd"}, + ), + ( + "data-dist-info-metadata='sha256=1234' data-core-metadata='sha256=abcd'", + True, + {"sha256": "abcd"}, + ), + ], +) +def test_metadata( + html_page_content: HTMLPageGetter, + metadata: str, + expected_has_metadata: bool, + expected_metadata_hashes: dict[str, str], +) -> None: + anchors = f'demo-0.1.whl' + content = html_page_content(anchors) + page = HTMLPage("https://example.org", content) + + link = next(page.links) + assert link.has_metadata is expected_has_metadata + assert link.metadata_hashes == expected_metadata_hashes + + @pytest.mark.parametrize( "anchor, base_url, expected", ( From 9f65db1c342556ab741e0047e7135011274f902f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Randy=20D=C3=B6ring?= <30527984+radoering@users.noreply.github.com> Date: Sat, 18 Nov 2023 13:44:41 +0100 Subject: [PATCH 3/7] apply review feedback, hash is optional --- src/poetry/repositories/http_repository.py | 28 +++++++++++--------- tests/repositories/test_legacy_repository.py | 10 ++++++- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/poetry/repositories/http_repository.py b/src/poetry/repositories/http_repository.py index 9882c321237..e80ddf624fd 100644 --- a/src/poetry/repositories/http_repository.py +++ b/src/poetry/repositories/http_repository.py @@ -302,7 +302,7 @@ def _links_to_data(self, links: list[Link], data: PackageInfo) -> dict[str, Any] f' "{data.version}"' ) urls = defaultdict(list) - metadata = {} + metadata: dict[str, pkginfo.Distribution] = {} files: list[dict[str, Any]] = [] for link in links: if link.yanked and not data.yanked: @@ -313,24 +313,26 @@ def _links_to_data(self, links: list[Link], data: PackageInfo) -> dict[str, Any] assert link.metadata_url is not None response = self.session.get(link.metadata_url) distribution = pkginfo.Distribution() - assert link.metadata_hash_name is not None - metadata_hash = getattr(hashlib, link.metadata_hash_name)( - response.text.encode() - ).hexdigest() - - if metadata_hash != link.metadata_hash: - self._log( - f"Metadata file hash ({metadata_hash}) does not match" - f" expected hash ({link.metadata_hash}).", - level="warning", - ) + if link.metadata_hash_name is not None: + metadata_hash = getattr(hashlib, link.metadata_hash_name)( + response.text.encode() + ).hexdigest() + + if metadata_hash != link.metadata_hash: + self._log( + f"Metadata file hash ({metadata_hash}) does not match" + f" expected hash ({link.metadata_hash})." + f" Metadata file for {link.filename} will be ignored.", + level="warning", + ) + continue distribution.parse(response.content) metadata[link.url] = distribution except requests.HTTPError: self._log( f"Failed to retrieve metadata at {link.metadata_url}", - level="debug", + level="warning", ) if link.is_wheel: diff --git a/tests/repositories/test_legacy_repository.py b/tests/repositories/test_legacy_repository.py index ae895507e01..478fffa4c86 100644 --- a/tests/repositories/test_legacy_repository.py +++ b/tests/repositories/test_legacy_repository.py @@ -186,7 +186,7 @@ def _get_mock(url: str, **__: Any) -> requests.Response: response.encoding = "application/text" response._content = MockRepository.FIXTURES.joinpath( "metadata", posixpath.basename(url) - ).read_bytes() + ).read_text().encode() return response raise requests.HTTPError() @@ -209,6 +209,14 @@ def test_get_package_information_pep_658(mocker: MockerFixture) -> None: assert package.name == "isort-metadata" assert package.version.text == isort_package.version.text == "4.3.4" assert package.description == isort_package.description + assert ( + package.requires == isort_package.requires == [Dependency("futures", "*")] + ) + assert ( + str(package.python_constraint) + == str(isort_package.python_constraint) + == ">=2.7,<3.0.dev0 || >=3.4.dev0" + ) def test_get_package_information_skips_dependencies_with_invalid_constraints() -> None: From 9c950c2e595c43a0e9b55cddfb92193958f0712b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Randy=20D=C3=B6ring?= <30527984+radoering@users.noreply.github.com> Date: Sun, 19 Nov 2023 05:37:15 +0100 Subject: [PATCH 4/7] repositories: add support for PEP 691 as fallback for PyPI --- src/poetry/repositories/http_repository.py | 33 +++-- src/poetry/repositories/link_sources/json.py | 17 ++- src/poetry/repositories/pypi_repository.py | 26 ++-- tests/repositories/conftest.py | 28 +++++ ...-metadata-4.3.4-py2-none-any.whl.metadata} | 0 ...t-metadata-4.3.4-py3-none-any.whl.metadata | 28 +++++ .../pypi.org/json/isort-metadata.json | 35 ++++++ .../pypi.org/json/isort-metadata/4.3.4.json | 117 ++++++++++++++++++ tests/repositories/link_sources/test_json.py | 79 ++++++++++++ tests/repositories/test_legacy_repository.py | 25 ++-- tests/repositories/test_pypi_repository.py | 26 ++++ tests/types.py | 6 + 12 files changed, 377 insertions(+), 43 deletions(-) rename tests/repositories/fixtures/{legacy/metadata/isort-metadata-4.3.4-py3-none-any.whl.metadata => metadata/isort-metadata-4.3.4-py2-none-any.whl.metadata} (100%) create mode 100644 tests/repositories/fixtures/metadata/isort-metadata-4.3.4-py3-none-any.whl.metadata create mode 100644 tests/repositories/fixtures/pypi.org/json/isort-metadata.json create mode 100644 tests/repositories/fixtures/pypi.org/json/isort-metadata/4.3.4.json create mode 100644 tests/repositories/link_sources/test_json.py diff --git a/src/poetry/repositories/http_repository.py b/src/poetry/repositories/http_repository.py index e80ddf624fd..127aded1d9b 100644 --- a/src/poetry/repositories/http_repository.py +++ b/src/poetry/repositories/http_repository.py @@ -295,17 +295,16 @@ def _get_info_from_urls( urls["sdist"][0], metadata ) or self._get_info_from_sdist(urls["sdist"][0]) - def _links_to_data(self, links: list[Link], data: PackageInfo) -> dict[str, Any]: - if not links: - raise PackageNotFound( - f'No valid distribution links found for package: "{data.name}" version:' - f' "{data.version}"' - ) + def _get_info_from_links( + self, + links: list[Link], + *, + ignore_yanked: bool = True, + ) -> PackageInfo: urls = defaultdict(list) metadata: dict[str, pkginfo.Distribution] = {} - files: list[dict[str, Any]] = [] for link in links: - if link.yanked and not data.yanked: + if link.yanked and ignore_yanked: # drop yanked files unless the entire release is yanked continue if link.has_metadata: @@ -342,6 +341,21 @@ def _links_to_data(self, links: list[Link], data: PackageInfo) -> dict[str, Any] ): urls["sdist"].append(link.url) + return self._get_info_from_urls(urls, metadata) + + def _links_to_data(self, links: list[Link], data: PackageInfo) -> dict[str, Any]: + if not links: + raise PackageNotFound( + f'No valid distribution links found for package: "{data.name}" version:' + f' "{data.version}"' + ) + + files: list[dict[str, Any]] = [] + for link in links: + if link.yanked and not data.yanked: + # drop yanked files unless the entire release is yanked + continue + file_hash: str | None for hash_name in ("sha512", "sha384", "sha256"): if hash_name in link.hashes: @@ -361,7 +375,8 @@ def _links_to_data(self, links: list[Link], data: PackageInfo) -> dict[str, Any] data.files = files - info = self._get_info_from_urls(urls, metadata) + # drop yanked files unless the entire release is yanked + info = self._get_info_from_links(links, ignore_yanked=not data.yanked) data.summary = info.summary data.requires_dist = info.requires_dist diff --git a/src/poetry/repositories/link_sources/json.py b/src/poetry/repositories/link_sources/json.py index 12b9c43ade1..f33a679ab28 100644 --- a/src/poetry/repositories/link_sources/json.py +++ b/src/poetry/repositories/link_sources/json.py @@ -28,7 +28,22 @@ def _link_cache(self) -> LinkCache: url = file["url"] requires_python = file.get("requires-python") yanked = file.get("yanked", False) - link = Link(url, requires_python=requires_python, yanked=yanked) + + # see https://peps.python.org/pep-0714/#clients + # and https://peps.python.org/pep-0691/#project-detail + metadata: dict[str, str] | bool = False + for metadata_key in ("core-metadata", "dist-info-metadata"): + if metadata_key in file: + metadata_value = file[metadata_key] + if metadata_value and isinstance(metadata_value, dict): + metadata = metadata_value + else: + metadata = bool(metadata_value) + break + + link = Link( + url, requires_python=requires_python, yanked=yanked, metadata=metadata + ) if link.ext not in self.SUPPORTED_FORMATS: continue diff --git a/src/poetry/repositories/pypi_repository.py b/src/poetry/repositories/pypi_repository.py index 468dc910836..066eb695335 100644 --- a/src/poetry/repositories/pypi_repository.py +++ b/src/poetry/repositories/pypi_repository.py @@ -2,7 +2,6 @@ import logging -from collections import defaultdict from typing import TYPE_CHECKING from typing import Any @@ -162,25 +161,18 @@ def _get_release_info( data.files = files if self._fallback and data.requires_dist is None: - self._log("No dependencies found, downloading archives", level="debug") + self._log( + "No dependencies found, downloading metadata and/or archives", + level="debug", + ) # No dependencies set (along with other information) # This might be due to actually no dependencies - # or badly set metadata when uploading + # or badly set metadata when uploading. # So, we need to make sure there is actually no - # dependencies by introspecting packages - urls = defaultdict(list) - for url in json_data["urls"]: - # Only get sdist and wheels if they exist - dist_type = url["packagetype"] - if dist_type not in SUPPORTED_PACKAGE_TYPES: - continue - - urls[dist_type].append(url["url"]) - - if not urls: - return data.asdict() - - info = self._get_info_from_urls(urls) + # dependencies by introspecting packages. + page = self.get_page(name) + links = list(page.links_for_version(name, version)) + info = self._get_info_from_links(links) data.requires_dist = info.requires_dist diff --git a/tests/repositories/conftest.py b/tests/repositories/conftest.py index 2661b46fc20..1f9a6d11946 100644 --- a/tests/repositories/conftest.py +++ b/tests/repositories/conftest.py @@ -1,12 +1,18 @@ from __future__ import annotations +import posixpath + +from pathlib import Path from typing import TYPE_CHECKING +from typing import Any import pytest +import requests if TYPE_CHECKING: from tests.types import HTMLPageGetter + from tests.types import RequestsSessionGet @pytest.fixture @@ -29,3 +35,25 @@ def _fixture(content: str, base_url: str | None = None) -> str: """ return _fixture + + +@pytest.fixture +def get_metadata_mock() -> RequestsSessionGet: + def metadata_mock(url: str, **__: Any) -> requests.Response: + if url.endswith(".metadata"): + response = requests.Response() + response.encoding = "application/text" + response._content = ( + ( + Path(__file__).parent + / "fixtures" + / "metadata" + / posixpath.basename(url) + ) + .read_text() + .encode() + ) + return response + raise requests.HTTPError() + + return metadata_mock diff --git a/tests/repositories/fixtures/legacy/metadata/isort-metadata-4.3.4-py3-none-any.whl.metadata b/tests/repositories/fixtures/metadata/isort-metadata-4.3.4-py2-none-any.whl.metadata similarity index 100% rename from tests/repositories/fixtures/legacy/metadata/isort-metadata-4.3.4-py3-none-any.whl.metadata rename to tests/repositories/fixtures/metadata/isort-metadata-4.3.4-py2-none-any.whl.metadata diff --git a/tests/repositories/fixtures/metadata/isort-metadata-4.3.4-py3-none-any.whl.metadata b/tests/repositories/fixtures/metadata/isort-metadata-4.3.4-py3-none-any.whl.metadata new file mode 100644 index 00000000000..bab8d017156 --- /dev/null +++ b/tests/repositories/fixtures/metadata/isort-metadata-4.3.4-py3-none-any.whl.metadata @@ -0,0 +1,28 @@ +Metadata-Version: 2.0 +Name: isort-metadata +Version: 4.3.4 +Summary: A Python utility / library to sort Python imports. +Home-page: https://github.com/timothycrosley/isort +Author: Timothy Crosley +Author-email: timothy.crosley@gmail.com +License: MIT +Keywords: Refactor,Python,Python2,Python3,Refactoring,Imports,Sort,Clean +Platform: UNKNOWN +Classifier: Development Status :: 6 - Mature +Classifier: Intended Audience :: Developers +Classifier: Natural Language :: English +Classifier: Environment :: Console +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Software Development :: Libraries +Classifier: Topic :: Utilities +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Requires-Dist: futures; python_version=="2.7" diff --git a/tests/repositories/fixtures/pypi.org/json/isort-metadata.json b/tests/repositories/fixtures/pypi.org/json/isort-metadata.json new file mode 100644 index 00000000000..7597a5d6895 --- /dev/null +++ b/tests/repositories/fixtures/pypi.org/json/isort-metadata.json @@ -0,0 +1,35 @@ +{ + "name": "isort-metadata", + "files": [ + { + "filename": "isort-metadata-4.3.4-py2-none-any.whl", + "url": "https://files.pythonhosted.org/packages/41/d8/a945da414f2adc1d9e2f7d6e7445b27f2be42766879062a2e63616ad4199/isort-metadata-4.3.4-py2-none-any.whl", + "core-metadata": true, + "hashes": { + "md5": "f0ad7704b6dc947073398ba290c3517f", + "sha256": "ec9ef8f4a9bc6f71eec99e1806bfa2de401650d996c59330782b89a5555c1497" + } + }, + { + "filename": "isort-metadata-4.3.4-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/1f/2c/22eee714d7199ae0464beda6ad5fedec8fee6a2f7ffd1e8f1840928fe318/isort-metadata-4.3.4-py3-none-any.whl", + "core-metadata": true, + "hashes": { + "md5": "fbaac4cd669ac21ea9e21ab1ea3180db", + "sha256": "1153601da39a25b14ddc54955dbbacbb6b2d19135386699e2ad58517953b34af" + } + }, + { + "filename": "isort-metadata-4.3.4.tar.gz", + "url": "https://files.pythonhosted.org/packages/b1/de/a628d16fdba0d38cafb3d7e34d4830f2c9cb3881384ce5c08c44762e1846/isort-metadata-4.3.4.tar.gz", + "hashes": { + "md5": "fb554e9c8f9aa76e333a03d470a5cf52", + "sha256": "b9c40e9750f3d77e6e4d441d8b0266cf555e7cdabdcff33c4fd06366ca761ef8" + } + } + ], + "meta": { + "api-version": "1.0", + "_last-serial": 3575149 + } +} diff --git a/tests/repositories/fixtures/pypi.org/json/isort-metadata/4.3.4.json b/tests/repositories/fixtures/pypi.org/json/isort-metadata/4.3.4.json new file mode 100644 index 00000000000..e08ac7272a5 --- /dev/null +++ b/tests/repositories/fixtures/pypi.org/json/isort-metadata/4.3.4.json @@ -0,0 +1,117 @@ +{ + "info": { + "author": "Timothy Crosley", + "author_email": "timothy.crosley@gmail.com", + "bugtrack_url": null, + "classifiers": [ + "Development Status :: 6 - Mature", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Software Development :: Libraries", + "Topic :: Utilities" + ], + "description": ".. image:: https://raw.github.com/timothycrosley/isort/master/logo.png\n :alt: isort\n\n########\n\n.. image:: https://badge.fury.io/py/isort.svg\n :target: https://badge.fury.io/py/isort\n :alt: PyPI version\n\n.. image:: https://travis-ci.org/timothycrosley/isort.svg?branch=master\n :target: https://travis-ci.org/timothycrosley/isort\n :alt: Build Status\n\n\n.. image:: https://coveralls.io/repos/timothycrosley/isort/badge.svg?branch=release%2F2.6.0&service=github\n :target: https://coveralls.io/github/timothycrosley/isort?branch=release%2F2.6.0\n :alt: Coverage\n\n.. image:: https://img.shields.io/github/license/mashape/apistatus.svg\n :target: https://pypi.python.org/pypi/hug/\n :alt: License\n\n.. image:: https://badges.gitter.im/Join%20Chat.svg\n :alt: Join the chat at https://gitter.im/timothycrosley/isort\n :target: https://gitter.im/timothycrosley/isort?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge\n\n\nisort your python imports for you so you don't have to.\n\nisort is a Python utility / library to sort imports alphabetically, and automatically separated into sections.\nIt provides a command line utility, Python library and `plugins for various editors `_ to quickly sort all your imports.\nIt currently cleanly supports Python 2.7 - 3.6 without any dependencies.\n\n.. image:: https://raw.github.com/timothycrosley/isort/develop/example.gif\n :alt: Example Usage\n\nBefore isort:\n\n.. code-block:: python\n\n from my_lib import Object\n\n print(\"Hey\")\n\n import os\n\n from my_lib import Object3\n\n from my_lib import Object2\n\n import sys\n\n from third_party import lib15, lib1, lib2, lib3, lib4, lib5, lib6, lib7, lib8, lib9, lib10, lib11, lib12, lib13, lib14\n\n import sys\n\n from __future__ import absolute_import\n\n from third_party import lib3\n\n print(\"yo\")\n\nAfter isort:\n\n.. code-block:: python\n\n from __future__ import absolute_import\n\n import os\n import sys\n\n from third_party import (lib1, lib2, lib3, lib4, lib5, lib6, lib7, lib8,\n lib9, lib10, lib11, lib12, lib13, lib14, lib15)\n\n from my_lib import Object, Object2, Object3\n\n print(\"Hey\")\n print(\"yo\")\n\nInstalling isort\n================\n\nInstalling isort is as simple as:\n\n.. code-block:: bash\n\n pip install isort\n\nor if you prefer\n\n.. code-block:: bash\n\n easy_install isort\n\nUsing isort\n===========\n\n**From the command line**:\n\n.. code-block:: bash\n\n isort mypythonfile.py mypythonfile2.py\n\nor recursively:\n\n.. code-block:: bash\n\n isort -rc .\n\n*which is equivalent to:*\n\n.. code-block:: bash\n\n isort **/*.py\n\nor to see the proposed changes without applying them:\n\n.. code-block:: bash\n\n isort mypythonfile.py --diff\n\nFinally, to atomically run isort against a project, only applying changes if they don't introduce syntax errors do:\n\n.. code-block:: bash\n\n isort -rc --atomic .\n\n(Note: this is disabled by default as it keeps isort from being able to run against code written using a different version of Python)\n\n**From within Python**:\n\n.. code-block:: bash\n\n from isort import SortImports\n\n SortImports(\"pythonfile.py\")\n\nor:\n\n.. code-block:: bash\n\n from isort import SortImports\n\n new_contents = SortImports(file_contents=old_contents).output\n\n**From within Kate:**\n\n.. code-block:: bash\n\n ctrl+[\n\nor:\n\n.. code-block:: bash\n\n menu > Python > Sort Imports\n\nInstalling isort's Kate plugin\n==============================\n\nFor KDE 4.13+ / Pate 2.0+:\n\n.. code-block:: bash\n\n wget https://raw.github.com/timothycrosley/isort/master/kate_plugin/isort_plugin.py --output-document ~/.kde/share/apps/kate/pate/isort_plugin.py\n wget https://raw.github.com/timothycrosley/isort/master/kate_plugin/isort_plugin_ui.rc --output-document ~/.kde/share/apps/kate/pate/isort_plugin_ui.rc\n wget https://raw.github.com/timothycrosley/isort/master/kate_plugin/katepart_isort.desktop --output-document ~/.kde/share/kde4/services/katepart_isort.desktop\n\nFor all older versions:\n\n.. code-block:: bash\n\n wget https://raw.github.com/timothycrosley/isort/master/kate_plugin/isort_plugin_old.py --output-document ~/.kde/share/apps/kate/pate/isort_plugin.py\n\nYou will then need to restart kate and enable Python Plugins as well as the isort plugin itself.\n\nInstalling isort's for your preferred text editor\n=================================================\n\nSeveral plugins have been written that enable to use isort from within a variety of text-editors.\nYou can find a full list of them `on the isort wiki `_.\nAdditionally, I will enthusiastically accept pull requests that include plugins for other text editors\nand add documentation for them as I am notified.\n\nHow does isort work?\n====================\n\nisort parses specified files for global level import lines (imports outside of try / except blocks, functions, etc..)\nand puts them all at the top of the file grouped together by the type of import:\n\n- Future\n- Python Standard Library\n- Third Party\n- Current Python Project\n- Explicitly Local (. before import, as in: ``from . import x``)\n- Custom Separate Sections (Defined by forced_separate list in configuration file)\n- Custom Sections (Defined by sections list in configuration file)\n\nInside of each section the imports are sorted alphabetically. isort automatically removes duplicate python imports,\nand wraps long from imports to the specified line length (defaults to 80).\n\nWhen will isort not work?\n=========================\n\nIf you ever have the situation where you need to have a try / except block in the middle of top-level imports or if\nyour import order is directly linked to precedence.\n\nFor example: a common practice in Django settings files is importing * from various settings files to form\na new settings file. In this case if any of the imports change order you are changing the settings definition itself.\n\nHowever, you can configure isort to skip over just these files - or even to force certain imports to the top.\n\nConfiguring isort\n=================\n\nIf you find the default isort settings do not work well for your project, isort provides several ways to adjust\nthe behavior.\n\nTo configure isort for a single user create a ``~/.isort.cfg`` file:\n\n.. code-block:: ini\n\n [settings]\n line_length=120\n force_to_top=file1.py,file2.py\n skip=file3.py,file4.py\n known_future_library=future,pies\n known_standard_library=std,std2\n known_third_party=randomthirdparty\n known_first_party=mylib1,mylib2\n indent=' '\n multi_line_output=3\n length_sort=1\n forced_separate=django.contrib,django.utils\n default_section=FIRSTPARTY\n no_lines_before=LOCALFOLDER\n\nAdditionally, you can specify project level configuration simply by placing a ``.isort.cfg`` file at the root of your\nproject. isort will look up to 25 directories up, from the file it is ran against, to find a project specific configuration.\n\nOr, if you prefer, you can add an isort section to your project's ``setup.cfg`` or ``tox.ini`` file with any desired settings.\n\nYou can then override any of these settings by using command line arguments, or by passing in override values to the\nSortImports class.\n\nFinally, as of version 3.0 isort supports editorconfig files using the standard syntax defined here:\nhttp://editorconfig.org/\n\nMeaning you place any standard isort configuration parameters within a .editorconfig file under the ``*.py`` section\nand they will be honored.\n\nFor a full list of isort settings and their meanings `take a look at the isort wiki `_.\n\nMulti line output modes\n=======================\n\nYou will notice above the \"multi_line_output\" setting. This setting defines how from imports wrap when they extend\npast the line_length limit and has 6 possible settings:\n\n**0 - Grid**\n\n.. code-block:: python\n\n from third_party import (lib1, lib2, lib3,\n lib4, lib5, ...)\n\n**1 - Vertical**\n\n.. code-block:: python\n\n from third_party import (lib1,\n lib2,\n lib3\n lib4,\n lib5,\n ...)\n\n**2 - Hanging Indent**\n\n.. code-block:: python\n\n from third_party import \\\n lib1, lib2, lib3, \\\n lib4, lib5, lib6\n\n**3 - Vertical Hanging Indent**\n\n.. code-block:: python\n\n from third_party import (\n lib1,\n lib2,\n lib3,\n lib4,\n )\n\n**4 - Hanging Grid**\n\n.. code-block:: python\n\n from third_party import (\n lib1, lib2, lib3, lib4,\n lib5, ...)\n\n**5 - Hanging Grid Grouped**\n\n.. code-block:: python\n\n from third_party import (\n lib1, lib2, lib3, lib4,\n lib5, ...\n )\n\n**6 - NOQA**\n\n.. code-block:: python\n\n from third_party import lib1, lib2, lib3, ... # NOQA\n\nAlternatively, you can set ``force_single_line`` to ``True`` (``-sl`` on the command line) and every import will appear on its\nown line:\n\n.. code-block:: python\n\n from third_party import lib1\n from third_party import lib2\n from third_party import lib3\n ...\n\nNote: to change the how constant indents appear - simply change the indent property with the following accepted formats:\n* Number of spaces you would like. For example: 4 would cause standard 4 space indentation.\n* Tab\n* A verbatim string with quotes around it.\n\nFor example:\n\n.. code-block:: python\n\n \" \"\n\nis equivalent to 4.\n\nFor the import styles that use parentheses, you can control whether or not to\ninclude a trailing comma after the last import with the ``include_trailing_comma``\noption (defaults to ``False``).\n\nIntelligently Balanced Multi-line Imports\n=========================================\n\nAs of isort 3.1.0 support for balanced multi-line imports has been added.\nWith this enabled isort will dynamically change the import length to the one that produces the most balanced grid,\nwhile staying below the maximum import length defined.\n\nExample:\n\n.. code-block:: python\n\n from __future__ import (absolute_import, division,\n print_function, unicode_literals)\n\nWill be produced instead of:\n\n.. code-block:: python\n\n from __future__ import (absolute_import, division, print_function,\n unicode_literals)\n\nTo enable this set ``balanced_wrapping`` to ``True`` in your config or pass the ``-e`` option into the command line utility.\n\nCustom Sections and Ordering\n============================\n\nYou can change the section order with ``sections`` option from the default of:\n\n.. code-block:: ini\n\n FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER\n\nto your preference:\n\n.. code-block:: ini\n\n sections=FUTURE,STDLIB,FIRSTPARTY,THIRDPARTY,LOCALFOLDER\n\nYou also can define your own sections and their order.\n\nExample:\n\n.. code-block:: ini\n\n known_django=django\n known_pandas=pandas,numpy\n sections=FUTURE,STDLIB,DJANGO,THIRDPARTY,PANDAS,FIRSTPARTY,LOCALFOLDER\n\nwould create two new sections with the specified known modules.\n\nThe ``no_lines_before`` option will prevent the listed sections from being split from the previous section by an empty line.\n\nExample:\n\n.. code-block:: ini\n\n sections=FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER\n no_lines_before=LOCALFOLDER\n\nwould produce a section with both FIRSTPARTY and LOCALFOLDER modules combined.\n\nAuto-comment import sections\n============================\n\nSome projects prefer to have import sections uniquely titled to aid in identifying the sections quickly\nwhen visually scanning. isort can automate this as well. To do this simply set the ``import_heading_{section_name}``\nsetting for each section you wish to have auto commented - to the desired comment.\n\nFor Example:\n\n.. code-block:: ini\n\n import_heading_stdlib=Standard Library\n import_heading_firstparty=My Stuff\n\nWould lead to output looking like the following:\n\n.. code-block:: python\n\n # Standard Library\n import os\n import sys\n\n import django.settings\n\n # My Stuff\n import myproject.test\n\nOrdering by import length\n=========================\n\nisort also makes it easy to sort your imports by length, simply by setting the ``length_sort`` option to ``True``.\nThis will result in the following output style:\n\n.. code-block:: python\n\n from evn.util import (\n Pool,\n Dict,\n Options,\n Constant,\n DecayDict,\n UnexpectedCodePath,\n )\n\nSkip processing of imports (outside of configuration)\n=====================================================\n\nTo make isort ignore a single import simply add a comment at the end of the import line containing the text ``isort:skip``:\n\n.. code-block:: python\n\n import module # isort:skip\n\nor:\n\n.. code-block:: python\n\n from xyz import (abc, # isort:skip\n yo,\n hey)\n\nTo make isort skip an entire file simply add ``isort:skip_file`` to the module's doc string:\n\n.. code-block:: python\n\n \"\"\" my_module.py\n Best module ever\n\n isort:skip_file\n \"\"\"\n\n import b\n import a\n\nAdding an import to multiple files\n==================================\n\nisort makes it easy to add an import statement across multiple files, while being assured it's correctly placed.\n\nFrom the command line:\n\n.. code-block:: bash\n\n isort -a \"from __future__ import print_function\" *.py\n\nfrom within Kate:\n\n.. code-block::\n\n ctrl+]\n\nor:\n\n.. code-block::\n\n menu > Python > Add Import\n\nRemoving an import from multiple files\n======================================\n\nisort also makes it easy to remove an import from multiple files, without having to be concerned with how it was originally\nformatted.\n\nFrom the command line:\n\n.. code-block:: bash\n\n isort -r \"os.system\" *.py\n\nfrom within Kate:\n\n.. code-block::\n\n ctrl+shift+]\n\nor:\n\n.. code-block::\n\n menu > Python > Remove Import\n\nUsing isort to verify code\n==========================\n\nThe ``--check-only`` option\n---------------------------\n\nisort can also be used to used to verify that code is correctly formatted by running it with ``-c``.\nAny files that contain incorrectly sorted and/or formatted imports will be outputted to ``stderr``.\n\n.. code-block:: bash\n\n isort **/*.py -c -vb\n\n SUCCESS: /home/timothy/Projects/Open_Source/isort/isort_kate_plugin.py Everything Looks Good!\n ERROR: /home/timothy/Projects/Open_Source/isort/isort/isort.py Imports are incorrectly sorted.\n\nOne great place this can be used is with a pre-commit git hook, such as this one by @acdha:\n\nhttps://gist.github.com/acdha/8717683\n\nThis can help to ensure a certain level of code quality throughout a project.\n\n\nGit hook\n--------\n\nisort provides a hook function that can be integrated into your Git pre-commit script to check\nPython code before committing.\n\nTo cause the commit to fail if there are isort errors (strict mode), include the following in\n``.git/hooks/pre-commit``:\n\n.. code-block:: python\n\n #!/usr/bin/env python\n import sys\n from isort.hooks import git_hook\n\n sys.exit(git_hook(strict=True))\n\nIf you just want to display warnings, but allow the commit to happen anyway, call ``git_hook`` without\nthe `strict` parameter.\n\nSetuptools integration\n----------------------\n\nUpon installation, isort enables a ``setuptools`` command that checks Python files\ndeclared by your project.\n\nRunning ``python setup.py isort`` on the command line will check the files\nlisted in your ``py_modules`` and ``packages``. If any warning is found,\nthe command will exit with an error code:\n\n.. code-block:: bash\n\n $ python setup.py isort\n\nAlso, to allow users to be able to use the command without having to install\nisort themselves, add isort to the setup_requires of your ``setup()`` like so:\n\n.. code-block:: python\n\n setup(\n name=\"project\",\n packages=[\"project\"],\n\n setup_requires=[\n \"isort\"\n ]\n )\n\n\nWhy isort?\n==========\n\nisort simply stands for import sort. It was originally called \"sortImports\" however I got tired of typing the extra\ncharacters and came to the realization camelCase is not pythonic.\n\nI wrote isort because in an organization I used to work in the manager came in one day and decided all code must\nhave alphabetically sorted imports. The code base was huge - and he meant for us to do it by hand. However, being a\nprogrammer - I'm too lazy to spend 8 hours mindlessly performing a function, but not too lazy to spend 16\nhours automating it. I was given permission to open source sortImports and here we are :)\n\n--------------------------------------------\n\nThanks and I hope you find isort useful!\n\n~Timothy Crosley\n", + "description_content_type": null, + "docs_url": null, + "download_url": "", + "downloads": { + "last_day": -1, + "last_month": -1, + "last_week": -1 + }, + "home_page": "https://github.com/timothycrosley/isort", + "keywords": "Refactor", + "license": "MIT", + "maintainer": "", + "maintainer_email": "", + "name": "isort", + "package_url": "https://pypi.org/project/isort/", + "platform": "", + "project_url": "https://pypi.org/project/isort/", + "project_urls": { + "Homepage": "https://github.com/timothycrosley/isort" + }, + "release_url": "https://pypi.org/project/isort/4.3.4/", + "requires_dist": null, + "requires_python": "", + "summary": "A Python utility / library to sort Python imports.", + "version": "4.3.4", + "yanked": false, + "yanked_reason": null + }, + "last_serial": 11968646, + "urls": [ + { + "comment_text": "", + "digests": { + "md5": "f0ad7704b6dc947073398ba290c3517f", + "sha256": "ec9ef8f4a9bc6f71eec99e1806bfa2de401650d996c59330782b89a5555c1497" + }, + "downloads": -1, + "filename": "isort-4.3.4-py2-none-any.whl", + "has_sig": false, + "md5_digest": "f0ad7704b6dc947073398ba290c3517f", + "packagetype": "bdist_wheel", + "python_version": "2.7", + "requires_python": null, + "size": 45393, + "upload_time": "2018-02-12T15:06:38", + "upload_time_iso_8601": "2018-02-12T15:06:38.441257Z", + "url": "https://files.pythonhosted.org/packages/41/d8/a945da414f2adc1d9e2f7d6e7445b27f2be42766879062a2e63616ad4199/isort-4.3.4-py2-none-any.whl", + "yanked": false, + "yanked_reason": null + }, + { + "comment_text": "", + "digests": { + "md5": "fbaac4cd669ac21ea9e21ab1ea3180db", + "sha256": "1153601da39a25b14ddc54955dbbacbb6b2d19135386699e2ad58517953b34af" + }, + "downloads": -1, + "filename": "isort-4.3.4-py3-none-any.whl", + "has_sig": false, + "md5_digest": "fbaac4cd669ac21ea9e21ab1ea3180db", + "packagetype": "bdist_wheel", + "python_version": "3.6", + "requires_python": null, + "size": 45352, + "upload_time": "2018-02-12T15:06:20", + "upload_time_iso_8601": "2018-02-12T15:06:20.089641Z", + "url": "https://files.pythonhosted.org/packages/1f/2c/22eee714d7199ae0464beda6ad5fedec8fee6a2f7ffd1e8f1840928fe318/isort-4.3.4-py3-none-any.whl", + "yanked": false, + "yanked_reason": null + }, + { + "comment_text": "", + "digests": { + "md5": "fb554e9c8f9aa76e333a03d470a5cf52", + "sha256": "b9c40e9750f3d77e6e4d441d8b0266cf555e7cdabdcff33c4fd06366ca761ef8" + }, + "downloads": -1, + "filename": "isort-4.3.4.tar.gz", + "has_sig": false, + "md5_digest": "fb554e9c8f9aa76e333a03d470a5cf52", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 56070, + "upload_time": "2018-02-12T15:06:16", + "upload_time_iso_8601": "2018-02-12T15:06:16.498194Z", + "url": "https://files.pythonhosted.org/packages/b1/de/a628d16fdba0d38cafb3d7e34d4830f2c9cb3881384ce5c08c44762e1846/isort-4.3.4.tar.gz", + "yanked": false, + "yanked_reason": null + } + ], + "vulnerabilities": [] +} diff --git a/tests/repositories/link_sources/test_json.py b/tests/repositories/link_sources/test_json.py new file mode 100644 index 00000000000..1f11d8b42d0 --- /dev/null +++ b/tests/repositories/link_sources/test_json.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import pytest + +from poetry.repositories.link_sources.json import SimpleJsonPage + + +@pytest.mark.parametrize( + ("metadata", "expected_has_metadata", "expected_metadata_hashes"), + [ + ({}, False, {}), + # new + ({"core-metadata": False}, False, {}), + ({"core-metadata": True}, True, {}), + ( + {"core-metadata": {"sha1": "1234", "sha256": "abcd"}}, + True, + {"sha1": "1234", "sha256": "abcd"}, + ), + ({"core-metadata": {}}, False, {}), + ( + {"core-metadata": {"sha1": "1234", "sha256": "abcd"}}, + True, + {"sha1": "1234", "sha256": "abcd"}, + ), + # old + ({"dist-info-metadata": False}, False, {}), + ({"dist-info-metadata": True}, True, {}), + ({"dist-info-metadata": {"sha256": "abcd"}}, True, {"sha256": "abcd"}), + ({"dist-info-metadata": {}}, False, {}), + ( + {"dist-info-metadata": {"sha1": "1234", "sha256": "abcd"}}, + True, + {"sha1": "1234", "sha256": "abcd"}, + ), + # conflicting (new wins) + ({"core-metadata": False, "dist-info-metadata": True}, False, {}), + ( + {"core-metadata": False, "dist-info-metadata": {"sha256": "abcd"}}, + False, + {}, + ), + ({"core-metadata": True, "dist-info-metadata": False}, True, {}), + ( + {"core-metadata": True, "dist-info-metadata": {"sha256": "abcd"}}, + True, + {}, + ), + ( + {"core-metadata": {"sha256": "abcd"}, "dist-info-metadata": False}, + True, + {"sha256": "abcd"}, + ), + ( + {"core-metadata": {"sha256": "abcd"}, "dist-info-metadata": True}, + True, + {"sha256": "abcd"}, + ), + ( + { + "core-metadata": {"sha256": "abcd"}, + "dist-info-metadata": {"sha256": "1234"}, + }, + True, + {"sha256": "abcd"}, + ), + ], +) +def test_metadata( + metadata: dict[str, bool | dict[str, str]], + expected_has_metadata: bool, + expected_metadata_hashes: dict[str, str], +) -> None: + content = {"files": [{"url": "https://example.org/demo-0.1.whl", **metadata}]} + page = SimpleJsonPage("https://example.org", content) + + link = next(page.links) + assert link.has_metadata is expected_has_metadata + assert link.metadata_hashes == expected_metadata_hashes diff --git a/tests/repositories/test_legacy_repository.py b/tests/repositories/test_legacy_repository.py index 478fffa4c86..e933e7708ff 100644 --- a/tests/repositories/test_legacy_repository.py +++ b/tests/repositories/test_legacy_repository.py @@ -1,7 +1,6 @@ from __future__ import annotations import base64 -import posixpath import re import shutil @@ -24,8 +23,6 @@ if TYPE_CHECKING: - from typing import Any - import httpretty from _pytest.monkeypatch import MonkeyPatch @@ -33,6 +30,7 @@ from pytest_mock import MockerFixture from poetry.config.config import Config + from tests.types import RequestsSessionGet @pytest.fixture(autouse=True) @@ -180,29 +178,24 @@ def test_get_package_information_fallback_read_setup() -> None: ) -def _get_mock(url: str, **__: Any) -> requests.Response: - if url.endswith(".metadata"): - response = requests.Response() - response.encoding = "application/text" - response._content = MockRepository.FIXTURES.joinpath( - "metadata", posixpath.basename(url) - ).read_text().encode() - return response - raise requests.HTTPError() - - -def test_get_package_information_pep_658(mocker: MockerFixture) -> None: +def test_get_package_information_pep_658( + mocker: MockerFixture, get_metadata_mock: RequestsSessionGet +) -> None: repo = MockRepository() isort_package = repo.package("isort", Version.parse("4.3.4")) - mocker.patch.object(repo.session, "get", _get_mock) + mocker.patch.object(repo.session, "get", get_metadata_mock) + spy = mocker.spy(repo, "_get_info_from_metadata") try: package = repo.package("isort-metadata", Version.parse("4.3.4")) except FileNotFoundError: pytest.fail("Metadata was not successfully retrieved") else: + assert spy.call_count > 0 + assert spy.spy_return is not None + assert package.source_type == isort_package.source_type == "legacy" assert package.source_reference == isort_package.source_reference == repo.name assert package.source_url == isort_package.source_url == repo.url diff --git a/tests/repositories/test_pypi_repository.py b/tests/repositories/test_pypi_repository.py index a5a437efac8..c53bb3cf555 100644 --- a/tests/repositories/test_pypi_repository.py +++ b/tests/repositories/test_pypi_repository.py @@ -26,6 +26,8 @@ from packaging.utils import NormalizedName from pytest_mock import MockerFixture + from tests.types import RequestsSessionGet + @pytest.fixture(autouse=True) def _use_simple_keyring(with_simple_keyring: None) -> None: @@ -247,6 +249,30 @@ def test_fallback_inspects_sdist_first_if_no_matching_wheels_can_be_found() -> N assert dep.python_versions == "~2.7" +def test_fallback_pep_658_metadata( + mocker: MockerFixture, get_metadata_mock: RequestsSessionGet +) -> None: + repo = MockRepository(fallback=True) + + mocker.patch.object(repo.session, "get", get_metadata_mock) + spy = mocker.spy(repo, "_get_info_from_metadata") + + try: + package = repo.package("isort-metadata", Version.parse("4.3.4")) + except FileNotFoundError: + pytest.fail("Metadata was not successfully retrieved") + else: + assert spy.call_count > 0 + assert spy.spy_return is not None + + assert package.name == "isort-metadata" + assert len(package.requires) == 1 + + dep = package.requires[0] + assert dep.name == "futures" + assert dep.python_versions == "~2.7" + + def test_fallback_can_read_setup_to_get_dependencies() -> None: repo = MockRepository(fallback=True) diff --git a/tests/types.py b/tests/types.py index 63392ccc6ce..0d9605c56dd 100644 --- a/tests/types.py +++ b/tests/types.py @@ -8,6 +8,8 @@ if TYPE_CHECKING: from pathlib import Path + import requests + from cleo.io.io import IO from cleo.testers.command_tester import CommandTester @@ -61,3 +63,7 @@ def __call__(self, relative_path: str, target: Path | None = None) -> Path: ... class HTMLPageGetter(Protocol): def __call__(self, content: str, base_url: str | None = None) -> str: ... + + +class RequestsSessionGet(Protocol): + def __call__(self, url: str, **__: Any) -> requests.Response: ... From 6c9bc2805c3d8789dc8678e7077ffeba6c7c4b8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Randy=20D=C3=B6ring?= <30527984+radoering@users.noreply.github.com> Date: Mon, 20 Nov 2023 17:43:42 +0100 Subject: [PATCH 5/7] only download metadata we need (instead of all metadata) --- src/poetry/repositories/http_repository.py | 153 ++++++++++----------- tests/repositories/test_http_repository.py | 17 +-- 2 files changed, 78 insertions(+), 92 deletions(-) diff --git a/src/poetry/repositories/http_repository.py b/src/poetry/repositories/http_repository.py index 127aded1d9b..5373e854c5e 100644 --- a/src/poetry/repositories/http_repository.py +++ b/src/poetry/repositories/http_repository.py @@ -3,7 +3,6 @@ import functools import hashlib -from collections import defaultdict from contextlib import contextmanager from pathlib import Path from typing import TYPE_CHECKING @@ -16,7 +15,6 @@ from poetry.core.constraints.version import parse_constraint from poetry.core.packages.dependency import Dependency -from poetry.core.packages.utils.link import Link from poetry.core.utils.helpers import temporary_directory from poetry.core.version.markers import parse_marker @@ -38,6 +36,7 @@ if TYPE_CHECKING: from packaging.utils import NormalizedName + from poetry.core.packages.utils.link import Link from poetry.repositories.link_sources.base import LinkSource from poetry.utils.authenticator import RepositoryCertificateConfig @@ -110,10 +109,9 @@ def _cached_or_downloaded_file( ) yield filepath - def _get_info_from_wheel(self, url: str) -> PackageInfo: + def _get_info_from_wheel(self, link: Link) -> PackageInfo: from poetry.inspection.info import PackageInfo - link = Link(url) netloc = link.netloc # If "lazy-wheel" is enabled and the domain supports range requests @@ -148,37 +146,73 @@ def _get_info_from_wheel(self, url: str) -> PackageInfo: level="debug", ) self._supports_range_requests[netloc] = True - return self._get_info_from_wheel(link.url) + return self._get_info_from_wheel(link) - def _get_info_from_sdist(self, url: str) -> PackageInfo: + def _get_info_from_sdist(self, link: Link) -> PackageInfo: from poetry.inspection.info import PackageInfo - with self._cached_or_downloaded_file(Link(url)) as filepath: + with self._cached_or_downloaded_file(link) as filepath: return PackageInfo.from_sdist(filepath) - @staticmethod - def _get_info_from_metadata( - url: str, metadata: dict[str, pkginfo.Distribution] - ) -> PackageInfo | None: - if url in metadata: - dist = metadata[url] - return PackageInfo( - name=dist.name, - version=dist.version, - summary=dist.summary, - requires_dist=list(dist.requires_dist), - requires_python=dist.requires_python, - ) + def _get_info_from_metadata(self, link: Link) -> PackageInfo | None: + if link.has_metadata: + try: + assert link.metadata_url is not None + response = self.session.get(link.metadata_url) + distribution = pkginfo.Distribution() + if link.metadata_hash_name is not None: + metadata_hash = getattr(hashlib, link.metadata_hash_name)( + response.text.encode() + ).hexdigest() + + if metadata_hash != link.metadata_hash: + self._log( + f"Metadata file hash ({metadata_hash}) does not match" + f" expected hash ({link.metadata_hash})." + f" Metadata file for {link.filename} will be ignored.", + level="warning", + ) + return None + + distribution.parse(response.content) + return PackageInfo( + name=distribution.name, + version=distribution.version, + summary=distribution.summary, + requires_dist=list(distribution.requires_dist), + requires_python=distribution.requires_python, + ) + + except requests.HTTPError: + self._log( + f"Failed to retrieve metadata at {link.metadata_url}", + level="warning", + ) + return None - def _get_info_from_urls( + def _get_info_from_links( self, - urls: dict[str, list[str]], - metadata: dict[str, pkginfo.Distribution] | None = None, + links: list[Link], + *, + ignore_yanked: bool = True, ) -> PackageInfo: - metadata = metadata or {} + # Sort links by distribution type + wheels: list[Link] = [] + sdists: list[Link] = [] + for link in links: + if link.yanked and ignore_yanked: + # drop yanked files unless the entire release is yanked + continue + if link.is_wheel: + wheels.append(link) + elif link.filename.endswith( + (".tar.gz", ".zip", ".bz2", ".xz", ".Z", ".tar") + ): + sdists.append(link) + # Prefer to read data from wheels: this is faster and more reliable - if wheels := urls.get("bdist_wheel"): + if wheels: # We ought just to be able to look at any of the available wheels to read # metadata, they all should give the same answer. # @@ -193,8 +227,7 @@ def _get_info_from_urls( universal_python3_wheel = None platform_specific_wheels = [] for wheel in wheels: - link = Link(wheel) - m = wheel_file_re.match(link.filename) + m = wheel_file_re.match(wheel.filename) if not m: continue @@ -215,17 +248,17 @@ def _get_info_from_urls( if universal_wheel is not None: return self._get_info_from_metadata( - universal_wheel, metadata + universal_wheel ) or self._get_info_from_wheel(universal_wheel) info = None if universal_python2_wheel and universal_python3_wheel: info = self._get_info_from_metadata( - universal_python2_wheel, metadata + universal_python2_wheel ) or self._get_info_from_wheel(universal_python2_wheel) py3_info = self._get_info_from_metadata( - universal_python3_wheel, metadata + universal_python3_wheel ) or self._get_info_from_wheel(universal_python3_wheel) if info.requires_python or py3_info.requires_python: @@ -277,71 +310,23 @@ def _get_info_from_urls( # Prefer non platform specific wheels if universal_python3_wheel: return self._get_info_from_metadata( - universal_python3_wheel, metadata + universal_python3_wheel ) or self._get_info_from_wheel(universal_python3_wheel) if universal_python2_wheel: return self._get_info_from_metadata( - universal_python2_wheel, metadata + universal_python2_wheel ) or self._get_info_from_wheel(universal_python2_wheel) if platform_specific_wheels: first_wheel = platform_specific_wheels[0] return self._get_info_from_metadata( - first_wheel, metadata + first_wheel ) or self._get_info_from_wheel(first_wheel) - return self._get_info_from_metadata( - urls["sdist"][0], metadata - ) or self._get_info_from_sdist(urls["sdist"][0]) - - def _get_info_from_links( - self, - links: list[Link], - *, - ignore_yanked: bool = True, - ) -> PackageInfo: - urls = defaultdict(list) - metadata: dict[str, pkginfo.Distribution] = {} - for link in links: - if link.yanked and ignore_yanked: - # drop yanked files unless the entire release is yanked - continue - if link.has_metadata: - try: - assert link.metadata_url is not None - response = self.session.get(link.metadata_url) - distribution = pkginfo.Distribution() - if link.metadata_hash_name is not None: - metadata_hash = getattr(hashlib, link.metadata_hash_name)( - response.text.encode() - ).hexdigest() - - if metadata_hash != link.metadata_hash: - self._log( - f"Metadata file hash ({metadata_hash}) does not match" - f" expected hash ({link.metadata_hash})." - f" Metadata file for {link.filename} will be ignored.", - level="warning", - ) - continue - - distribution.parse(response.content) - metadata[link.url] = distribution - except requests.HTTPError: - self._log( - f"Failed to retrieve metadata at {link.metadata_url}", - level="warning", - ) - - if link.is_wheel: - urls["bdist_wheel"].append(link.url) - elif link.filename.endswith( - (".tar.gz", ".zip", ".bz2", ".xz", ".Z", ".tar") - ): - urls["sdist"].append(link.url) - - return self._get_info_from_urls(urls, metadata) + return self._get_info_from_metadata(sdists[0]) or self._get_info_from_sdist( + sdists[0] + ) def _links_to_data(self, links: list[Link], data: PackageInfo) -> dict[str, Any]: if not links: diff --git a/tests/repositories/test_http_repository.py b/tests/repositories/test_http_repository.py index c1221f7b07c..eb0de86819d 100644 --- a/tests/repositories/test_http_repository.py +++ b/tests/repositories/test_http_repository.py @@ -10,6 +10,7 @@ import pytest from packaging.metadata import parse_email +from poetry.core.packages.utils.link import Link from poetry.inspection.lazy_wheel import HTTPRangeRequestUnsupported from poetry.repositories.http_repository import HTTPRepository @@ -61,7 +62,7 @@ def test_get_info_from_wheel( if lazy_wheel and supports_range_requests is not None: repo._supports_range_requests[domain] = supports_range_requests - info = repo._get_info_from_wheel(url) + info = repo._get_info_from_wheel(Link(url)) assert info.name == "poetry-core" assert info.version == "1.5.0" assert info.requires_dist == [ @@ -110,18 +111,18 @@ def test_get_info_from_wheel_state_sequence(mocker: MockerFixture) -> None: filename = "poetry_core-1.5.0-py3-none-any.whl" domain = "foo.com" - url = f"https://{domain}/{filename}" + link = Link(f"https://{domain}/{filename}") repo = MockRepository() # 1. range request and download mock_metadata_from_wheel_url.side_effect = HTTPRangeRequestUnsupported - repo._get_info_from_wheel(url) + repo._get_info_from_wheel(link) assert mock_metadata_from_wheel_url.call_count == 1 assert mock_download.call_count == 1 assert mock_download.call_args[1]["raise_accepts_ranges"] is False # 2. only download - repo._get_info_from_wheel(url) + repo._get_info_from_wheel(link) assert mock_metadata_from_wheel_url.call_count == 1 assert mock_download.call_count == 2 assert mock_download.call_args[1]["raise_accepts_ranges"] is True @@ -129,26 +130,26 @@ def test_get_info_from_wheel_state_sequence(mocker: MockerFixture) -> None: # 3. download and range request mock_metadata_from_wheel_url.side_effect = None mock_download.side_effect = HTTPRangeRequestSupported - repo._get_info_from_wheel(url) + repo._get_info_from_wheel(link) assert mock_metadata_from_wheel_url.call_count == 2 assert mock_download.call_count == 3 assert mock_download.call_args[1]["raise_accepts_ranges"] is True # 4. only range request - repo._get_info_from_wheel(url) + repo._get_info_from_wheel(link) assert mock_metadata_from_wheel_url.call_count == 3 assert mock_download.call_count == 3 # 5. range request and download mock_metadata_from_wheel_url.side_effect = HTTPRangeRequestUnsupported mock_download.side_effect = None - repo._get_info_from_wheel(url) + repo._get_info_from_wheel(link) assert mock_metadata_from_wheel_url.call_count == 4 assert mock_download.call_count == 4 assert mock_download.call_args[1]["raise_accepts_ranges"] is False # 6. only range request mock_metadata_from_wheel_url.side_effect = None - repo._get_info_from_wheel(url) + repo._get_info_from_wheel(link) assert mock_metadata_from_wheel_url.call_count == 5 assert mock_download.call_count == 4 From 971f78048f5296bde9ae9eb0e7de10ce2fc8da34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Randy=20D=C3=B6ring?= <30527984+radoering@users.noreply.github.com> Date: Mon, 5 Feb 2024 17:36:49 +0100 Subject: [PATCH 6/7] handle deprecation of link.metadata_hash --- src/poetry/repositories/http_repository.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/poetry/repositories/http_repository.py b/src/poetry/repositories/http_repository.py index 5373e854c5e..50d40680fbc 100644 --- a/src/poetry/repositories/http_repository.py +++ b/src/poetry/repositories/http_repository.py @@ -160,15 +160,18 @@ def _get_info_from_metadata(self, link: Link) -> PackageInfo | None: assert link.metadata_url is not None response = self.session.get(link.metadata_url) distribution = pkginfo.Distribution() - if link.metadata_hash_name is not None: - metadata_hash = getattr(hashlib, link.metadata_hash_name)( + if link.metadata_hashes and ( + hash_name := get_highest_priority_hash_type( + set(link.metadata_hashes.keys()), f"{link.filename}.metadata" + ) + ): + metadata_hash = getattr(hashlib, hash_name)( response.text.encode() ).hexdigest() - - if metadata_hash != link.metadata_hash: + if metadata_hash != link.metadata_hashes[hash_name]: self._log( f"Metadata file hash ({metadata_hash}) does not match" - f" expected hash ({link.metadata_hash})." + f" expected hash ({link.metadata_hashes[hash_name]})." f" Metadata file for {link.filename} will be ignored.", level="warning", ) From b1bcbc24ef4f73ed5d461b2c594bf85343e92562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Randy=20D=C3=B6ring?= <30527984+radoering@users.noreply.github.com> Date: Mon, 5 Feb 2024 17:42:07 +0100 Subject: [PATCH 7/7] use packaging.metadata instead of pkginfo --- src/poetry/repositories/http_repository.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/poetry/repositories/http_repository.py b/src/poetry/repositories/http_repository.py index 50d40680fbc..788f7cef9ae 100644 --- a/src/poetry/repositories/http_repository.py +++ b/src/poetry/repositories/http_repository.py @@ -9,10 +9,10 @@ from typing import Any from typing import Iterator -import pkginfo import requests import requests.adapters +from packaging.metadata import parse_email from poetry.core.constraints.version import parse_constraint from poetry.core.packages.dependency import Dependency from poetry.core.utils.helpers import temporary_directory @@ -159,7 +159,6 @@ def _get_info_from_metadata(self, link: Link) -> PackageInfo | None: try: assert link.metadata_url is not None response = self.session.get(link.metadata_url) - distribution = pkginfo.Distribution() if link.metadata_hashes and ( hash_name := get_highest_priority_hash_type( set(link.metadata_hashes.keys()), f"{link.filename}.metadata" @@ -177,14 +176,8 @@ def _get_info_from_metadata(self, link: Link) -> PackageInfo | None: ) return None - distribution.parse(response.content) - return PackageInfo( - name=distribution.name, - version=distribution.version, - summary=distribution.summary, - requires_dist=list(distribution.requires_dist), - requires_python=distribution.requires_python, - ) + metadata, _ = parse_email(response.content) + return PackageInfo.from_metadata(metadata) except requests.HTTPError: self._log(