stop using the json API at pypi

python-poetry · Mar 18, 2024 · c3d7060 · c3d7060
1 parent 1b0caf5
commit c3d7060
Show file tree

Hide file tree

Showing 172 changed files with 68,776 additions and 5,668 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,2 @@
+# Do not mess with line endings in metadata files or the hash will be wrong.
+*.metadata binary
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -6,8 +6,9 @@ repos:
     rev: v4.5.0
     hooks:
       - id: trailing-whitespace
+        exclude: tests/repositories/fixtures/pypi\.org/metadata/.*\.metadata
       - id: end-of-file-fixer
-        exclude: ^.*\.egg-info/
+        exclude: ^.*\.egg-info/|tests/repositories/fixtures/pypi\.org/metadata/.*\.metadata
       - id: check-merge-conflict
       - id: check-case-conflict
       - id: check-json

diff --git a/src/poetry/repositories/http_repository.py b/src/poetry/repositories/http_repository.py
@@ -36,6 +36,8 @@
 
 if TYPE_CHECKING:
     from packaging.utils import NormalizedName
+    from poetry.core.constraints.version import Version
+    from poetry.core.packages.package import Package
     from poetry.core.packages.utils.link import Link
 
     from poetry.repositories.link_sources.base import LinkSource
@@ -90,6 +92,36 @@ def certificates(self) -> RepositoryCertificateConfig:
     def authenticated_url(self) -> str:
         return self._authenticator.authenticated_url(url=self.url)
 
+    def find_links_for_package(self, package: Package) -> list[Link]:
+        try:
+            page = self.get_page(package.name)
+        except PackageNotFound:
+            return []
+
+        return list(page.links_for_version(package.name, package.version))
+
+    def _get_release_info(
+        self, name: NormalizedName, version: Version
+    ) -> dict[str, Any]:
+        page = self.get_page(name)
+
+        links = list(page.links_for_version(name, version))
+        yanked = page.yanked(name, version)
+
+        return self._links_to_data(
+            links,
+            PackageInfo(
+                name=name,
+                version=version.text,
+                summary="",
+                requires_dist=[],
+                requires_python=None,
+                files=[],
+                yanked=yanked,
+                cache_version=str(self.CACHE_VERSION),
+            ),
+        )
+
     def _download(
         self, url: str, dest: Path, *, raise_accepts_ranges: bool = False
     ) -> None:
@@ -158,19 +190,21 @@ def _get_info_from_sdist(self, link: Link) -> PackageInfo:
         with self._cached_or_downloaded_file(link) as filepath:
             return PackageInfo.from_sdist(filepath)
 
+    def _get_metadata(self, metadata_url: str) -> bytes:
+        response = self.session.get(metadata_url)
+        return response.content
+
     def _get_info_from_metadata(self, link: Link) -> PackageInfo | None:
         if link.has_metadata:
             try:
                 assert link.metadata_url is not None
-                response = self.session.get(link.metadata_url)
+                data = self._get_metadata(link.metadata_url)
                 if link.metadata_hashes and (
                     hash_name := get_highest_priority_hash_type(
                         set(link.metadata_hashes.keys()), f"{link.filename}.metadata"
                     )
                 ):
-                    metadata_hash = getattr(hashlib, hash_name)(
-                        response.content
-                    ).hexdigest()
+                    metadata_hash = getattr(hashlib, hash_name)(data).hexdigest()
                     if metadata_hash != link.metadata_hashes[hash_name]:
                         self._log(
                             f"Metadata file hash ({metadata_hash}) does not match"
@@ -180,7 +214,7 @@ def _get_info_from_metadata(self, link: Link) -> PackageInfo | None:
                         )
                         return None
 
-                metadata, _ = parse_email(response.content)
+                metadata, _ = parse_email(data)
                 return PackageInfo.from_metadata(metadata)
 
             except requests.HTTPError:

diff --git a/src/poetry/repositories/legacy_repository.py b/src/poetry/repositories/legacy_repository.py
@@ -3,13 +3,11 @@
 from contextlib import suppress
 from functools import cached_property
 from typing import TYPE_CHECKING
-from typing import Any
 
 import requests.adapters
 
 from poetry.core.packages.package import Package
 
-from poetry.inspection.info import PackageInfo
 from poetry.repositories.exceptions import PackageNotFound
 from poetry.repositories.http_repository import HTTPRepository
 from poetry.repositories.link_sources.html import SimpleRepositoryPage
@@ -20,7 +18,6 @@
     from packaging.utils import NormalizedName
     from poetry.core.constraints.version import Version
     from poetry.core.constraints.version import VersionConstraint
-    from poetry.core.packages.utils.link import Link
 
     from poetry.config.config import Config
 
@@ -65,14 +62,6 @@ def package(
 
             return package
 
-    def find_links_for_package(self, package: Package) -> list[Link]:
-        try:
-            page = self.get_page(package.name)
-        except PackageNotFound:
-            return []
-
-        return list(page.links_for_version(package.name, package.version))
-
     def _find_packages(
         self, name: NormalizedName, constraint: VersionConstraint
     ) -> list[Package]:
@@ -103,28 +92,6 @@ def _find_packages(
             for version, yanked in versions
         ]
 
-    def _get_release_info(
-        self, name: NormalizedName, version: Version
-    ) -> dict[str, Any]:
-        page = self.get_page(name)
-
-        links = list(page.links_for_version(name, version))
-        yanked = page.yanked(name, version)
-
-        return self._links_to_data(
-            links,
-            PackageInfo(
-                name=name,
-                version=version.text,
-                summary="",
-                requires_dist=[],
-                requires_python=None,
-                files=[],
-                yanked=yanked,
-                cache_version=str(self.CACHE_VERSION),
-            ),
-        )
-
     def _get_page(self, name: NormalizedName) -> SimpleRepositoryPage:
         if not (response := self._get_response(f"/{name}/")):
             raise PackageNotFound(f"Package [{name}] not found.")

diff --git a/src/poetry/repositories/link_sources/json.py b/src/poetry/repositories/link_sources/json.py
@@ -41,8 +41,13 @@ def _link_cache(self) -> LinkCache:
                         metadata = bool(metadata_value)
                     break
 
+            hashes = file.get("hashes")
             link = Link(
-                url, requires_python=requires_python, yanked=yanked, metadata=metadata
+                url,
+                requires_python=requires_python,
+                hashes=hashes,
+                yanked=yanked,
+                metadata=metadata,
             )
 
             if link.ext not in self.SUPPORTED_FORMATS:

diff --git a/src/poetry/repositories/pypi_repository.py b/src/poetry/repositories/pypi_repository.py
@@ -10,7 +10,6 @@
 
 from cachecontrol.controller import logger as cache_control_logger
 from poetry.core.packages.package import Package
-from poetry.core.packages.utils.link import Link
 from poetry.core.version.exceptions import InvalidVersion
 
 from poetry.repositories.exceptions import PackageNotFound
@@ -26,18 +25,14 @@
 
 if TYPE_CHECKING:
     from packaging.utils import NormalizedName
-    from poetry.core.constraints.version import Version
     from poetry.core.constraints.version import VersionConstraint
 
-SUPPORTED_PACKAGE_TYPES = {"sdist", "bdist_wheel"}
-
 
 class PyPiRepository(HTTPRepository):
     def __init__(
         self,
         url: str = "https://pypi.org/",
         disable_cache: bool = False,
-        fallback: bool = True,
         pool_size: int = requests.adapters.DEFAULT_POOLSIZE,
     ) -> None:
         super().__init__(
@@ -48,7 +43,6 @@ def __init__(
         )
 
         self._base_url = url
-        self._fallback = fallback
 
     def search(self, query: str) -> list[Package]:
         results = []
@@ -110,79 +104,6 @@ def _get_package_info(self, name: NormalizedName) -> dict[str, Any]:
 
         return info
 
-    def find_links_for_package(self, package: Package) -> list[Link]:
-        json_data = self._get(f"pypi/{package.name}/{package.version}/json")
-        if json_data is None:
-            return []
-
-        links = []
-        for url in json_data["urls"]:
-            if url["packagetype"] in SUPPORTED_PACKAGE_TYPES:
-                h = f"sha256={url['digests']['sha256']}"
-                links.append(Link(url["url"] + "#" + h, yanked=self._get_yanked(url)))
-
-        return links
-
-    def _get_release_info(
-        self, name: NormalizedName, version: Version
-    ) -> dict[str, Any]:
-        from poetry.inspection.info import PackageInfo
-
-        self._log(f"Getting info for {name} ({version}) from PyPI", "debug")
-
-        json_data = self._get(f"pypi/{name}/{version}/json")
-        if json_data is None:
-            raise PackageNotFound(f"Package [{name}] not found.")
-
-        info = json_data["info"]
-
-        data = PackageInfo(
-            name=info["name"],
-            version=info["version"],
-            summary=info["summary"],
-            requires_dist=info["requires_dist"],
-            requires_python=info["requires_python"],
-            yanked=self._get_yanked(info),
-            cache_version=str(self.CACHE_VERSION),
-        )
-
-        try:
-            version_info = json_data["urls"]
-        except KeyError:
-            version_info = []
-
-        files = info.get("files", [])
-        for file_info in version_info:
-            if file_info["packagetype"] in SUPPORTED_PACKAGE_TYPES:
-                files.append(
-                    {
-                        "file": file_info["filename"],
-                        "hash": "sha256:" + file_info["digests"]["sha256"],
-                    }
-                )
-        data.files = files
-
-        if self._fallback and data.requires_dist is None:
-            self._log(
-                "No dependencies found, downloading metadata and/or archives",
-                level="debug",
-            )
-            # No dependencies set (along with other information)
-            # This might be due to actually no dependencies
-            # or badly set metadata when uploading.
-            # So, we need to make sure there is actually no
-            # dependencies by introspecting packages.
-            page = self.get_page(name)
-            links = list(page.links_for_version(name, version))
-            info = self._get_info_from_links(links)
-
-            data.requires_dist = info.requires_dist
-
-            if not data.requires_python:
-                data.requires_python = info.requires_python
-
-        return data.asdict()
-
     def _get_page(self, name: NormalizedName) -> SimpleJsonPage:
         source = self._base_url + f"simple/{name}/"
         info = self.get_package_info(name)

diff --git a/tests/installation/fixtures/with-pypi-repository.test b/tests/installation/fixtures/with-pypi-repository.test
@@ -7,8 +7,8 @@ description = "Classes Without Boilerplate"
 optional = false
 python-versions = "*"
 files = [
-    {file = "attrs-17.4.0-py2.py3-none-any.whl", hash = "sha256:d38e57f381e891928357c68e300d28d3d4dcddc50486d5f8dfaf743d40477619"},
-    {file = "attrs-17.4.0.tar.gz", hash = "sha256:eb7536a1e6928190b3008c5b350bdf9850d619fff212341cd096f87a27a5e564"},
+    {file = "attrs-17.4.0-py2.py3-none-any.whl", hash = "sha256:a17a9573a6f475c99b551c0e0a812707ddda1ec9653bed04c13841404ed6f450"},
+    {file = "attrs-17.4.0.tar.gz", hash = "sha256:1c7960ccfd6a005cd9f7ba884e6316b5e430a3f1a6c37c5f87d8b43f83b54ec9"},
 ]
 
 [package.extras]
@@ -23,8 +23,8 @@ description = "Cross-platform colored terminal text."
 optional = false
 python-versions = "*"
 files = [
-    {file = "colorama-0.3.9-py2.py3-none-any.whl", hash = "sha256:5b632359f1ed2b7676a869812ba0edaacb99be04679b29eb56c07a5e137ab5a2"},
-    {file = "colorama-0.3.9.tar.gz", hash = "sha256:4c5a15209723ce1330a5c193465fe221098f761e9640d823a2ce7c03f983137f"},
+    {file = "colorama-0.3.9-py2.py3-none-any.whl", hash = "sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda"},
+    {file = "colorama-0.3.9.tar.gz", hash = "sha256:48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1"},
 ]
 
 [[package]]
@@ -34,9 +34,9 @@ description = "More routines for operating on iterables, beyond itertools"
 optional = false
 python-versions = "*"
 files = [
-    {file = "more-itertools-4.1.0.tar.gz", hash = "sha256:bab2dc6f4be8f9a4a72177842c5283e2dff57c167439a03e3d8d901e854f0f2e"},
-    {file = "more_itertools-4.1.0-py2-none-any.whl", hash = "sha256:5dd7dfd88d2fdaea446da478ffef8d7151fdf26ee92ac7ed7b14e8d71efe4b62"},
-    {file = "more_itertools-4.1.0-py3-none-any.whl", hash = "sha256:29b1e1661aaa56875ce090fa219fa84dfc13daecb52cd4fae321f6f57b419ec4"},
+    {file = "more-itertools-4.1.0.tar.gz", hash = "sha256:c9ce7eccdcb901a2c75d326ea134e0886abfbea5f93e91cc95de9507c0816c44"},
+    {file = "more_itertools-4.1.0-py2-none-any.whl", hash = "sha256:11a625025954c20145b37ff6309cd54e39ca94f72f6bb9576d1195db6fa2442e"},
+    {file = "more_itertools-4.1.0-py3-none-any.whl", hash = "sha256:0dd8f72eeab0d2c3bd489025bb2f6a1b8342f9b198f6fc37b52d15cfa4531fea"},
 ]
 
 [package.dependencies]
@@ -49,7 +49,7 @@ description = "plugin and hook calling mechanisms for python"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
-    {file = "pluggy-0.6.0.tar.gz", hash = "sha256:a982e208d054867661d27c6d2a86b17ba05fbb6b1bdc01f42660732dd107f865"},
+    {file = "pluggy-0.6.0.tar.gz", hash = "sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff"},
 ]
 
 [[package]]
@@ -59,19 +59,19 @@ description = "library with cross-python path, ini-parsing, io, code, log facili
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
-    {file = "py-1.5.3-py2.py3-none-any.whl", hash = "sha256:43ee6c7f95e0ec6a906de49906b79d138d89728fff17109d49f086abc2fdd985"},
-    {file = "py-1.5.3.tar.gz", hash = "sha256:2df2c513c3af11de15f58189ba5539ddc4768c6f33816dc5c03950c8bd6180fa"},
+    {file = "py-1.5.3-py2.py3-none-any.whl", hash = "sha256:983f77f3331356039fdd792e9220b7b8ee1aa6bd2b25f567a963ff1de5a64f6a"},
+    {file = "py-1.5.3.tar.gz", hash = "sha256:29c9fab495d7528e80ba1e343b958684f4ace687327e6f789a94bf3d1915f881"},
 ]
 
 [[package]]
 name = "pytest"
-version = "3.5.0"
+version = "3.5.1"
 description = "pytest: simple powerful testing with Python"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
-    {file = "pytest-3.5.0-py2.py3-none-any.whl", hash = "sha256:28e4d9c2ae3196d74805c2eba24f350ae4c791a5b9b397c79b41506a48dc64ca"},
-    {file = "pytest-3.5.0.tar.gz", hash = "sha256:677b1d6decd29c041fe64276f29f79fbe66e40c59e445eb251366b4a8ab8bf68"},
+    {file = "pytest-3.5.1-py2.py3-none-any.whl", hash = "sha256:829230122facf05a5f81a6d4dfe6454a04978ea3746853b2b84567ecf8e5c526"},
+    {file = "pytest-3.5.1.tar.gz", hash = "sha256:54713b26c97538db6ff0703a12b19aeaeb60b5e599de542e7fca0ec83b9038e8"},
 ]
 
 [package.dependencies]
@@ -91,7 +91,7 @@ optional = false
 python-versions = ">=3.7"
 files = [
     {file = "setuptools-67.6.1-py3-none-any.whl", hash = "sha256:e728ca814a823bf7bf60162daf9db95b93d532948c4c0bea762ce62f60189078"},
-    {file = "setuptools-67.6.1.tar.gz", hash = "sha256:a737d365c957dd3fced9ddd246118e95dce7a62c3dc49f37e7fdd9e93475d785"},
+    {file = "setuptools-67.6.1.tar.gz", hash = "sha256:257de92a9d50a60b8e22abfcbb771571fde0dbf3ec234463212027a4eeecbe9a"},
 ]
 
 [package.extras]
@@ -106,8 +106,8 @@ description = "Python 2 and 3 compatibility utilities"
 optional = false
 python-versions = "*"
 files = [
-    {file = "six-1.11.0-py2.py3-none-any.whl", hash = "sha256:112f5b46e6aa106db3e4e2494a03694c938f41c4c4535edbdfc816c2e0cb50f2"},
-    {file = "six-1.11.0.tar.gz", hash = "sha256:268a4ccb159c1a2d2c79336b02e75058387b0cdbb4cea2f07846a758f48a356d"},
+    {file = "six-1.11.0-py2.py3-none-any.whl", hash = "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"},
+    {file = "six-1.11.0.tar.gz", hash = "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9"},
 ]
 
 [metadata]

diff --git a/tests/installation/test_executor.py b/tests/installation/test_executor.py
@@ -126,7 +126,6 @@ def io_not_decorated() -> BufferedIO:
 def pool(pypi_repository: PyPiRepository) -> RepositoryPool:
     pool = RepositoryPool()
 
-    pypi_repository._fallback = True
     pool.add_repository(pypi_repository)
 
     return pool