Skip to content

Commit

Permalink
stop using the json API at pypi
Browse files Browse the repository at this point in the history
  • Loading branch information
dimbleby committed Mar 23, 2024
1 parent 8a8f21b commit da5ca99
Show file tree
Hide file tree
Showing 63 changed files with 70 additions and 5,604 deletions.
32 changes: 32 additions & 0 deletions src/poetry/repositories/http_repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@

if TYPE_CHECKING:
from packaging.utils import NormalizedName
from poetry.core.constraints.version import Version
from poetry.core.packages.package import Package
from poetry.core.packages.utils.link import Link

from poetry.repositories.link_sources.base import LinkSource
Expand Down Expand Up @@ -91,6 +93,36 @@ def certificates(self) -> RepositoryCertificateConfig:
def authenticated_url(self) -> str:
return self._authenticator.authenticated_url(url=self.url)

def find_links_for_package(self, package: Package) -> list[Link]:
try:
page = self.get_page(package.name)
except PackageNotFound:
return []

return list(page.links_for_version(package.name, package.version))

def _get_release_info(
self, name: NormalizedName, version: Version
) -> dict[str, Any]:
page = self.get_page(name)

links = list(page.links_for_version(name, version))
yanked = page.yanked(name, version)

return self._links_to_data(
links,
PackageInfo(
name=name,
version=version.text,
summary="",
requires_dist=[],
requires_python=None,
files=[],
yanked=yanked,
cache_version=str(self.CACHE_VERSION),
),
)

def _download(
self, url: str, dest: Path, *, raise_accepts_ranges: bool = False
) -> None:
Expand Down
33 changes: 0 additions & 33 deletions src/poetry/repositories/legacy_repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,11 @@
from contextlib import suppress
from functools import cached_property
from typing import TYPE_CHECKING
from typing import Any

import requests.adapters

from poetry.core.packages.package import Package

from poetry.inspection.info import PackageInfo
from poetry.repositories.exceptions import PackageNotFound
from poetry.repositories.http_repository import HTTPRepository
from poetry.repositories.link_sources.html import SimpleRepositoryPage
Expand All @@ -20,7 +18,6 @@
from packaging.utils import NormalizedName
from poetry.core.constraints.version import Version
from poetry.core.constraints.version import VersionConstraint
from poetry.core.packages.utils.link import Link

from poetry.config.config import Config

Expand Down Expand Up @@ -65,14 +62,6 @@ def package(

return package

def find_links_for_package(self, package: Package) -> list[Link]:
try:
page = self.get_page(package.name)
except PackageNotFound:
return []

return list(page.links_for_version(package.name, package.version))

def _find_packages(
self, name: NormalizedName, constraint: VersionConstraint
) -> list[Package]:
Expand Down Expand Up @@ -103,28 +92,6 @@ def _find_packages(
for version, yanked in versions
]

def _get_release_info(
self, name: NormalizedName, version: Version
) -> dict[str, Any]:
page = self.get_page(name)

links = list(page.links_for_version(name, version))
yanked = page.yanked(name, version)

return self._links_to_data(
links,
PackageInfo(
name=name,
version=version.text,
summary="",
requires_dist=[],
requires_python=None,
files=[],
yanked=yanked,
cache_version=str(self.CACHE_VERSION),
),
)

def _get_page(self, name: NormalizedName) -> SimpleRepositoryPage:
if not (response := self._get_response(f"/{name}/")):
raise PackageNotFound(f"Package [{name}] not found.")
Expand Down
7 changes: 6 additions & 1 deletion src/poetry/repositories/link_sources/json.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,13 @@ def _link_cache(self) -> LinkCache:
metadata = bool(metadata_value)
break

hashes = file.get("hashes")
link = Link(
url, requires_python=requires_python, yanked=yanked, metadata=metadata
url,
requires_python=requires_python,
hashes=hashes,
yanked=yanked,
metadata=metadata,
)

if link.ext not in self.SUPPORTED_FORMATS:
Expand Down
79 changes: 0 additions & 79 deletions src/poetry/repositories/pypi_repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@

from cachecontrol.controller import logger as cache_control_logger
from poetry.core.packages.package import Package
from poetry.core.packages.utils.link import Link
from poetry.core.version.exceptions import InvalidVersion

from poetry.repositories.exceptions import PackageNotFound
Expand All @@ -26,18 +25,14 @@

if TYPE_CHECKING:
from packaging.utils import NormalizedName
from poetry.core.constraints.version import Version
from poetry.core.constraints.version import VersionConstraint

SUPPORTED_PACKAGE_TYPES = {"sdist", "bdist_wheel"}


class PyPiRepository(HTTPRepository):
def __init__(
self,
url: str = "https://pypi.org/",
disable_cache: bool = False,
fallback: bool = True,
pool_size: int = requests.adapters.DEFAULT_POOLSIZE,
) -> None:
super().__init__(
Expand All @@ -48,7 +43,6 @@ def __init__(
)

self._base_url = url
self._fallback = fallback

def search(self, query: str) -> list[Package]:
results = []
Expand Down Expand Up @@ -110,79 +104,6 @@ def _get_package_info(self, name: NormalizedName) -> dict[str, Any]:

return info

def find_links_for_package(self, package: Package) -> list[Link]:
json_data = self._get(f"pypi/{package.name}/{package.version}/json")
if json_data is None:
return []

links = []
for url in json_data["urls"]:
if url["packagetype"] in SUPPORTED_PACKAGE_TYPES:
h = f"sha256={url['digests']['sha256']}"
links.append(Link(url["url"] + "#" + h, yanked=self._get_yanked(url)))

return links

def _get_release_info(
self, name: NormalizedName, version: Version
) -> dict[str, Any]:
from poetry.inspection.info import PackageInfo

self._log(f"Getting info for {name} ({version}) from PyPI", "debug")

json_data = self._get(f"pypi/{name}/{version}/json")
if json_data is None:
raise PackageNotFound(f"Package [{name}] not found.")

info = json_data["info"]

data = PackageInfo(
name=info["name"],
version=info["version"],
summary=info["summary"],
requires_dist=info["requires_dist"],
requires_python=info["requires_python"],
yanked=self._get_yanked(info),
cache_version=str(self.CACHE_VERSION),
)

try:
version_info = json_data["urls"]
except KeyError:
version_info = []

files = info.get("files", [])
for file_info in version_info:
if file_info["packagetype"] in SUPPORTED_PACKAGE_TYPES:
files.append(
{
"file": file_info["filename"],
"hash": "sha256:" + file_info["digests"]["sha256"],
}
)
data.files = files

if self._fallback and data.requires_dist is None:
self._log(
"No dependencies found, downloading metadata and/or archives",
level="debug",
)
# No dependencies set (along with other information)
# This might be due to actually no dependencies
# or badly set metadata when uploading.
# So, we need to make sure there is actually no
# dependencies by introspecting packages.
page = self.get_page(name)
links = list(page.links_for_version(name, version))
info = self._get_info_from_links(links)

data.requires_dist = info.requires_dist

if not data.requires_python:
data.requires_python = info.requires_python

return data.asdict()

def _get_page(self, name: NormalizedName) -> SimpleJsonPage:
source = self._base_url + f"simple/{name}/"
info = self.get_package_info(name)
Expand Down
2 changes: 1 addition & 1 deletion tests/installation/fixtures/with-pypi-repository.test
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ name = "pluggy"
version = "0.6.0"
description = "plugin and hook calling mechanisms for python"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
python-versions = ">=2.7,<3.0.dev0 || >=3.4.dev0"
files = [
{file = "pluggy-0.6.0-py2-none-any.whl", hash = "sha256:9b835f86bfe5498c87ace7f4899cb1b0c40e71c9277377f6851c74a307879285"},
{file = "pluggy-0.6.0-py3-none-any.whl", hash = "sha256:8c646771f5eab7557d1f3924077c55408e86bdfb700f7d86a6d83abeabff4c66"},
Expand Down
1 change: 0 additions & 1 deletion tests/installation/test_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,6 @@ def io_not_decorated() -> BufferedIO:
def pool(pypi_repository: PyPiRepository) -> RepositoryPool:
pool = RepositoryPool()

pypi_repository._fallback = True
pool.add_repository(pypi_repository)

return pool
Expand Down
6 changes: 3 additions & 3 deletions tests/installation/test_installer.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
from poetry.repositories import RepositoryPool
from poetry.repositories.installed_repository import InstalledRepository
from poetry.toml.file import TOMLFile
from poetry.utils._compat import WINDOWS
from poetry.utils.env import MockEnv
from poetry.utils.env import NullEnv
from tests.helpers import MOCK_DEFAULT_GIT_REVISION
Expand Down Expand Up @@ -1900,8 +1901,6 @@ def test_installer_required_extras_should_not_be_removed_when_updating_single_de
config: Config,
pypi_repository: PyPiRepository,
) -> None:
mocker.patch("sys.platform", "darwin")

pool = RepositoryPool()
pool.add_repository(pypi_repository)

Expand Down Expand Up @@ -1954,7 +1953,8 @@ def test_installer_required_extras_should_not_be_removed_when_updating_single_de
result = installer.run()
assert result == 0

assert installer.executor.installations_count == 7
installations = 8 if WINDOWS else 7
assert installer.executor.installations_count == installations
assert installer.executor.updates_count == 0
assert installer.executor.removals_count == 0

Expand Down
38 changes: 19 additions & 19 deletions tests/puzzle/test_solver.py
Original file line number Diff line number Diff line change
Expand Up @@ -3016,25 +3016,6 @@ def test_solver_can_solve_with_legacy_repository_using_proper_python_compatible_
)


def test_solver_skips_invalid_versions(
package: ProjectPackage, io: NullIO, pypi_repository: PyPiRepository
) -> None:
package.python_versions = "^3.9"

pool = RepositoryPool([pypi_repository])

solver = Solver(package, pool, [], [], io)

package.add_dependency(Factory.create_dependency("six-unknown-version", "^1.11"))

transaction = solver.solve()

check_solver_result(
transaction,
[{"job": "install", "package": get_package("six-unknown-version", "1.11.0")}],
)


def test_multiple_constraints_on_root(
package: ProjectPackage, solver: Solver, repo: Repository
) -> None:
Expand Down Expand Up @@ -4660,6 +4641,25 @@ def test_update_with_use_latest_vs_lock(
)


def test_solver_skips_invalid_versions(
package: ProjectPackage, io: NullIO, pypi_repository: PyPiRepository
) -> None:
package.python_versions = "^3.9"

pool = RepositoryPool([pypi_repository])

solver = Solver(package, pool, [], [], io)

package.add_dependency(Factory.create_dependency("six-unknown-version", "^1.11"))

transaction = solver.solve()

check_solver_result(
transaction,
[{"job": "install", "package": get_package("six-unknown-version", "1.11.0")}],
)


@pytest.mark.parametrize("with_extra", [False, True])
def test_solver_resolves_duplicate_dependency_in_extra(
package: ProjectPackage,
Expand Down

0 comments on commit da5ca99

Please sign in to comment.