From 991f08f131ac0cc3ce42d84a0bacfc28d9984f78 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 16 Jan 2022 18:38:40 +0000 Subject: [PATCH 01/26] Add implementation of core metadata as a dataclass --- .pre-commit-config.yaml | 2 +- packaging/metadata.py | 416 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 417 insertions(+), 1 deletion(-) create mode 100644 packaging/metadata.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 49ae0d4e..97e3959b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.812 + rev: v0.931 hooks: - id: mypy exclude: '^(docs|tasks|tests)|setup\.py' diff --git a/packaging/metadata.py b/packaging/metadata.py new file mode 100644 index 00000000..270e8463 --- /dev/null +++ b/packaging/metadata.py @@ -0,0 +1,416 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import dataclasses +import math +import sys +import textwrap +from email import message_from_bytes +from email.contentmanager import raw_data_manager +from email.headerregistry import Address, AddressHeader +from email.message import EmailMessage +from email.policy import EmailPolicy, Policy +from functools import lru_cache, reduce +from itertools import chain +from typing import ( + TYPE_CHECKING, + Any, + ClassVar, + Collection, + Dict, + Iterable, + Iterator, + Mapping, + NamedTuple, + Set, + Tuple, + Type, + TypeVar, + Union, + cast, +) + +from .requirements import InvalidRequirement, Requirement +from .specifiers import SpecifierSet +from .version import Version + +T = TypeVar("T", bound="CoreMetadata") +A = TypeVar("A") +B = TypeVar("B") + +if sys.version_info[:2] >= (3, 8) and TYPE_CHECKING: # pragma: no cover + from typing import Literal + + NormalizedDynamicFields = Literal[ + "platform", + "summary", + "description", + "keywords", + "home-page", + "author", + "author-email", + "license", + "supported-platform", + "download-url", + "classifier", + "maintainer", + "maintainer-email", + "requires-dist", + "requires-python", + "requires-external", + "project-url", + "provides-extra", + "provides-dist", + "obsoletes-dist", + "description-content-type", + ] +else: + NormalizedDynamicFields = str + + +def _normalize_field_name_for_dynamic(field: str) -> NormalizedDynamicFields: + """Normalize a metadata field name that is acceptable in `dynamic`. + + The field name will be normalized to lower-case. JSON field names are + also acceptable and will be translated accordingly. + + """ + return cast(NormalizedDynamicFields, field.lower().replace("_", "-")) + + +# Bypass frozen dataclass for __post_init__, this approach is documented in: +# https://docs.python.org/3/library/dataclasses.html#frozen-instances +_setattr = object.__setattr__ + + +class EmailAddress(NamedTuple): + """Named tuple representing an email address. + For values without a display name use ``EmailAddress(None, "your@email.com")`` + """ + + display_name: Union[str, None] + value: str + + def __str__(self) -> str: + return str(Address(self.display_name or "", addr_spec=self.value)) + + +@dataclasses.dataclass(frozen=True) +class CoreMetadata: + """ + Core metadata for Python packages, represented as an immutable + :obj:`dataclass `. + + Specification: https://packaging.python.org/en/latest/specifications/core-metadata/ + + Attribute names follow :pep:`PEP 566's JSON guidelines + <566#json-compatible-metadata>`. + """ + + # 1.0 + name: str + version: Union[Version, None] = None + platform: Collection[str] = () + summary: str = "" + description: str = "" + keywords: Collection[str] = () + home_page: str = "" + author: str = "" + author_email: Collection[EmailAddress] = () + license: str = "" + # license_file: Collection[str] = () # not standard yet + # 1.1 + supported_platform: Collection[str] = () + download_url: str = "" + classifier: Collection[str] = () + # 1.2 + maintainer: str = "" + maintainer_email: Collection[EmailAddress] = () + requires_dist: Collection[Requirement] = () + requires_python: SpecifierSet = dataclasses.field(default_factory=SpecifierSet) + requires_external: Collection[str] = () + project_url: Mapping[str, str] = dataclasses.field(default_factory=dict) + provides_extra: Collection[str] = () + provides_dist: Collection[Requirement] = () + obsoletes_dist: Collection[Requirement] = () + # 2.1 + description_content_type: str = "" + # 2.2 + dynamic: Collection[NormalizedDynamicFields] = () + + def __post_init__(self) -> None: + """Perform required data conversions, validations, and ensure immutability""" + + _should_be_set = (self._MULTIPLE_USE | {"keywords"}) - {"project_url"} + + for field in self._fields(): + value = getattr(self, field) + if field.endswith("dist"): + reqs = (self._convert_single_req(v) for v in value) + _setattr(self, field, frozenset(reqs)) + elif field.endswith("email"): + emails = self._convert_emails(value) + _setattr(self, field, frozenset(emails)) + elif field in _should_be_set: + _setattr(self, field, frozenset(value)) + elif field in {"description", "summary"}: + _setattr(self, field, value.strip()) + + urls = self.project_url + if not isinstance(urls, Mapping) and isinstance(urls, Iterable): + urls = {} + for url in cast(Iterable[str], self.project_url): + key, _, value = url.partition(",") + urls[key.strip()] = value.strip() + _setattr(self, "project_url", ImmutableMapping(urls)) + + # Dataclasses don't enforce data types at runtime + if not isinstance(self.requires_python, SpecifierSet): + requires_python = self._parse_requires_python(self.requires_python) + _setattr(self, "requires_python", requires_python) + if self.version and not isinstance(self.version, Version): + _setattr(self, "version", Version(self.version)) + + if self.dynamic: + values = (_normalize_field_name_for_dynamic(f) for f in self.dynamic) + dynamic = frozenset(v for v in values if self._validate_dynamic(v)) + _setattr(self, "dynamic", dynamic) + + @property + def metadata_version(self) -> str: + """ + The data structure is always compatible with the latest approved + version of the spec, even when parsing files that use previous versions. + """ + return "2.2" + + @classmethod + @lru_cache(maxsize=None) + def _fields(cls) -> Collection[str]: + return frozenset(f.name for f in dataclasses.fields(cls)) + + @classmethod + def _read_pkg_info(cls, pkg_info: bytes, /) -> Dict[str, Any]: + """Parse PKG-INFO data.""" + + msg = message_from_bytes(pkg_info, EmailMessage, policy=cls._PARSING_POLICY) + info = cast(EmailMessage, msg) + + attrs: Dict[str, Any] = {} + for key in info.keys(): + field = key.lower().replace("-", "_") + if field in cls._UPDATES: + field = cls._UPDATES[field] + + value = str(info.get(key)) # email.header.Header.__str__ handles encoding + + if field == "keywords": + attrs[field] = " ".join(value.splitlines()).split(",") + elif field == "description": + attrs[field] = cls._unescape_description(value) + elif field.endswith("email"): + attrs[field] = cls._parse_emails(value) + elif field in cls._MULTIPLE_USE: + attrs[field] = (str(v) for v in info.get_all(key)) + elif field in cls._fields(): + attrs[field] = value + + if "description" not in attrs: + attrs["description"] = info.get_content(content_manager=raw_data_manager) + + return attrs + + @classmethod + def from_pkg_info(cls: Type[T], pkg_info: bytes, /) -> T: + """Parse PKG-INFO data.""" + + return cls(**cls._read_pkg_info(pkg_info)) + + @classmethod + def from_dist_info_metadata(cls: Type[T], metadata_source: bytes, /) -> T: + """Parse METADATA data.""" + + attrs = cls._read_pkg_info(metadata_source) + _attrs = attrs.copy() + _attrs.pop("description", None) + + if attrs.get("dynamic"): + raise DynamicNotAllowed(attrs["dynamic"]) + + missing_fields = [k for k in cls._MANDATORY if not attrs.get(k)] + if missing_fields: + raise MissingRequiredFields(missing_fields) + + return cls(**attrs) + + def to_pkg_info(self) -> bytes: + """Generate PKG-INFO data.""" + + info = EmailMessage(self._PARSING_POLICY) + info.add_header("Metadata-Version", self.metadata_version) + # Use `sorted` in collections to improve reproducibility + for field in self._fields(): + value = getattr(self, field) + if not value: + continue + key = self._canonical_field(field) + if field == "keywords": + info.add_header(key, ",".join(sorted(value))) + elif field.endswith("email"): + _emails = (str(v) for v in value) + emails = ", ".join(sorted(v for v in _emails if v)) + info.add_header(key, emails) + elif field == "project_url": + for kind in sorted(value): + info.add_header(key, f"{kind}, {value[kind]}") + elif field == "description": + info.set_content(value, content_manager=raw_data_manager) + elif field in self._MULTIPLE_USE: + for single_value in sorted(str(v) for v in value): + info.add_header(key, single_value) + else: + info.add_header(key, str(value)) + + return info.as_bytes() + + def to_dist_info_metadata(self) -> bytes: + """Generate METADATA data.""" + if self.dynamic: + raise DynamicNotAllowed(self.dynamic) + return self.to_pkg_info() + + # --- Auxiliary Methods and Properties --- + # Not part of the API, but can be overwritten by subclasses + # (useful when providing a prof-of-concept for new PEPs) + + _MANDATORY: ClassVar[Set[str]] = {"name", "version"} + _NOT_DYNAMIC: ClassVar[Set[str]] = {"metadata_version", "name", "dynamic"} + _MULTIPLE_USE: ClassVar[Set[str]] = { + "dynamic", + "platform", + "supported_platform", + "classifier", + "requires_dist", + "requires_external", + "project_url", + "provides_extra", + "provides_dist", + "obsoletes_dist", + } + _UPDATES: ClassVar[Dict[str, str]] = { + "requires": "requires_dist", # PEP 314 => PEP 345 + "provides": "provides_dist", # PEP 314 => PEP 345 + "obsoletes": "obsoletes_dist", # PEP 314 => PEP 345 + } + _PARSING_POLICY: ClassVar[Policy] = EmailPolicy(max_line_length=math.inf, utf8=True) + + @classmethod + def _canonical_field(cls, field: str) -> str: + words = _normalize_field_name_for_dynamic(field).split("-") + ucfirst = "-".join(w[0].upper() + w[1:] for w in words) + replacements = {"Url": "URL", "Email": "email", "Page": "page"}.items() + return reduce(lambda acc, x: acc.replace(x[0], x[1]), replacements, ucfirst) + + @classmethod + def _parse_requires_python(cls, value: str) -> SpecifierSet: + if value and value[0].isnumeric(): + value = f"=={value}" + return SpecifierSet(value) + + @classmethod + def _parse_req(cls, value: str) -> Requirement: + try: + return Requirement(value) + except InvalidRequirement: + # Some old examples in PEPs use "()" around versions without an operator + # e.g.: `Provides: xmltools (1.3)` + name, _, rest = value.strip().partition("(") + value = f"{name}(=={rest}" + return Requirement(value) + + @classmethod + def _convert_single_req(cls, value: Union[Requirement, str]) -> Requirement: + return value if isinstance(value, Requirement) else cls._parse_req(value) + + @classmethod + def _convert_emails( + cls, value: Collection[Union[str, Tuple[str, str]]] + ) -> Iterator[EmailAddress]: + for email in value: + if isinstance(email, str): + yield from cls._parse_emails(email) + elif isinstance(email, tuple) and email[1]: + yield EmailAddress(email[0], email[1]) + + @classmethod + def _parse_emails(cls, value: str) -> Iterator[EmailAddress]: + singleline = " ".join(value.splitlines()) + if singleline.strip() == "UNKNOWN": + return + address_list = AddressHeader.value_parser(singleline) + for mailbox in address_list.all_mailboxes: + yield EmailAddress(mailbox.display_name, mailbox.addr_spec) + + @classmethod + def _unescape_description(cls, content: str) -> str: + """Reverse RFC-822 escaping by removing leading whitespaces from content.""" + lines = content.splitlines() + if not lines: + return "" + + first_line = lines[0].lstrip() + text = textwrap.dedent("\n".join(lines[1:])) + other_lines = (line.lstrip("|") for line in text.splitlines()) + return "\n".join(chain([first_line], other_lines)) + + def _validate_dynamic(self, normalized: str) -> bool: + field = normalized.lower().replace("-", "_") + if not hasattr(self, field): + raise InvalidCoreMetadataField(normalized) + if field in self._NOT_DYNAMIC: + raise InvalidDynamicField(normalized) + if getattr(self, field): + raise StaticFieldCannotBeDynamic(normalized) + return True + + +class ImmutableMapping(Mapping[A, B]): + def __init__(self, value: Mapping[A, B]): + self._value = value + + def __getitem__(self, key: A) -> B: + return self._value[key] + + def __iter__(self) -> Iterator[A]: + return iter(self._value) + + def __len__(self) -> int: + return len(self._value) + + +class InvalidCoreMetadataField(ValueError): + def __init__(self, field: str): + super().__init__(f"{field!r} is not a valid core metadata field") + + +class InvalidDynamicField(ValueError): + def __init__(self, field: str): + super().__init__(f"{field!r} cannot be dynamic") + + +class StaticFieldCannotBeDynamic(ValueError): + def __init__(self, field: str): + super().__init__(f"{field!r} specified both dynamically and statically") + + +class DynamicNotAllowed(ValueError): + def __init__(self, fields: Collection[str]): + given = ", ".join(fields) + super().__init__(f"Dynamic fields not allowed in this context (given: {given})") + + +class MissingRequiredFields(ValueError): + def __init__(self, fields: Collection[str]): + missing = ", ".join(fields) + super().__init__(f"Required fields are missing: {missing}") From db51418d4fd90986945bb1919cfaef282dba943b Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 16 Jan 2022 18:40:01 +0000 Subject: [PATCH 02/26] Make Requirement hashable --- packaging/requirements.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packaging/requirements.py b/packaging/requirements.py index 53f9a3aa..063a0694 100644 --- a/packaging/requirements.py +++ b/packaging/requirements.py @@ -5,7 +5,7 @@ import re import string import urllib.parse -from typing import List, Optional as TOptional, Set +from typing import Any, List, Optional as TOptional, Set from pyparsing import ( # noqa Combine, @@ -144,3 +144,9 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"" + + def __hash__(self) -> int: + return hash((self.__class__.__name__, str(self))) + + def __eq__(self, other: Any) -> bool: + return bool(self.__class__ == other.__class__ and str(self) == str(other)) From b421d1d1ac8e5b65529a6fc58a724e8f054246a5 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 16 Jan 2022 18:44:25 +0000 Subject: [PATCH 03/26] Add tests for core metadata --- .gitignore | 1 + tests/metadata_examples.csv | 18 ++ tests/test_metadata.py | 390 ++++++++++++++++++++++++++++++++++++ 3 files changed, 409 insertions(+) create mode 100644 tests/metadata_examples.csv create mode 100644 tests/test_metadata.py diff --git a/.gitignore b/.gitignore index 05e554a6..3f602651 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ _build/ build/ dist/ htmlcov/ +tests/downloads diff --git a/tests/metadata_examples.csv b/tests/metadata_examples.csv new file mode 100644 index 00000000..b01eea9d --- /dev/null +++ b/tests/metadata_examples.csv @@ -0,0 +1,18 @@ +appdirs,1.3.0 +boto3,1.20.37 +click,0.3 +enscons,0.28.0 +Flask,2.0.2 +flit,3.6.0 +hatch,0.23.1 +pdm,1.12.6 +poetry,1.1.12 +ptpython,0.5 +PyScaffold,4.1.4 +requests,2.27.1 +scikit-build,0.12.0 +setuptools,60.5.0 +six,1.16.0 +tomli,2.0.0 +urllib3,1.26.8 +wheelfile,0.0.8 diff --git a/tests/test_metadata.py b/tests/test_metadata.py new file mode 100644 index 00000000..65610611 --- /dev/null +++ b/tests/test_metadata.py @@ -0,0 +1,390 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import dataclasses +import json +import tarfile +from email.policy import compat32 +from hashlib import md5 +from itertools import chain +from pathlib import Path +from textwrap import dedent +from typing import Iterator, List +from urllib.request import urlopen +from zipfile import ZipFile + +import pytest + +from packaging.metadata import ( + CoreMetadata, + DynamicNotAllowed, + InvalidCoreMetadataField, + InvalidDynamicField, + MissingRequiredFields, + StaticFieldCannotBeDynamic, +) +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + +HERE = Path(__file__).parent +EXAMPLES = HERE / "metadata_examples.csv" +DOWNLOADS = HERE / "downloads" + + +class TestCoreMetadata: + def test_simple(self): + example = {"name": "simple", "version": "0.1", "requires_dist": ["appdirs>1.2"]} + metadata = CoreMetadata(**example) + req = next(iter(metadata.requires_dist)) + assert isinstance(req, Requirement) + + def test_replace(self): + example = { + "name": "simple", + "dynamic": ["version"], + "author_email": ["me@example.com"], + "requires_dist": ["appdirs>1.2"], + } + metadata = CoreMetadata(**example) + + # Make sure replace goes through validations and transformations + attrs = { + "version": "0.2", + "dynamic": [], + "author_email": [("name", "me@example.com")], + "requires_dist": ["appdirs>1.4"], + } + metadata1 = dataclasses.replace(metadata, **attrs) + req = next(iter(metadata1.requires_dist)) + assert req == Requirement("appdirs>1.4") + + with pytest.raises(InvalidCoreMetadataField): + dataclasses.replace(metadata, dynamic=["myfield"]) + with pytest.raises(InvalidDynamicField): + dataclasses.replace(metadata, dynamic=["name"]) + with pytest.raises(StaticFieldCannotBeDynamic): + dataclasses.replace(metadata, version="0.1") + + PER_VERSION_EXAMPLES = { + "1.1": """\ + Metadata-Version: 1.1 + Name: BeagleVote + Version: 1.0a2 + Platform: ObscureUnix, RareDOS + Supported-Platform: RedHat 7.2 + Supported-Platform: i386-win32-2791 + Summary: A module for collecting votes from beagles. + Description: This module collects votes from beagles + in order to determine their electoral wishes. + Do *not* try to use this module with basset hounds; + it makes them grumpy. + Keywords: dog puppy voting election + Home-page: http://www.example.com/~cschultz/bvote/ + Author: C. Schultz, Universal Features Syndicate, + Los Angeles, CA + Author-email: "C. Schultz" + License: This software may only be obtained by sending the + author a postcard, and then the user promises not + to redistribute it. + Classifier: Development Status :: 4 - Beta + Classifier: Environment :: Console (Text Based) + Requires: re + Requires: sys + Requires: zlib + Requires: xml.parsers.expat (>1.0) + Requires: psycopg + Provides: xml + Provides: xml.utils + Provides: xml.utils.iso8601 + Provides: xml.dom + Provides: xmltools (1.3) + Obsoletes: Gorgon + """, # based on PEP 314 + "2.1": """\ + Metadata-Version: 2.1 + Name: BeagleVote + Version: 1.0a2 + Platform: ObscureUnix, RareDOS + Supported-Platform: RedHat 7.2 + Supported-Platform: i386-win32-2791 + Summary: A module for collecting votes from beagles. + Description: This project provides powerful math functions + |For example, you can use `sum()` to sum numbers: + | + |Example:: + | + | >>> sum(1, 2) + | 3 + | + Keywords: dog puppy voting election + Home-page: http://www.example.com/~cschultz/bvote/ + Author: C. Schultz, Universal Features Syndicate, + Los Angeles, CA + Author-email: "C. Schultz" + Maintainer: C. Schultz, Universal Features Syndicate, + Los Angeles, CA + Maintainer-email: "C. Schultz" + License: This software may only be obtained by sending the + author a postcard, and then the user promises not + to redistribute it. + Classifier: Development Status :: 4 - Beta + Classifier: Environment :: Console (Text Based) + Requires-Dist: pkginfo + Requires-Dist: PasteDeploy + Requires-Dist: zope.interface (>3.5.0) + Provides-Dist: OtherProject + Provides-Dist: AnotherProject (3.4) + Provides-Dist: virtual_package + Obsoletes-Dist: Gorgon + Obsoletes-Dist: OtherProject (<3.0) + Requires-Python: 2.5 + Requires-Python: >2.1 + Requires-Python: >=2.3.4 + Requires-Python: >=2.5,<2.7 + Requires-External: C + Requires-External: libpng (>=1.5) + Project-URL: Bug Tracker, https://github.com/pypa/setuptools/issues + Project-URL: Documentation, https://setuptools.readthedocs.io/ + Project-URL: Funding, https://donate.pypi.org + Requires-Dist: pywin32 (>1.0); sys.platform == 'win32' + Obsoletes-Dist: pywin31; sys.platform == 'win32' + Requires-Dist: foo (1,!=1.3); platform.machine == 'i386' + Requires-Dist: bar; python_version == '2.4' or python_version == '2.5' + Requires-Dist: baz (>=1,!=1.3); platform.machine == 'i386' + Requires-External: libxslt; 'linux' in sys.platform + Provides-Extra: docs + Description-Content-Type: text/x-rst; charset=UTF-8 + """, # based on PEP 345 / PEP 566 + "2022-01-16": """\ + Metadata-Version: 2.2 + Name: BeagleVote + Version: 1.0a2 + Platform: ObscureUnix + Platform: RareDOS + Supported-Platform: RedHat 7.2 + Supported-Platform: i386-win32-2791 + Keywords: dog,puppy,voting,election + Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM + Author-email: cschuoltz@example.com, snoopy@peanuts.com + License: GPL version 3, excluding DRM provisions + Requires-Dist: pkginfo + Requires-Dist: PasteDeploy + Requires-Dist: zope.interface (>3.5.0) + Requires-Dist: pywin32 >1.0; sys_platform == 'win32' + Requires-Python: >2.6,!=3.0.*,!=3.1.* + Requires-External: C + Requires-External: libpng (>=1.5) + Requires-External: make; sys_platform != "win32" + Project-URL: Bug Tracker, http://bitbucket.org/tarek/distribute/issues/ + Provides-Extra: pdf + Requires-Dist: reportlab; extra == 'pdf' + Provides-Dist: OtherProject + Provides-Dist: AnotherProject (3.4) + Provides-Dist: virtual_package; python_version >= "3.4" + Obsoletes-Dist: Foo; os_name == "posix" + Dynamic: Maintainer + Dynamic: Maintainer-email + + This project provides powerful math functions + For example, you can use `sum()` to sum numbers: + + Example:: + + >>> sum(1, 2) + 3 + + """, # https://packaging.python.org/en/latest/specifications/core-metadata + } + + @pytest.mark.parametrize("spec", PER_VERSION_EXAMPLES.keys()) + def test_parsing(self, spec: str) -> None: + text = bytes(dedent(self.PER_VERSION_EXAMPLES[spec]), "UTF-8") + pkg_info = CoreMetadata.from_pkg_info(text) + if not pkg_info.dynamic: + metadata = CoreMetadata.from_dist_info_metadata(text) + assert metadata == pkg_info + else: + with pytest.raises(DynamicNotAllowed): + CoreMetadata.from_dist_info_metadata(text) + for field in ("requires_dist", "provides_dist", "obsoletes_dist"): + for value in getattr(pkg_info, field): + assert isinstance(value, Requirement) + if pkg_info.description: + desc = pkg_info.description.splitlines() + for line in desc: + assert not line.strip().startswith("|") + + @pytest.mark.parametrize("spec", PER_VERSION_EXAMPLES.keys()) + def test_serliazing(self, spec: str) -> None: + text = bytes(dedent(self.PER_VERSION_EXAMPLES[spec]), "UTF-8") + pkg_info = CoreMetadata.from_pkg_info(text) + if not pkg_info.dynamic: + assert isinstance(pkg_info.to_dist_info_metadata(), bytes) + else: + with pytest.raises(DynamicNotAllowed): + pkg_info.to_dist_info_metadata() + pkg_info_text = pkg_info.to_pkg_info() + assert len(pkg_info_text) > 0 + assert isinstance(pkg_info_text, bytes) + + def test_missing_required_fields(self): + with pytest.raises(MissingRequiredFields): + CoreMetadata.from_dist_info_metadata(b"") + + example = {"name": "pkg", "requires_dist": ["appdirs>1.2"]} + metadata = CoreMetadata(**example) + serialized = metadata.to_pkg_info() + with pytest.raises(MissingRequiredFields): + CoreMetadata.from_dist_info_metadata(serialized) + + def test_empty_fields(self): + metadata = CoreMetadata.from_pkg_info(b"Name: pkg\nDescription:\n") + assert metadata.description == "" + metadata = CoreMetadata.from_pkg_info(b"Name: pkg\nAuthor-email:\n") + assert metadata.description == "" + assert len(metadata.author_email) == 0 + + def test_single_line_description(self): + metadata = CoreMetadata.from_pkg_info(b"Name: pkg\nDescription: Hello World") + assert metadata.description == "Hello World" + + def test_empty_email(self): + example = {"name": "pkg", "maintainer_email": ["", "", ("", "")]} + metadata = CoreMetadata(**example) + serialized = metadata.to_pkg_info() + assert b"Maintainer-email:" not in serialized + + +# --- Integration Tests --- + + +def examples() -> List[List[str]]: + lines = EXAMPLES.read_text().splitlines() + return [[v.strip() for v in line.split(",")] for line in lines] + + +class TestIntegration: + @pytest.mark.parametrize("pkg, version", examples()) + def test_parse(self, pkg: str, version: str) -> None: + for dist in download_dists(pkg, version): + if dist.suffix == ".whl": + orig = read_metadata(dist) + from_ = CoreMetadata.from_dist_info_metadata + to_ = CoreMetadata.to_dist_info_metadata + else: + orig = read_pkg_info(dist) + from_ = CoreMetadata.from_pkg_info + to_ = CoreMetadata.to_pkg_info + + # Given PKG-INFO or METADATA from existing packages on PyPI + # - Make sure they can be parsed + metadata = from_(orig) + assert metadata.name.lower() == pkg.lower() + assert str(metadata.version) == version + # - Make sure they can be converted back into PKG-INFO or METADATA + recons_file = to_(metadata) + assert len(recons_file) >= 0 + # - Make sure that the reconstructed file can be parsed and the data + # remains unchanged + recons_data = from_(recons_file) + description = metadata.description.replace("\r\n", "\n") + metadata = dataclasses.replace(metadata, description=description) + assert metadata == recons_data + # - Make sure the reconstructed file can be parsed with compat32 + attrs = dataclasses.asdict(_Compat32Metadata.from_pkg_info(recons_file)) + assert CoreMetadata(**attrs) + # - Make sure that successive calls to `to_...` and `from_...` + # always return the same result + file_contents = recons_file + data = recons_data + for _ in range(3): + result_contents = to_(data) + assert file_contents == result_contents + result_data = from_(result_contents) + assert data == result_data + file_contents, data = result_contents, result_data + + +# --- Helper Functions/Classes --- + + +class _Compat32Metadata(CoreMetadata): + """The Core Metadata spec requires the file to be parse-able with compat32. + The implementation uses a different approach to ensure UTF-8 can be used. + Therefore it is important to test against compat32 to make sure nothing + goes wrong. + """ + + _PARSING_POLICY = compat32 + + +def download(url: str, dest: Path, md5_digest: str) -> Path: + with urlopen(url) as f: + data = f.read() + + assert md5(data).hexdigest() == md5_digest + + with open(dest, "wb") as f: + f.write(data) + + assert dest.exists() + + return dest + + +def download_dists(pkg: str, version: str) -> List[Path]: + """Either use cached dist file or download it from PyPI""" + DOWNLOADS.mkdir(exist_ok=True) + + distributions = retrieve_pypi_dist_metadata(pkg, version) + filenames = {dist["filename"] for dist in distributions} + + # Remove old files to prevent cache to grow indefinitely + canonical = canonicalize_name(pkg) + names = [pkg, canonical, canonical.replace("-", "_")] + for file in chain.from_iterable(DOWNLOADS.glob(f"{n}*") for n in names): + if file.name not in filenames: + file.unlink() + + dist_files = [] + for dist in retrieve_pypi_dist_metadata(pkg, version): + dest = DOWNLOADS / dist["filename"] + if not dest.exists(): + download(dist["url"], dest, dist["md5_digest"]) + dist_files.append(dest) + + return dist_files + + +def retrieve_pypi_dist_metadata(package: str, version: str) -> Iterator[dict]: + # https://warehouse.pypa.io/api-reference/json.html + id_ = f"{package}/{version}" + with urlopen(f"https://pypi.org/pypi/{id_}/json") as f: + metadata = json.load(f) + + if metadata["info"]["yanked"]: + raise ValueError(f"Release for {package} {version} was yanked") + + version = metadata["info"]["version"] + for dist in metadata["releases"][version]: + if any(dist["filename"].endswith(ext) for ext in (".tar.gz", ".whl")): + yield dist + + +def read_metadata(wheel: Path) -> bytes: + with ZipFile(wheel, "r") as zipfile: + for member in zipfile.namelist(): + if member.endswith(".dist-info/METADATA"): + return zipfile.read(member) + raise FileNotFoundError(f"METADATA not found in {wheel}") + + +def read_pkg_info(sdist: Path) -> bytes: + with tarfile.open(sdist, mode="r:gz") as tar: + for member in tar.getmembers(): + if member.name.endswith("PKG-INFO"): + file = tar.extractfile(member) + if file is not None: + return file.read() + raise FileNotFoundError(f"PKG-INFO not found in {sdist}") From 598288a5a08fd1fdfa3b1365c4d151f9aa7bc346 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Sun, 16 Jan 2022 19:01:04 +0000 Subject: [PATCH 04/26] Remove positonal only arguments (Not supported in old versions of Python) --- packaging/metadata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packaging/metadata.py b/packaging/metadata.py index 270e8463..9406b5b1 100644 --- a/packaging/metadata.py +++ b/packaging/metadata.py @@ -191,7 +191,7 @@ def _fields(cls) -> Collection[str]: return frozenset(f.name for f in dataclasses.fields(cls)) @classmethod - def _read_pkg_info(cls, pkg_info: bytes, /) -> Dict[str, Any]: + def _read_pkg_info(cls, pkg_info: bytes) -> Dict[str, Any]: """Parse PKG-INFO data.""" msg = message_from_bytes(pkg_info, EmailMessage, policy=cls._PARSING_POLICY) @@ -222,13 +222,13 @@ def _read_pkg_info(cls, pkg_info: bytes, /) -> Dict[str, Any]: return attrs @classmethod - def from_pkg_info(cls: Type[T], pkg_info: bytes, /) -> T: + def from_pkg_info(cls: Type[T], pkg_info: bytes) -> T: """Parse PKG-INFO data.""" return cls(**cls._read_pkg_info(pkg_info)) @classmethod - def from_dist_info_metadata(cls: Type[T], metadata_source: bytes, /) -> T: + def from_dist_info_metadata(cls: Type[T], metadata_source: bytes) -> T: """Parse METADATA data.""" attrs = cls._read_pkg_info(metadata_source) From e8b51d2e1b98fff123bc5bf72ccca004aa6fa72d Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 18 Jan 2022 10:38:50 +0000 Subject: [PATCH 05/26] Remove custom mapping class in metadata --- packaging/metadata.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/packaging/metadata.py b/packaging/metadata.py index 9406b5b1..4d0560a6 100644 --- a/packaging/metadata.py +++ b/packaging/metadata.py @@ -163,7 +163,7 @@ def __post_init__(self) -> None: for url in cast(Iterable[str], self.project_url): key, _, value = url.partition(",") urls[key.strip()] = value.strip() - _setattr(self, "project_url", ImmutableMapping(urls)) + _setattr(self, "project_url", urls) # Dataclasses don't enforce data types at runtime if not isinstance(self.requires_python, SpecifierSet): @@ -375,20 +375,6 @@ def _validate_dynamic(self, normalized: str) -> bool: return True -class ImmutableMapping(Mapping[A, B]): - def __init__(self, value: Mapping[A, B]): - self._value = value - - def __getitem__(self, key: A) -> B: - return self._value[key] - - def __iter__(self) -> Iterator[A]: - return iter(self._value) - - def __len__(self) -> int: - return len(self._value) - - class InvalidCoreMetadataField(ValueError): def __init__(self, field: str): super().__init__(f"{field!r} is not a valid core metadata field") From 800844c0ebac8a861be7de0320b09253b97fd366 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 18 Jan 2022 10:54:53 +0000 Subject: [PATCH 06/26] Remove micro-optmisation (cache) from metadata --- packaging/metadata.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packaging/metadata.py b/packaging/metadata.py index 4d0560a6..af5311d6 100644 --- a/packaging/metadata.py +++ b/packaging/metadata.py @@ -11,7 +11,7 @@ from email.headerregistry import Address, AddressHeader from email.message import EmailMessage from email.policy import EmailPolicy, Policy -from functools import lru_cache, reduce +from functools import reduce from itertools import chain from typing import ( TYPE_CHECKING, @@ -186,7 +186,6 @@ def metadata_version(self) -> str: return "2.2" @classmethod - @lru_cache(maxsize=None) def _fields(cls) -> Collection[str]: return frozenset(f.name for f in dataclasses.fields(cls)) From 3aaa73ec328f70019db472df7da7c0012f02e642 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 18 Jan 2022 11:42:31 +0000 Subject: [PATCH 07/26] Use flags in examples for metadata tests As mentioned in the review for #498, conditions in tests should not be based in what the code being tested, otherwise they might end up hiding other problems. The solution is to pass flags in the test parameters themselves. --- tests/test_metadata.py | 289 +++++++++++++++++++++-------------------- 1 file changed, 151 insertions(+), 138 deletions(-) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 65610611..8c9bc52c 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -67,161 +67,174 @@ def test_replace(self): dataclasses.replace(metadata, version="0.1") PER_VERSION_EXAMPLES = { - "1.1": """\ - Metadata-Version: 1.1 - Name: BeagleVote - Version: 1.0a2 - Platform: ObscureUnix, RareDOS - Supported-Platform: RedHat 7.2 - Supported-Platform: i386-win32-2791 - Summary: A module for collecting votes from beagles. - Description: This module collects votes from beagles - in order to determine their electoral wishes. - Do *not* try to use this module with basset hounds; - it makes them grumpy. - Keywords: dog puppy voting election - Home-page: http://www.example.com/~cschultz/bvote/ - Author: C. Schultz, Universal Features Syndicate, - Los Angeles, CA - Author-email: "C. Schultz" - License: This software may only be obtained by sending the - author a postcard, and then the user promises not - to redistribute it. - Classifier: Development Status :: 4 - Beta - Classifier: Environment :: Console (Text Based) - Requires: re - Requires: sys - Requires: zlib - Requires: xml.parsers.expat (>1.0) - Requires: psycopg - Provides: xml - Provides: xml.utils - Provides: xml.utils.iso8601 - Provides: xml.dom - Provides: xmltools (1.3) - Obsoletes: Gorgon - """, # based on PEP 314 - "2.1": """\ - Metadata-Version: 2.1 - Name: BeagleVote - Version: 1.0a2 - Platform: ObscureUnix, RareDOS - Supported-Platform: RedHat 7.2 - Supported-Platform: i386-win32-2791 - Summary: A module for collecting votes from beagles. - Description: This project provides powerful math functions - |For example, you can use `sum()` to sum numbers: - | - |Example:: - | - | >>> sum(1, 2) - | 3 - | - Keywords: dog puppy voting election - Home-page: http://www.example.com/~cschultz/bvote/ - Author: C. Schultz, Universal Features Syndicate, - Los Angeles, CA - Author-email: "C. Schultz" - Maintainer: C. Schultz, Universal Features Syndicate, - Los Angeles, CA - Maintainer-email: "C. Schultz" - License: This software may only be obtained by sending the - author a postcard, and then the user promises not - to redistribute it. - Classifier: Development Status :: 4 - Beta - Classifier: Environment :: Console (Text Based) - Requires-Dist: pkginfo - Requires-Dist: PasteDeploy - Requires-Dist: zope.interface (>3.5.0) - Provides-Dist: OtherProject - Provides-Dist: AnotherProject (3.4) - Provides-Dist: virtual_package - Obsoletes-Dist: Gorgon - Obsoletes-Dist: OtherProject (<3.0) - Requires-Python: 2.5 - Requires-Python: >2.1 - Requires-Python: >=2.3.4 - Requires-Python: >=2.5,<2.7 - Requires-External: C - Requires-External: libpng (>=1.5) - Project-URL: Bug Tracker, https://github.com/pypa/setuptools/issues - Project-URL: Documentation, https://setuptools.readthedocs.io/ - Project-URL: Funding, https://donate.pypi.org - Requires-Dist: pywin32 (>1.0); sys.platform == 'win32' - Obsoletes-Dist: pywin31; sys.platform == 'win32' - Requires-Dist: foo (1,!=1.3); platform.machine == 'i386' - Requires-Dist: bar; python_version == '2.4' or python_version == '2.5' - Requires-Dist: baz (>=1,!=1.3); platform.machine == 'i386' - Requires-External: libxslt; 'linux' in sys.platform - Provides-Extra: docs - Description-Content-Type: text/x-rst; charset=UTF-8 - """, # based on PEP 345 / PEP 566 - "2022-01-16": """\ - Metadata-Version: 2.2 - Name: BeagleVote - Version: 1.0a2 - Platform: ObscureUnix - Platform: RareDOS - Supported-Platform: RedHat 7.2 - Supported-Platform: i386-win32-2791 - Keywords: dog,puppy,voting,election - Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM - Author-email: cschuoltz@example.com, snoopy@peanuts.com - License: GPL version 3, excluding DRM provisions - Requires-Dist: pkginfo - Requires-Dist: PasteDeploy - Requires-Dist: zope.interface (>3.5.0) - Requires-Dist: pywin32 >1.0; sys_platform == 'win32' - Requires-Python: >2.6,!=3.0.*,!=3.1.* - Requires-External: C - Requires-External: libpng (>=1.5) - Requires-External: make; sys_platform != "win32" - Project-URL: Bug Tracker, http://bitbucket.org/tarek/distribute/issues/ - Provides-Extra: pdf - Requires-Dist: reportlab; extra == 'pdf' - Provides-Dist: OtherProject - Provides-Dist: AnotherProject (3.4) - Provides-Dist: virtual_package; python_version >= "3.4" - Obsoletes-Dist: Foo; os_name == "posix" - Dynamic: Maintainer - Dynamic: Maintainer-email - - This project provides powerful math functions - For example, you can use `sum()` to sum numbers: - - Example:: - - >>> sum(1, 2) - 3 - - """, # https://packaging.python.org/en/latest/specifications/core-metadata + "1.1": { + "has_dynamic_fields": False, + "is_final_metadata": True, + "file_contents": """\ + Metadata-Version: 1.1 + Name: BeagleVote + Version: 1.0a2 + Platform: ObscureUnix, RareDOS + Supported-Platform: RedHat 7.2 + Supported-Platform: i386-win32-2791 + Summary: A module for collecting votes from beagles. + Description: This module collects votes from beagles + in order to determine their electoral wishes. + Do *not* try to use this module with basset hounds; + it makes them grumpy. + Keywords: dog puppy voting election + Home-page: http://www.example.com/~cschultz/bvote/ + Author: C. Schultz, Universal Features Syndicate, + Los Angeles, CA + Author-email: "C. Schultz" + License: This software may only be obtained by sending the + author a postcard, and then the user promises not + to redistribute it. + Classifier: Development Status :: 4 - Beta + Classifier: Environment :: Console (Text Based) + Requires: re + Requires: sys + Requires: zlib + Requires: xml.parsers.expat (>1.0) + Requires: psycopg + Provides: xml + Provides: xml.utils + Provides: xml.utils.iso8601 + Provides: xml.dom + Provides: xmltools (1.3) + Obsoletes: Gorgon + """, # based on PEP 314 + }, + "2.1": { + "has_dynamic_fields": False, + "is_final_metadata": True, + "file_contents": """\ + Metadata-Version: 2.1 + Name: BeagleVote + Version: 1.0a2 + Platform: ObscureUnix, RareDOS + Supported-Platform: RedHat 7.2 + Supported-Platform: i386-win32-2791 + Summary: A module for collecting votes from beagles. + Description: This project provides powerful math functions + |For example, you can use `sum()` to sum numbers: + | + |Example:: + | + | >>> sum(1, 2) + | 3 + | + Keywords: dog puppy voting election + Home-page: http://www.example.com/~cschultz/bvote/ + Author: C. Schultz, Universal Features Syndicate, + Los Angeles, CA + Author-email: "C. Schultz" + Maintainer: C. Schultz, Universal Features Syndicate, + Los Angeles, CA + Maintainer-email: "C. Schultz" + License: This software may only be obtained by sending the + author a postcard, and then the user promises not + to redistribute it. + Classifier: Development Status :: 4 - Beta + Classifier: Environment :: Console (Text Based) + Requires-Dist: pkginfo + Requires-Dist: PasteDeploy + Requires-Dist: zope.interface (>3.5.0) + Provides-Dist: OtherProject + Provides-Dist: AnotherProject (3.4) + Provides-Dist: virtual_package + Obsoletes-Dist: Gorgon + Obsoletes-Dist: OtherProject (<3.0) + Requires-Python: 2.5 + Requires-Python: >2.1 + Requires-Python: >=2.3.4 + Requires-Python: >=2.5,<2.7 + Requires-External: C + Requires-External: libpng (>=1.5) + Project-URL: Bug Tracker, https://github.com/pypa/setuptools/issues + Project-URL: Documentation, https://setuptools.readthedocs.io/ + Project-URL: Funding, https://donate.pypi.org + Requires-Dist: pywin32 (>1.0); sys.platform == 'win32' + Obsoletes-Dist: pywin31; sys.platform == 'win32' + Requires-Dist: foo (1,!=1.3); platform.machine == 'i386' + Requires-Dist: bar; python_version == '2.4' or python_version == '2.5' + Requires-Dist: baz (>=1,!=1.3); platform.machine == 'i386' + Requires-External: libxslt; 'linux' in sys.platform + Provides-Extra: docs + Description-Content-Type: text/x-rst; charset=UTF-8 + """, # based on PEP 345 / PEP 566 + }, + "2022-01-16": { + "has_dynamic_fields": True, + "is_final_metadata": False, + "file_contents": """\ + Metadata-Version: 2.2 + Name: BeagleVote + Version: 1.0a2 + Platform: ObscureUnix + Platform: RareDOS + Supported-Platform: RedHat 7.2 + Supported-Platform: i386-win32-2791 + Keywords: dog,puppy,voting,election + Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM + Author-email: cschuoltz@example.com, snoopy@peanuts.com + License: GPL version 3, excluding DRM provisions + Requires-Dist: pkginfo + Requires-Dist: PasteDeploy + Requires-Dist: zope.interface (>3.5.0) + Requires-Dist: pywin32 >1.0; sys_platform == 'win32' + Requires-Python: >2.6,!=3.0.*,!=3.1.* + Requires-External: C + Requires-External: libpng (>=1.5) + Requires-External: make; sys_platform != "win32" + Project-URL: Bug Tracker, http://bitbucket.org/tarek/distribute/issues/ + Provides-Extra: pdf + Requires-Dist: reportlab; extra == 'pdf' + Provides-Dist: OtherProject + Provides-Dist: AnotherProject (3.4) + Provides-Dist: virtual_package; python_version >= "3.4" + Obsoletes-Dist: Foo; os_name == "posix" + Dynamic: Maintainer + Dynamic: Maintainer-email + + This project provides powerful math functions + For example, you can use `sum()` to sum numbers: + + Example:: + + >>> sum(1, 2) + 3 + + """, # https://packaging.python.org/en/latest/specifications/core-metadata + }, } @pytest.mark.parametrize("spec", PER_VERSION_EXAMPLES.keys()) def test_parsing(self, spec: str) -> None: - text = bytes(dedent(self.PER_VERSION_EXAMPLES[spec]), "UTF-8") + example = self.PER_VERSION_EXAMPLES[spec] + text = bytes(dedent(example["file_contents"]), "UTF-8") pkg_info = CoreMetadata.from_pkg_info(text) - if not pkg_info.dynamic: + if example["is_final_metadata"]: metadata = CoreMetadata.from_dist_info_metadata(text) assert metadata == pkg_info - else: + if example["has_dynamic_fields"]: with pytest.raises(DynamicNotAllowed): CoreMetadata.from_dist_info_metadata(text) for field in ("requires_dist", "provides_dist", "obsoletes_dist"): for value in getattr(pkg_info, field): assert isinstance(value, Requirement) - if pkg_info.description: - desc = pkg_info.description.splitlines() - for line in desc: - assert not line.strip().startswith("|") + desc = pkg_info.description.splitlines() + for line in desc: + assert not line.strip().startswith("|") @pytest.mark.parametrize("spec", PER_VERSION_EXAMPLES.keys()) def test_serliazing(self, spec: str) -> None: - text = bytes(dedent(self.PER_VERSION_EXAMPLES[spec]), "UTF-8") + example = self.PER_VERSION_EXAMPLES[spec] + text = bytes(dedent(example["file_contents"]), "UTF-8") pkg_info = CoreMetadata.from_pkg_info(text) - if not pkg_info.dynamic: + if example["is_final_metadata"]: assert isinstance(pkg_info.to_dist_info_metadata(), bytes) - else: + if example["has_dynamic_fields"]: with pytest.raises(DynamicNotAllowed): pkg_info.to_dist_info_metadata() pkg_info_text = pkg_info.to_pkg_info() From 50c13b7338b34bec2ae0875061132f1b17acb2fe Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 18 Jan 2022 12:00:26 +0000 Subject: [PATCH 08/26] Improve grammar for comment in metadata Co-authored-by: Brett Cannon --- packaging/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/metadata.py b/packaging/metadata.py index af5311d6..66e6ee4c 100644 --- a/packaging/metadata.py +++ b/packaging/metadata.py @@ -165,7 +165,7 @@ def __post_init__(self) -> None: urls[key.strip()] = value.strip() _setattr(self, "project_url", urls) - # Dataclasses don't enforce data types at runtime + # Dataclasses don't enforce data types at runtime. if not isinstance(self.requires_python, SpecifierSet): requires_python = self._parse_requires_python(self.requires_python) _setattr(self, "requires_python", requires_python) From 147592bd6e8010c1746ff79196123bc817e7c762 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 18 Jan 2022 12:03:44 +0000 Subject: [PATCH 09/26] Improve condition in metadata Co-authored-by: Brett Cannon --- packaging/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/metadata.py b/packaging/metadata.py index 66e6ee4c..5c4c02bd 100644 --- a/packaging/metadata.py +++ b/packaging/metadata.py @@ -234,7 +234,7 @@ def from_dist_info_metadata(cls: Type[T], metadata_source: bytes) -> T: _attrs = attrs.copy() _attrs.pop("description", None) - if attrs.get("dynamic"): + if "dynamic" in attrs: raise DynamicNotAllowed(attrs["dynamic"]) missing_fields = [k for k in cls._MANDATORY if not attrs.get(k)] From 3c8952501e7e163561a4886eddf1f4380731c6a2 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 18 Jan 2022 14:51:53 +0000 Subject: [PATCH 10/26] Remove unused instructions in metadata --- packaging/metadata.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packaging/metadata.py b/packaging/metadata.py index 5c4c02bd..adf32569 100644 --- a/packaging/metadata.py +++ b/packaging/metadata.py @@ -231,8 +231,6 @@ def from_dist_info_metadata(cls: Type[T], metadata_source: bytes) -> T: """Parse METADATA data.""" attrs = cls._read_pkg_info(metadata_source) - _attrs = attrs.copy() - _attrs.pop("description", None) if "dynamic" in attrs: raise DynamicNotAllowed(attrs["dynamic"]) From 0b7442ca4c6b5181078de3067536f7a8290f830b Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 18 Jan 2022 16:05:20 +0000 Subject: [PATCH 11/26] Preserve order fields are defined in metadata --- packaging/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/metadata.py b/packaging/metadata.py index adf32569..7b5ad71a 100644 --- a/packaging/metadata.py +++ b/packaging/metadata.py @@ -187,7 +187,7 @@ def metadata_version(self) -> str: @classmethod def _fields(cls) -> Collection[str]: - return frozenset(f.name for f in dataclasses.fields(cls)) + return [f.name for f in dataclasses.fields(cls)] @classmethod def _read_pkg_info(cls, pkg_info: bytes) -> Dict[str, Any]: From 7dce2a3783cc14442bcf31a5943506caaaf46186 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 18 Jan 2022 16:06:33 +0000 Subject: [PATCH 12/26] Make email headers don't leak into the metadata --- packaging/metadata.py | 5 ++--- tests/test_metadata.py | 4 ++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packaging/metadata.py b/packaging/metadata.py index 7b5ad71a..ca71a7b5 100644 --- a/packaging/metadata.py +++ b/packaging/metadata.py @@ -7,7 +7,6 @@ import sys import textwrap from email import message_from_bytes -from email.contentmanager import raw_data_manager from email.headerregistry import Address, AddressHeader from email.message import EmailMessage from email.policy import EmailPolicy, Policy @@ -216,7 +215,7 @@ def _read_pkg_info(cls, pkg_info: bytes) -> Dict[str, Any]: attrs[field] = value if "description" not in attrs: - attrs["description"] = info.get_content(content_manager=raw_data_manager) + attrs["description"] = str(info.get_payload(decode=True), "utf-8") return attrs @@ -262,7 +261,7 @@ def to_pkg_info(self) -> bytes: for kind in sorted(value): info.add_header(key, f"{kind}, {value[kind]}") elif field == "description": - info.set_content(value, content_manager=raw_data_manager) + info.set_payload(bytes(value, "utf-8")) elif field in self._MULTIPLE_USE: for single_value in sorted(str(v) for v in value): info.add_header(key, single_value) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 8c9bc52c..a88989d4 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -241,6 +241,10 @@ def test_serliazing(self, spec: str) -> None: assert len(pkg_info_text) > 0 assert isinstance(pkg_info_text, bytes) + # Make sure email-specific headers don't leak into the generated document + assert b"Content-Transfer-Encoding" not in pkg_info_text + assert b"MIME-Version" not in pkg_info_text + def test_missing_required_fields(self): with pytest.raises(MissingRequiredFields): CoreMetadata.from_dist_info_metadata(b"") From 10439f0c4308c57794d28cd47ab5fd679a76130f Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 18 Jan 2022 16:12:43 +0000 Subject: [PATCH 13/26] Ensure serialized metadata is not empty --- tests/test_metadata.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index a88989d4..3b6a6efb 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -238,9 +238,11 @@ def test_serliazing(self, spec: str) -> None: with pytest.raises(DynamicNotAllowed): pkg_info.to_dist_info_metadata() pkg_info_text = pkg_info.to_pkg_info() - assert len(pkg_info_text) > 0 assert isinstance(pkg_info_text, bytes) - + # Make sure generated document is not empty + assert len(pkg_info_text.strip()) > 0 + assert b"Name" in pkg_info_text + assert b"Metadata-Version" in pkg_info_text # Make sure email-specific headers don't leak into the generated document assert b"Content-Transfer-Encoding" not in pkg_info_text assert b"MIME-Version" not in pkg_info_text From dd57033e2fc89103ff8cc565d9fc8df60a573571 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 18 Jan 2022 17:09:11 +0000 Subject: [PATCH 14/26] Rename _read_pkg_info to _parse_pkg_info --- packaging/metadata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packaging/metadata.py b/packaging/metadata.py index ca71a7b5..9212da94 100644 --- a/packaging/metadata.py +++ b/packaging/metadata.py @@ -189,7 +189,7 @@ def _fields(cls) -> Collection[str]: return [f.name for f in dataclasses.fields(cls)] @classmethod - def _read_pkg_info(cls, pkg_info: bytes) -> Dict[str, Any]: + def _parse_pkg_info(cls, pkg_info: bytes) -> Dict[str, Any]: """Parse PKG-INFO data.""" msg = message_from_bytes(pkg_info, EmailMessage, policy=cls._PARSING_POLICY) @@ -223,13 +223,13 @@ def _read_pkg_info(cls, pkg_info: bytes) -> Dict[str, Any]: def from_pkg_info(cls: Type[T], pkg_info: bytes) -> T: """Parse PKG-INFO data.""" - return cls(**cls._read_pkg_info(pkg_info)) + return cls(**cls._parse_pkg_info(pkg_info)) @classmethod def from_dist_info_metadata(cls: Type[T], metadata_source: bytes) -> T: """Parse METADATA data.""" - attrs = cls._read_pkg_info(metadata_source) + attrs = cls._parse_pkg_info(metadata_source) if "dynamic" in attrs: raise DynamicNotAllowed(attrs["dynamic"]) From 991d65f6aeecb2ccb57ba7663208a567e179fd45 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 18 Jan 2022 20:01:37 +0000 Subject: [PATCH 15/26] Use a plain tuple to represent emails in metadata --- packaging/metadata.py | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/packaging/metadata.py b/packaging/metadata.py index 9212da94..f78a9ec7 100644 --- a/packaging/metadata.py +++ b/packaging/metadata.py @@ -21,7 +21,6 @@ Iterable, Iterator, Mapping, - NamedTuple, Set, Tuple, Type, @@ -83,18 +82,6 @@ def _normalize_field_name_for_dynamic(field: str) -> NormalizedDynamicFields: _setattr = object.__setattr__ -class EmailAddress(NamedTuple): - """Named tuple representing an email address. - For values without a display name use ``EmailAddress(None, "your@email.com")`` - """ - - display_name: Union[str, None] - value: str - - def __str__(self) -> str: - return str(Address(self.display_name or "", addr_spec=self.value)) - - @dataclasses.dataclass(frozen=True) class CoreMetadata: """ @@ -116,7 +103,7 @@ class CoreMetadata: keywords: Collection[str] = () home_page: str = "" author: str = "" - author_email: Collection[EmailAddress] = () + author_email: Collection[Tuple[Union[str, None], str]] = () license: str = "" # license_file: Collection[str] = () # not standard yet # 1.1 @@ -125,7 +112,7 @@ class CoreMetadata: classifier: Collection[str] = () # 1.2 maintainer: str = "" - maintainer_email: Collection[EmailAddress] = () + maintainer_email: Collection[Tuple[Union[str, None], str]] = () requires_dist: Collection[Requirement] = () requires_python: SpecifierSet = dataclasses.field(default_factory=SpecifierSet) requires_external: Collection[str] = () @@ -254,7 +241,7 @@ def to_pkg_info(self) -> bytes: if field == "keywords": info.add_header(key, ",".join(sorted(value))) elif field.endswith("email"): - _emails = (str(v) for v in value) + _emails = (self._serialize_email(v) for v in value) emails = ", ".join(sorted(v for v in _emails if v)) info.add_header(key, emails) elif field == "project_url": @@ -332,21 +319,25 @@ def _convert_single_req(cls, value: Union[Requirement, str]) -> Requirement: @classmethod def _convert_emails( cls, value: Collection[Union[str, Tuple[str, str]]] - ) -> Iterator[EmailAddress]: + ) -> Iterator[Tuple[Union[str, None], str]]: for email in value: if isinstance(email, str): yield from cls._parse_emails(email) elif isinstance(email, tuple) and email[1]: - yield EmailAddress(email[0], email[1]) + yield email @classmethod - def _parse_emails(cls, value: str) -> Iterator[EmailAddress]: + def _parse_emails(cls, value: str) -> Iterator[Tuple[Union[str, None], str]]: singleline = " ".join(value.splitlines()) if singleline.strip() == "UNKNOWN": return address_list = AddressHeader.value_parser(singleline) for mailbox in address_list.all_mailboxes: - yield EmailAddress(mailbox.display_name, mailbox.addr_spec) + yield (mailbox.display_name, mailbox.addr_spec) + + @classmethod + def _serialize_email(cls, value: Tuple[Union[str, None], str]) -> str: + return str(Address(value[0] or "", addr_spec=value[1])) @classmethod def _unescape_description(cls, content: str) -> str: From dda204b44d28338ba98398598eba8daa8ad2d356 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 18 Jan 2022 19:48:22 +0000 Subject: [PATCH 16/26] Revert "Make Requirement hashable" This reverts commit 215c1723cf9ea3190c5dee307a1b9e3e41d3306c. --- packaging/requirements.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/packaging/requirements.py b/packaging/requirements.py index 063a0694..53f9a3aa 100644 --- a/packaging/requirements.py +++ b/packaging/requirements.py @@ -5,7 +5,7 @@ import re import string import urllib.parse -from typing import Any, List, Optional as TOptional, Set +from typing import List, Optional as TOptional, Set from pyparsing import ( # noqa Combine, @@ -144,9 +144,3 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"" - - def __hash__(self) -> int: - return hash((self.__class__.__name__, str(self))) - - def __eq__(self, other: Any) -> bool: - return bool(self.__class__ == other.__class__ and str(self) == str(other)) From f1625d383a36fdbddd3d40a6dc695d353ce1d179 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Tue, 18 Jan 2022 20:35:54 +0000 Subject: [PATCH 17/26] Change metadata tests to not rely on __hash__/__eq__ --- packaging/metadata.py | 7 ++++++- tests/test_metadata.py | 22 ++++++++++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/packaging/metadata.py b/packaging/metadata.py index f78a9ec7..97b8121d 100644 --- a/packaging/metadata.py +++ b/packaging/metadata.py @@ -82,7 +82,12 @@ def _normalize_field_name_for_dynamic(field: str) -> NormalizedDynamicFields: _setattr = object.__setattr__ -@dataclasses.dataclass(frozen=True) +# In the following we use `frozen` to prevent inconsistencies, specially with `dynamic`. +# Comparison is disabled because currently `Requirement` objects are +# unhashable/not-comparable. + + +@dataclasses.dataclass(frozen=True, eq=False) class CoreMetadata: """ Core metadata for Python packages, represented as an immutable diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 3b6a6efb..86f2765c 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -57,7 +57,7 @@ def test_replace(self): } metadata1 = dataclasses.replace(metadata, **attrs) req = next(iter(metadata1.requires_dist)) - assert req == Requirement("appdirs>1.4") + assert str(req) == "appdirs>1.4" with pytest.raises(InvalidCoreMetadataField): dataclasses.replace(metadata, dynamic=["myfield"]) @@ -216,7 +216,7 @@ def test_parsing(self, spec: str) -> None: pkg_info = CoreMetadata.from_pkg_info(text) if example["is_final_metadata"]: metadata = CoreMetadata.from_dist_info_metadata(text) - assert metadata == pkg_info + assert_equal_metadata(metadata, pkg_info) if example["has_dynamic_fields"]: with pytest.raises(DynamicNotAllowed): CoreMetadata.from_dist_info_metadata(text) @@ -309,7 +309,7 @@ def test_parse(self, pkg: str, version: str) -> None: recons_data = from_(recons_file) description = metadata.description.replace("\r\n", "\n") metadata = dataclasses.replace(metadata, description=description) - assert metadata == recons_data + assert_equal_metadata(metadata, recons_data) # - Make sure the reconstructed file can be parsed with compat32 attrs = dataclasses.asdict(_Compat32Metadata.from_pkg_info(recons_file)) assert CoreMetadata(**attrs) @@ -321,13 +321,27 @@ def test_parse(self, pkg: str, version: str) -> None: result_contents = to_(data) assert file_contents == result_contents result_data = from_(result_contents) - assert data == result_data + assert_equal_metadata(data, result_data) file_contents, data = result_contents, result_data # --- Helper Functions/Classes --- +def assert_equal_metadata(metadata1: CoreMetadata, metadata2: CoreMetadata): + fields = (f.name for f in dataclasses.fields(CoreMetadata)) + for field in fields: + value1, value2 = getattr(metadata1, field), getattr(metadata2, field) + if field.endswith("dist"): + # Currently `Requirement` objects are not directly comparable, + # therefore sets containing those objects are also not comparable. + # The best approach is to convert requirements to strings first. + req1, req2 = set(map(str, value1)), set(map(str, value2)) + assert req1 == req2 + else: + assert value1 == value2 + + class _Compat32Metadata(CoreMetadata): """The Core Metadata spec requires the file to be parse-able with compat32. The implementation uses a different approach to ensure UTF-8 can be used. From 95638b31261e37ef62eb07cfb2d6b3e67f0df57b Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Mon, 31 Jan 2022 10:46:25 +0000 Subject: [PATCH 18/26] Use inspect.clean doc to unescape description in core metadata --- packaging/metadata.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packaging/metadata.py b/packaging/metadata.py index 97b8121d..a2c78f9d 100644 --- a/packaging/metadata.py +++ b/packaging/metadata.py @@ -5,12 +5,12 @@ import dataclasses import math import sys -import textwrap from email import message_from_bytes from email.headerregistry import Address, AddressHeader from email.message import EmailMessage from email.policy import EmailPolicy, Policy from functools import reduce +from inspect import cleandoc from itertools import chain from typing import ( TYPE_CHECKING, @@ -347,14 +347,12 @@ def _serialize_email(cls, value: Tuple[Union[str, None], str]) -> str: @classmethod def _unescape_description(cls, content: str) -> str: """Reverse RFC-822 escaping by removing leading whitespaces from content.""" - lines = content.splitlines() + lines = cleandoc(content).splitlines() if not lines: return "" - first_line = lines[0].lstrip() - text = textwrap.dedent("\n".join(lines[1:])) - other_lines = (line.lstrip("|") for line in text.splitlines()) - return "\n".join(chain([first_line], other_lines)) + continuation = (line.lstrip("|") for line in lines[1:]) + return "\n".join(chain(lines[:1], continuation)) def _validate_dynamic(self, normalized: str) -> bool: field = normalized.lower().replace("-", "_") From 31f1d2260a25096bf97f3b18122ca799c7f45587 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Mon, 31 Jan 2022 19:08:43 +0000 Subject: [PATCH 19/26] Remove __post_init__ --- packaging/metadata.py | 181 ++++++++++++++++++++--------------------- tests/test_metadata.py | 28 ++++--- 2 files changed, 104 insertions(+), 105 deletions(-) diff --git a/packaging/metadata.py b/packaging/metadata.py index a2c78f9d..6f5644a0 100644 --- a/packaging/metadata.py +++ b/packaging/metadata.py @@ -130,44 +130,6 @@ class CoreMetadata: # 2.2 dynamic: Collection[NormalizedDynamicFields] = () - def __post_init__(self) -> None: - """Perform required data conversions, validations, and ensure immutability""" - - _should_be_set = (self._MULTIPLE_USE | {"keywords"}) - {"project_url"} - - for field in self._fields(): - value = getattr(self, field) - if field.endswith("dist"): - reqs = (self._convert_single_req(v) for v in value) - _setattr(self, field, frozenset(reqs)) - elif field.endswith("email"): - emails = self._convert_emails(value) - _setattr(self, field, frozenset(emails)) - elif field in _should_be_set: - _setattr(self, field, frozenset(value)) - elif field in {"description", "summary"}: - _setattr(self, field, value.strip()) - - urls = self.project_url - if not isinstance(urls, Mapping) and isinstance(urls, Iterable): - urls = {} - for url in cast(Iterable[str], self.project_url): - key, _, value = url.partition(",") - urls[key.strip()] = value.strip() - _setattr(self, "project_url", urls) - - # Dataclasses don't enforce data types at runtime. - if not isinstance(self.requires_python, SpecifierSet): - requires_python = self._parse_requires_python(self.requires_python) - _setattr(self, "requires_python", requires_python) - if self.version and not isinstance(self.version, Version): - _setattr(self, "version", Version(self.version)) - - if self.dynamic: - values = (_normalize_field_name_for_dynamic(f) for f in self.dynamic) - dynamic = frozenset(v for v in values if self._validate_dynamic(v)) - _setattr(self, "dynamic", dynamic) - @property def metadata_version(self) -> str: """ @@ -181,13 +143,47 @@ def _fields(cls) -> Collection[str]: return [f.name for f in dataclasses.fields(cls)] @classmethod - def _parse_pkg_info(cls, pkg_info: bytes) -> Dict[str, Any]: + def _process_attrs( + cls, attrs: Iterable[Tuple[str, Any]] + ) -> Iterable[Tuple[str, Any]]: + """Transform input data to the matching attribute types.""" + + _as_set = (cls._MULTIPLE_USE | {"keywords"}) - {"project_url"} + _available_fields = cls._fields() + + for field, value in attrs: + if field == "version": + yield ("version", Version(value)) + elif field == "keywords": + yield (field, frozenset(value.split(","))) + elif field == "requires_python": + yield (field, cls._parse_requires_python(value)) + elif field == "project_url": + urls = {} + for url in value: + key, _, value = url.partition(",") + urls[key.strip()] = value.strip() + yield (field, urls) + elif field == "dynamic": + values = (_normalize_field_name_for_dynamic(f) for f in value) + yield (field, frozenset(values)) + elif field.endswith("email"): + yield (field, frozenset(cls._parse_emails(value.strip()))) + elif field.endswith("dist"): + yield (field, frozenset(cls._parse_req(v) for v in value)) + elif field in _as_set: + yield (field, frozenset(value)) + elif field in _available_fields: + yield (field, value) + + @classmethod + def _parse_pkg_info(cls, pkg_info: bytes) -> Iterable[Tuple[str, Any]]: """Parse PKG-INFO data.""" msg = message_from_bytes(pkg_info, EmailMessage, policy=cls._PARSING_POLICY) info = cast(EmailMessage, msg) + has_description = False - attrs: Dict[str, Any] = {} for key in info.keys(): field = key.lower().replace("-", "_") if field in cls._UPDATES: @@ -195,46 +191,41 @@ def _parse_pkg_info(cls, pkg_info: bytes) -> Dict[str, Any]: value = str(info.get(key)) # email.header.Header.__str__ handles encoding - if field == "keywords": - attrs[field] = " ".join(value.splitlines()).split(",") + if field in {"keywords", "summary"} or field.endswith("email"): + yield (field, cls._ensure_single_line(value)) elif field == "description": - attrs[field] = cls._unescape_description(value) - elif field.endswith("email"): - attrs[field] = cls._parse_emails(value) + has_description = True + yield (field, cls._unescape_description(value)) elif field in cls._MULTIPLE_USE: - attrs[field] = (str(v) for v in info.get_all(key)) - elif field in cls._fields(): - attrs[field] = value - - if "description" not in attrs: - attrs["description"] = str(info.get_payload(decode=True), "utf-8") + yield (field, (str(v) for v in info.get_all(key))) + else: + yield (field, value) - return attrs + if not has_description: + yield ("description", str(info.get_payload(decode=True), "utf-8")) @classmethod def from_pkg_info(cls: Type[T], pkg_info: bytes) -> T: """Parse PKG-INFO data.""" - return cls(**cls._parse_pkg_info(pkg_info)) + attrs = cls._process_attrs(cls._parse_pkg_info(pkg_info)) + obj = cls(**dict(attrs)) + obj._validate_dynamic() + return obj @classmethod def from_dist_info_metadata(cls: Type[T], metadata_source: bytes) -> T: """Parse METADATA data.""" - attrs = cls._parse_pkg_info(metadata_source) - - if "dynamic" in attrs: - raise DynamicNotAllowed(attrs["dynamic"]) - - missing_fields = [k for k in cls._MANDATORY if not attrs.get(k)] - if missing_fields: - raise MissingRequiredFields(missing_fields) - - return cls(**attrs) + obj = cls.from_pkg_info(metadata_source) + obj._validate_final_metadata() + return obj def to_pkg_info(self) -> bytes: """Generate PKG-INFO data.""" + self._validate_dynamic() + info = EmailMessage(self._PARSING_POLICY) info.add_header("Metadata-Version", self.metadata_version) # Use `sorted` in collections to improve reproducibility @@ -243,12 +234,13 @@ def to_pkg_info(self) -> bytes: if not value: continue key = self._canonical_field(field) - if field == "keywords": + if field in "keywords": info.add_header(key, ",".join(sorted(value))) elif field.endswith("email"): - _emails = (self._serialize_email(v) for v in value) + _emails = (self._serialize_email(v) for v in value if any(v)) emails = ", ".join(sorted(v for v in _emails if v)) - info.add_header(key, emails) + if emails: + info.add_header(key, emails) elif field == "project_url": for kind in sorted(value): info.add_header(key, f"{kind}, {value[kind]}") @@ -264,8 +256,8 @@ def to_pkg_info(self) -> bytes: def to_dist_info_metadata(self) -> bytes: """Generate METADATA data.""" - if self.dynamic: - raise DynamicNotAllowed(self.dynamic) + + self._validate_final_metadata() return self.to_pkg_info() # --- Auxiliary Methods and Properties --- @@ -300,6 +292,13 @@ def _canonical_field(cls, field: str) -> str: replacements = {"Url": "URL", "Email": "email", "Page": "page"}.items() return reduce(lambda acc, x: acc.replace(x[0], x[1]), replacements, ucfirst) + @classmethod + def _ensure_single_line(cls, value: str) -> str: + """Existing distributions might include metadata with fields such as 'keywords' + or 'summary' showing up as multiline strings. + """ + return " ".join(value.splitlines()) + @classmethod def _parse_requires_python(cls, value: str) -> SpecifierSet: if value and value[0].isnumeric(): @@ -317,26 +316,11 @@ def _parse_req(cls, value: str) -> Requirement: value = f"{name}(=={rest}" return Requirement(value) - @classmethod - def _convert_single_req(cls, value: Union[Requirement, str]) -> Requirement: - return value if isinstance(value, Requirement) else cls._parse_req(value) - - @classmethod - def _convert_emails( - cls, value: Collection[Union[str, Tuple[str, str]]] - ) -> Iterator[Tuple[Union[str, None], str]]: - for email in value: - if isinstance(email, str): - yield from cls._parse_emails(email) - elif isinstance(email, tuple) and email[1]: - yield email - @classmethod def _parse_emails(cls, value: str) -> Iterator[Tuple[Union[str, None], str]]: - singleline = " ".join(value.splitlines()) - if singleline.strip() == "UNKNOWN": + if value == "UNKNOWN": return - address_list = AddressHeader.value_parser(singleline) + address_list = AddressHeader.value_parser(value) for mailbox in address_list.all_mailboxes: yield (mailbox.display_name, mailbox.addr_spec) @@ -354,14 +338,25 @@ def _unescape_description(cls, content: str) -> str: continuation = (line.lstrip("|") for line in lines[1:]) return "\n".join(chain(lines[:1], continuation)) - def _validate_dynamic(self, normalized: str) -> bool: - field = normalized.lower().replace("-", "_") - if not hasattr(self, field): - raise InvalidCoreMetadataField(normalized) - if field in self._NOT_DYNAMIC: - raise InvalidDynamicField(normalized) - if getattr(self, field): - raise StaticFieldCannotBeDynamic(normalized) + def _validate_dynamic(self) -> bool: + for normalized in self.dynamic: + field = normalized.lower().replace("-", "_") + if not hasattr(self, field): + raise InvalidCoreMetadataField(normalized) + if field in self._NOT_DYNAMIC: + raise InvalidDynamicField(normalized) + if getattr(self, field): + raise StaticFieldCannotBeDynamic(normalized) + return True + + def _validate_final_metadata(self) -> bool: + if self.dynamic: + raise DynamicNotAllowed(self.dynamic) + + missing_fields = [k for k in self._MANDATORY if not getattr(self, k)] + if missing_fields: + raise MissingRequiredFields(missing_fields) + return True diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 86f2765c..9c68ce95 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -34,17 +34,19 @@ class TestCoreMetadata: def test_simple(self): - example = {"name": "simple", "version": "0.1", "requires_dist": ["appdirs>1.2"]} - metadata = CoreMetadata(**example) - req = next(iter(metadata.requires_dist)) - assert isinstance(req, Requirement) + example = { + "name": "simple", + "version": "0.1", + "requires_dist": [Requirement("appdirs>1.2")], + } + CoreMetadata(**example) def test_replace(self): example = { "name": "simple", "dynamic": ["version"], - "author_email": ["me@example.com"], - "requires_dist": ["appdirs>1.2"], + "author_email": [(None, "me@example.com")], + "requires_dist": [Requirement("appdirs>1.2")], } metadata = CoreMetadata(**example) @@ -53,18 +55,18 @@ def test_replace(self): "version": "0.2", "dynamic": [], "author_email": [("name", "me@example.com")], - "requires_dist": ["appdirs>1.4"], + "requires_dist": [Requirement("appdirs>1.4")], } metadata1 = dataclasses.replace(metadata, **attrs) req = next(iter(metadata1.requires_dist)) assert str(req) == "appdirs>1.4" with pytest.raises(InvalidCoreMetadataField): - dataclasses.replace(metadata, dynamic=["myfield"]) + dataclasses.replace(metadata, dynamic=["myfield"]).to_pkg_info() with pytest.raises(InvalidDynamicField): - dataclasses.replace(metadata, dynamic=["name"]) + dataclasses.replace(metadata, dynamic=["name"]).to_pkg_info() with pytest.raises(StaticFieldCannotBeDynamic): - dataclasses.replace(metadata, version="0.1") + dataclasses.replace(metadata, version="0.1").to_pkg_info() PER_VERSION_EXAMPLES = { "1.1": { @@ -249,7 +251,7 @@ def test_serliazing(self, spec: str) -> None: def test_missing_required_fields(self): with pytest.raises(MissingRequiredFields): - CoreMetadata.from_dist_info_metadata(b"") + CoreMetadata.from_dist_info_metadata(b"Name: pkg") example = {"name": "pkg", "requires_dist": ["appdirs>1.2"]} metadata = CoreMetadata(**example) @@ -269,7 +271,7 @@ def test_single_line_description(self): assert metadata.description == "Hello World" def test_empty_email(self): - example = {"name": "pkg", "maintainer_email": ["", "", ("", "")]} + example = {"name": "pkg", "maintainer_email": [("", "")]} metadata = CoreMetadata(**example) serialized = metadata.to_pkg_info() assert b"Maintainer-email:" not in serialized @@ -338,6 +340,8 @@ def assert_equal_metadata(metadata1: CoreMetadata, metadata2: CoreMetadata): # The best approach is to convert requirements to strings first. req1, req2 = set(map(str, value1)), set(map(str, value2)) assert req1 == req2 + elif not value1: + assert not value2 else: assert value1 == value2 From adb8ec13d7a646dbb9d8c1568224a08c546150c2 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 2 Feb 2022 09:49:04 +0000 Subject: [PATCH 20/26] Extract url parsing in its own method --- packaging/metadata.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packaging/metadata.py b/packaging/metadata.py index 6f5644a0..d4c2e0fd 100644 --- a/packaging/metadata.py +++ b/packaging/metadata.py @@ -159,11 +159,7 @@ def _process_attrs( elif field == "requires_python": yield (field, cls._parse_requires_python(value)) elif field == "project_url": - urls = {} - for url in value: - key, _, value = url.partition(",") - urls[key.strip()] = value.strip() - yield (field, urls) + yield (field, cls._parse_url(value)) elif field == "dynamic": values = (_normalize_field_name_for_dynamic(f) for f in value) yield (field, frozenset(values)) @@ -316,6 +312,14 @@ def _parse_req(cls, value: str) -> Requirement: value = f"{name}(=={rest}" return Requirement(value) + @classmethod + def _parse_url(cls, value: Iterable[str]) -> Dict[str, str]: + urls = {} + for url in value: + key, _, value = url.partition(",") + urls[key.strip()] = value.strip() + return urls + @classmethod def _parse_emails(cls, value: str) -> Iterator[Tuple[Union[str, None], str]]: if value == "UNKNOWN": From dd6db440850b604881263f97ad575a8c9c6f2d2c Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 2 Feb 2022 10:18:29 +0000 Subject: [PATCH 21/26] Fix dynamic validation --- packaging/metadata.py | 38 +++++++++++++++++++------------------- tests/test_metadata.py | 9 +++------ 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/packaging/metadata.py b/packaging/metadata.py index d4c2e0fd..42e4c780 100644 --- a/packaging/metadata.py +++ b/packaging/metadata.py @@ -77,6 +77,11 @@ def _normalize_field_name_for_dynamic(field: str) -> NormalizedDynamicFields: return cast(NormalizedDynamicFields, field.lower().replace("_", "-")) +def _field_name(field: str) -> str: + """Equivalent field name in the class representing metadata""" + return field.lower().replace("-", "_") + + # Bypass frozen dataclass for __post_init__, this approach is documented in: # https://docs.python.org/3/library/dataclasses.html#frozen-instances _setattr = object.__setattr__ @@ -181,7 +186,7 @@ def _parse_pkg_info(cls, pkg_info: bytes) -> Iterable[Tuple[str, Any]]: has_description = False for key in info.keys(): - field = key.lower().replace("-", "_") + field = _field_name(key) if field in cls._UPDATES: field = cls._UPDATES[field] @@ -343,19 +348,18 @@ def _unescape_description(cls, content: str) -> str: return "\n".join(chain(lines[:1], continuation)) def _validate_dynamic(self) -> bool: - for normalized in self.dynamic: - field = normalized.lower().replace("-", "_") + for item in self.dynamic: + field = _field_name(item) if not hasattr(self, field): - raise InvalidCoreMetadataField(normalized) + raise InvalidCoreMetadataField(item) if field in self._NOT_DYNAMIC: - raise InvalidDynamicField(normalized) - if getattr(self, field): - raise StaticFieldCannotBeDynamic(normalized) + raise InvalidDynamicField(item) return True def _validate_final_metadata(self) -> bool: - if self.dynamic: - raise DynamicNotAllowed(self.dynamic) + unresolved = [k for k in self.dynamic if not getattr(self, _field_name(k))] + if unresolved: + raise UnfilledDynamicFields(unresolved) missing_fields = [k for k in self._MANDATORY if not getattr(self, k)] if missing_fields: @@ -374,18 +378,14 @@ def __init__(self, field: str): super().__init__(f"{field!r} cannot be dynamic") -class StaticFieldCannotBeDynamic(ValueError): - def __init__(self, field: str): - super().__init__(f"{field!r} specified both dynamically and statically") - - -class DynamicNotAllowed(ValueError): - def __init__(self, fields: Collection[str]): - given = ", ".join(fields) - super().__init__(f"Dynamic fields not allowed in this context (given: {given})") +class UnfilledDynamicFields(ValueError): + def __init__(self, fields: Iterable[str]): + given = ", ".join(repr(f) for f in fields) + msg = f"Unfilled dynamic fields not allowed in this context (given: {given})" + super().__init__(msg) class MissingRequiredFields(ValueError): - def __init__(self, fields: Collection[str]): + def __init__(self, fields: Iterable[str]): missing = ", ".join(fields) super().__init__(f"Required fields are missing: {missing}") diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 9c68ce95..6eb9ae82 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -18,11 +18,10 @@ from packaging.metadata import ( CoreMetadata, - DynamicNotAllowed, InvalidCoreMetadataField, InvalidDynamicField, MissingRequiredFields, - StaticFieldCannotBeDynamic, + UnfilledDynamicFields, ) from packaging.requirements import Requirement from packaging.utils import canonicalize_name @@ -65,8 +64,6 @@ def test_replace(self): dataclasses.replace(metadata, dynamic=["myfield"]).to_pkg_info() with pytest.raises(InvalidDynamicField): dataclasses.replace(metadata, dynamic=["name"]).to_pkg_info() - with pytest.raises(StaticFieldCannotBeDynamic): - dataclasses.replace(metadata, version="0.1").to_pkg_info() PER_VERSION_EXAMPLES = { "1.1": { @@ -220,7 +217,7 @@ def test_parsing(self, spec: str) -> None: metadata = CoreMetadata.from_dist_info_metadata(text) assert_equal_metadata(metadata, pkg_info) if example["has_dynamic_fields"]: - with pytest.raises(DynamicNotAllowed): + with pytest.raises(UnfilledDynamicFields): CoreMetadata.from_dist_info_metadata(text) for field in ("requires_dist", "provides_dist", "obsoletes_dist"): for value in getattr(pkg_info, field): @@ -237,7 +234,7 @@ def test_serliazing(self, spec: str) -> None: if example["is_final_metadata"]: assert isinstance(pkg_info.to_dist_info_metadata(), bytes) if example["has_dynamic_fields"]: - with pytest.raises(DynamicNotAllowed): + with pytest.raises(UnfilledDynamicFields): pkg_info.to_dist_info_metadata() pkg_info_text = pkg_info.to_pkg_info() assert isinstance(pkg_info_text, bytes) From f020115695f03d4885e8c75a48d8627635ee2a0c Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 2 Feb 2022 16:21:53 +0000 Subject: [PATCH 22/26] Separate required fields validation --- packaging/metadata.py | 7 ++++++- tests/test_metadata.py | 46 ++++++++++++++++-------------------------- 2 files changed, 23 insertions(+), 30 deletions(-) diff --git a/packaging/metadata.py b/packaging/metadata.py index 42e4c780..bb2d5059 100644 --- a/packaging/metadata.py +++ b/packaging/metadata.py @@ -211,6 +211,7 @@ def from_pkg_info(cls: Type[T], pkg_info: bytes) -> T: attrs = cls._process_attrs(cls._parse_pkg_info(pkg_info)) obj = cls(**dict(attrs)) + obj._validate_required_fields() obj._validate_dynamic() return obj @@ -225,6 +226,7 @@ def from_dist_info_metadata(cls: Type[T], metadata_source: bytes) -> T: def to_pkg_info(self) -> bytes: """Generate PKG-INFO data.""" + self._validate_required_fields() self._validate_dynamic() info = EmailMessage(self._PARSING_POLICY) @@ -266,7 +268,7 @@ def to_dist_info_metadata(self) -> bytes: # (useful when providing a prof-of-concept for new PEPs) _MANDATORY: ClassVar[Set[str]] = {"name", "version"} - _NOT_DYNAMIC: ClassVar[Set[str]] = {"metadata_version", "name", "dynamic"} + _NOT_DYNAMIC: ClassVar[Set[str]] = {"metadata_version", "dynamic"} | _MANDATORY _MULTIPLE_USE: ClassVar[Set[str]] = { "dynamic", "platform", @@ -361,6 +363,9 @@ def _validate_final_metadata(self) -> bool: if unresolved: raise UnfilledDynamicFields(unresolved) + return True + + def _validate_required_fields(self) -> bool: missing_fields = [k for k in self._MANDATORY if not getattr(self, k)] if missing_fields: raise MissingRequiredFields(missing_fields) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 6eb9ae82..98bf458f 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -25,6 +25,7 @@ ) from packaging.requirements import Requirement from packaging.utils import canonicalize_name +from packaging.version import Version HERE = Path(__file__).parent EXAMPLES = HERE / "metadata_examples.csv" @@ -35,31 +36,23 @@ class TestCoreMetadata: def test_simple(self): example = { "name": "simple", - "version": "0.1", + "version": Version("0.1"), "requires_dist": [Requirement("appdirs>1.2")], } - CoreMetadata(**example) + metadata = CoreMetadata(**example) + assert isinstance(metadata.to_pkg_info(), bytes) - def test_replace(self): + def test_invalid(self): example = { "name": "simple", - "dynamic": ["version"], "author_email": [(None, "me@example.com")], "requires_dist": [Requirement("appdirs>1.2")], } metadata = CoreMetadata(**example) + with pytest.raises(MissingRequiredFields): # version is missing + metadata.to_pkg_info() - # Make sure replace goes through validations and transformations - attrs = { - "version": "0.2", - "dynamic": [], - "author_email": [("name", "me@example.com")], - "requires_dist": [Requirement("appdirs>1.4")], - } - metadata1 = dataclasses.replace(metadata, **attrs) - req = next(iter(metadata1.requires_dist)) - assert str(req) == "appdirs>1.4" - + metadata = dataclasses.replace(metadata, version=Version("0.42")) with pytest.raises(InvalidCoreMetadataField): dataclasses.replace(metadata, dynamic=["myfield"]).to_pkg_info() with pytest.raises(InvalidDynamicField): @@ -246,29 +239,24 @@ def test_serliazing(self, spec: str) -> None: assert b"Content-Transfer-Encoding" not in pkg_info_text assert b"MIME-Version" not in pkg_info_text - def test_missing_required_fields(self): - with pytest.raises(MissingRequiredFields): - CoreMetadata.from_dist_info_metadata(b"Name: pkg") - - example = {"name": "pkg", "requires_dist": ["appdirs>1.2"]} - metadata = CoreMetadata(**example) - serialized = metadata.to_pkg_info() - with pytest.raises(MissingRequiredFields): - CoreMetadata.from_dist_info_metadata(serialized) - def test_empty_fields(self): - metadata = CoreMetadata.from_pkg_info(b"Name: pkg\nDescription:\n") + metadata = CoreMetadata.from_pkg_info(b"Name: pkg\nVersion: 1\nDescription:\n") assert metadata.description == "" - metadata = CoreMetadata.from_pkg_info(b"Name: pkg\nAuthor-email:\n") + metadata = CoreMetadata.from_pkg_info(b"Name: pkg\nVersion: 1\nAuthor-email:\n") assert metadata.description == "" assert len(metadata.author_email) == 0 def test_single_line_description(self): - metadata = CoreMetadata.from_pkg_info(b"Name: pkg\nDescription: Hello World") + serialized = b"Name: pkg\nVersion: 1\nDescription: Hello World" + metadata = CoreMetadata.from_pkg_info(serialized) assert metadata.description == "Hello World" def test_empty_email(self): - example = {"name": "pkg", "maintainer_email": [("", "")]} + example = { + "name": "pkg", + "version": Version("1"), + "maintainer_email": [("", "")], + } metadata = CoreMetadata(**example) serialized = metadata.to_pkg_info() assert b"Maintainer-email:" not in serialized From bac39bce0cc092bf0d461937f94c20b2abee516d Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 2 Feb 2022 15:55:18 +0000 Subject: [PATCH 23/26] Remove unused _setattr --- packaging/metadata.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packaging/metadata.py b/packaging/metadata.py index bb2d5059..8672ff61 100644 --- a/packaging/metadata.py +++ b/packaging/metadata.py @@ -82,11 +82,6 @@ def _field_name(field: str) -> str: return field.lower().replace("-", "_") -# Bypass frozen dataclass for __post_init__, this approach is documented in: -# https://docs.python.org/3/library/dataclasses.html#frozen-instances -_setattr = object.__setattr__ - - # In the following we use `frozen` to prevent inconsistencies, specially with `dynamic`. # Comparison is disabled because currently `Requirement` objects are # unhashable/not-comparable. From 1ed4123457f2296960d8c5cd330877bf573a0263 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 2 Feb 2022 15:58:38 +0000 Subject: [PATCH 24/26] Remove 'frozen' behaviour from metadata --- packaging/metadata.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/packaging/metadata.py b/packaging/metadata.py index 8672ff61..f8c38867 100644 --- a/packaging/metadata.py +++ b/packaging/metadata.py @@ -82,12 +82,11 @@ def _field_name(field: str) -> str: return field.lower().replace("-", "_") -# In the following we use `frozen` to prevent inconsistencies, specially with `dynamic`. -# Comparison is disabled because currently `Requirement` objects are -# unhashable/not-comparable. +# In the following, comparison is disabled because currently `Requirement` +# objects are unhashable/not-comparable. -@dataclasses.dataclass(frozen=True, eq=False) +@dataclasses.dataclass(eq=False) class CoreMetadata: """ Core metadata for Python packages, represented as an immutable @@ -155,20 +154,20 @@ def _process_attrs( if field == "version": yield ("version", Version(value)) elif field == "keywords": - yield (field, frozenset(value.split(","))) + yield (field, set(value.split(","))) elif field == "requires_python": yield (field, cls._parse_requires_python(value)) elif field == "project_url": yield (field, cls._parse_url(value)) elif field == "dynamic": values = (_normalize_field_name_for_dynamic(f) for f in value) - yield (field, frozenset(values)) + yield (field, set(values)) elif field.endswith("email"): - yield (field, frozenset(cls._parse_emails(value.strip()))) + yield (field, set(cls._parse_emails(value.strip()))) elif field.endswith("dist"): - yield (field, frozenset(cls._parse_req(v) for v in value)) + yield (field, {cls._parse_req(v) for v in value}) elif field in _as_set: - yield (field, frozenset(value)) + yield (field, set(value)) elif field in _available_fields: yield (field, value) From 1ca35734469d08fe07d40ef028471c6a74c70647 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Wed, 2 Feb 2022 16:37:11 +0000 Subject: [PATCH 25/26] Unify {to/from}_pkg_info with {to/from}_dist_info_metadata As discussed in #383 instead of having 2 separated sets of methods (one for `PKG-INFO` files in sdists and one for `METADATA` files in wheels) we can have a single pair to/from functions with an `allow_unfilled_dynamic` keyword argument. --- packaging/metadata.py | 36 ++++++++++++++++-------------------- tests/test_metadata.py | 17 +++++++++-------- 2 files changed, 25 insertions(+), 28 deletions(-) diff --git a/packaging/metadata.py b/packaging/metadata.py index f8c38867..ca28ceb8 100644 --- a/packaging/metadata.py +++ b/packaging/metadata.py @@ -200,28 +200,21 @@ def _parse_pkg_info(cls, pkg_info: bytes) -> Iterable[Tuple[str, Any]]: yield ("description", str(info.get_payload(decode=True), "utf-8")) @classmethod - def from_pkg_info(cls: Type[T], pkg_info: bytes) -> T: + def from_pkg_info( + cls: Type[T], pkg_info: bytes, *, allow_unfilled_dynamic: bool = True + ) -> T: """Parse PKG-INFO data.""" attrs = cls._process_attrs(cls._parse_pkg_info(pkg_info)) obj = cls(**dict(attrs)) - obj._validate_required_fields() - obj._validate_dynamic() - return obj - - @classmethod - def from_dist_info_metadata(cls: Type[T], metadata_source: bytes) -> T: - """Parse METADATA data.""" + obj._validate(allow_unfilled_dynamic) - obj = cls.from_pkg_info(metadata_source) - obj._validate_final_metadata() return obj - def to_pkg_info(self) -> bytes: + def to_pkg_info(self, *, allow_unfilled_dynamic: bool = True) -> bytes: """Generate PKG-INFO data.""" - self._validate_required_fields() - self._validate_dynamic() + self._validate(allow_unfilled_dynamic) info = EmailMessage(self._PARSING_POLICY) info.add_header("Metadata-Version", self.metadata_version) @@ -251,12 +244,6 @@ def to_pkg_info(self) -> bytes: return info.as_bytes() - def to_dist_info_metadata(self) -> bytes: - """Generate METADATA data.""" - - self._validate_final_metadata() - return self.to_pkg_info() - # --- Auxiliary Methods and Properties --- # Not part of the API, but can be overwritten by subclasses # (useful when providing a prof-of-concept for new PEPs) @@ -343,6 +330,14 @@ def _unescape_description(cls, content: str) -> str: continuation = (line.lstrip("|") for line in lines[1:]) return "\n".join(chain(lines[:1], continuation)) + def _validate(self, allow_unfilled_dynamic: bool) -> bool: + self._validate_required_fields() + self._validate_dynamic() + if not allow_unfilled_dynamic: + self._validate_unfilled_dynamic() + + return True + def _validate_dynamic(self) -> bool: for item in self.dynamic: field = _field_name(item) @@ -350,9 +345,10 @@ def _validate_dynamic(self) -> bool: raise InvalidCoreMetadataField(item) if field in self._NOT_DYNAMIC: raise InvalidDynamicField(item) + return True - def _validate_final_metadata(self) -> bool: + def _validate_unfilled_dynamic(self) -> bool: unresolved = [k for k in self.dynamic if not getattr(self, _field_name(k))] if unresolved: raise UnfilledDynamicFields(unresolved) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 98bf458f..ca0212e1 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -6,6 +6,7 @@ import json import tarfile from email.policy import compat32 +from functools import partial from hashlib import md5 from itertools import chain from pathlib import Path @@ -207,11 +208,11 @@ def test_parsing(self, spec: str) -> None: text = bytes(dedent(example["file_contents"]), "UTF-8") pkg_info = CoreMetadata.from_pkg_info(text) if example["is_final_metadata"]: - metadata = CoreMetadata.from_dist_info_metadata(text) + metadata = CoreMetadata.from_pkg_info(text, allow_unfilled_dynamic=False) assert_equal_metadata(metadata, pkg_info) if example["has_dynamic_fields"]: with pytest.raises(UnfilledDynamicFields): - CoreMetadata.from_dist_info_metadata(text) + CoreMetadata.from_pkg_info(text, allow_unfilled_dynamic=False) for field in ("requires_dist", "provides_dist", "obsoletes_dist"): for value in getattr(pkg_info, field): assert isinstance(value, Requirement) @@ -225,10 +226,10 @@ def test_serliazing(self, spec: str) -> None: text = bytes(dedent(example["file_contents"]), "UTF-8") pkg_info = CoreMetadata.from_pkg_info(text) if example["is_final_metadata"]: - assert isinstance(pkg_info.to_dist_info_metadata(), bytes) + assert isinstance(pkg_info.to_pkg_info(allow_unfilled_dynamic=False), bytes) if example["has_dynamic_fields"]: with pytest.raises(UnfilledDynamicFields): - pkg_info.to_dist_info_metadata() + pkg_info.to_pkg_info(allow_unfilled_dynamic=False) pkg_info_text = pkg_info.to_pkg_info() assert isinstance(pkg_info_text, bytes) # Make sure generated document is not empty @@ -274,14 +275,14 @@ class TestIntegration: @pytest.mark.parametrize("pkg, version", examples()) def test_parse(self, pkg: str, version: str) -> None: for dist in download_dists(pkg, version): + from_ = CoreMetadata.from_pkg_info + to_ = CoreMetadata.to_pkg_info if dist.suffix == ".whl": orig = read_metadata(dist) - from_ = CoreMetadata.from_dist_info_metadata - to_ = CoreMetadata.to_dist_info_metadata + from_ = partial(from_, allow_unfilled_dynamic=False) + to_ = partial(to_, allow_unfilled_dynamic=False) else: orig = read_pkg_info(dist) - from_ = CoreMetadata.from_pkg_info - to_ = CoreMetadata.to_pkg_info # Given PKG-INFO or METADATA from existing packages on PyPI # - Make sure they can be parsed From 73eef47178b64b3ede259a5b11a7be02fd3fece1 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 4 Feb 2022 10:56:07 +0000 Subject: [PATCH 26/26] Use type_extensions in TYPE_CHECKING guard As indicated in the [code review](https://github.com/pypa/packaging/pull/498#discussion_r798857966) type checkers consider `typing_extensions` as part of the stdlib, so it does not need to be explicitly installed. --- packaging/metadata.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packaging/metadata.py b/packaging/metadata.py index ca28ceb8..1375df52 100644 --- a/packaging/metadata.py +++ b/packaging/metadata.py @@ -4,7 +4,6 @@ import dataclasses import math -import sys from email import message_from_bytes from email.headerregistry import Address, AddressHeader from email.message import EmailMessage @@ -37,8 +36,8 @@ A = TypeVar("A") B = TypeVar("B") -if sys.version_info[:2] >= (3, 8) and TYPE_CHECKING: # pragma: no cover - from typing import Literal +if TYPE_CHECKING: # pragma: no cover + from typing_extensions import Literal NormalizedDynamicFields = Literal[ "platform",