From 96fb27cad288902f90ae6f05c316eba9334ec82a Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Mon, 5 Feb 2024 22:03:02 -0800 Subject: [PATCH 01/12] salt 3006 --- packages.ini | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/packages.ini b/packages.ini index 8a92fbff..a59b0f87 100644 --- a/packages.ini +++ b/packages.ini @@ -21,6 +21,8 @@ python_versions = <3.11 [anyio==3.6.1] [anyio==3.7.1] +[apache-libcloud==2.5.0] + [asgiref==3.5.2] [asgiref==3.6.0] [asgiref==3.7.2] @@ -152,6 +154,10 @@ brew_requires = libffi [charset-normalizer==3.3.0] [charset-normalizer==3.3.2] +[cheroot==8.5.2] + +[cherrypy==18.6.1] + [click==7.1.2] [click==8.0.4] [click==8.1.3] @@ -396,8 +402,10 @@ validate_incorrect_missing_deps = six [frozenlist==1.4.1] [gitdb==4.0.5] +[gitdb==4.0.7] [gitpython==3.1.12] +[gitpython==3.1.41] [gojsonnet==0.20.0] apt_requires = golang @@ -536,6 +544,7 @@ validate_extras = pytest [idna==2.8] [idna==2.10] +[idna==3.2] [idna==3.3] [idna==3.4] [idna==3.6] @@ -547,6 +556,7 @@ validate_extras = pytest [importlib-metadata==3.10.1] [importlib-metadata==4.6.4] [importlib-metadata==4.12.0] +[importlib-metadata==6.0.0] [importlib-resources==5.8.0] [importlib-resources==5.9.0] @@ -564,9 +574,18 @@ validate_extras = pytest [itsdangerous==2.1.2] +[jaraco-classes==3.2.1] + +[jaraco-collections==3.4.0] + +[jaraco-functools==2.0] + +[jaraco-text==3.5.1] + [jinja2==3.0.3] [jinja2==3.1.0] [jinja2==3.1.2] +[jinja2==3.1.3] [jmespath==0.10.0] [jmespath==1.0.1] @@ -614,6 +633,10 @@ python_versions = <3.11 [libcst==0.4.7] python_versions = <3.11 +[linode-python==1.1.1] + +[looseversion==1.0.2] + [lxml==4.6.5] apt_requires = libxml2-dev @@ -689,6 +712,7 @@ brew_requires = libmaxminddb [mock==4.0.3] +[more-itertools==8.2.0] [more-itertools==8.13.0] [more-itertools==8.14.0] [more-itertools==9.0.0] @@ -696,7 +720,6 @@ brew_requires = libmaxminddb [more-itertools==10.1.0] [msgpack==1.0.2] -python_versions = <3.11 [msgpack==1.0.4] [msgpack-types==0.2.0] @@ -764,6 +787,7 @@ python_versions = <3.11 [outcome==1.3.0.post0] [packaging==21.3] +[packaging==22.0] [packaging==23.0] [packaging==23.1] [packaging==23.2] @@ -847,6 +871,8 @@ python_versions = <3.11 [pluggy==1.3.0] [pluggy==1.4.0] +[portend==2.6] + [pre-commit==2.18.1] [pre-commit==2.20.0] [pre-commit==3.0.4] @@ -925,6 +951,7 @@ brew_requires = [pycparser==2.21] [pycryptodomex==3.9.8] +[pycryptodomex==3.19.1] [pydantic==1.10.2] [pydantic==1.10.9] @@ -951,6 +978,7 @@ brew_requires = [pyopenssl==22.0.0] [pyopenssl==23.0.0] +[pyopenssl==23.2.0] [pyparsing==3.0.9] @@ -1078,6 +1106,7 @@ brew_requires = libyaml apt_requires = libyaml-dev brew_requires = libyaml +[pyzmq==23.2.0] [pyzmq==24.0.0] [rb==1.9.0] @@ -1149,6 +1178,9 @@ python_versions = <3.11 [s3transfer==0.6.1] [s3transfer==0.10.0] +[salt==3006.6] +python_versions = <3.11 + [selenium==4.3.0] [selenium==4.4.3] [selenium==4.11.2] @@ -1463,6 +1495,8 @@ python_versions = >=3.10 [sentry-usage-accountant==0.0.10] [sentry-usage-accountant==0.0.11] +[setproctitle==1.3.2] + [setuptools==56.0.0] [setuptools==62.3.2] [setuptools==63.1.0] @@ -1492,6 +1526,7 @@ python_versions = >=3.10 [six==1.16.0] [smmap==3.0.2] +[smmap==4.0.0] [sniffio==1.2.0] [sniffio==1.3.0] @@ -1611,6 +1646,8 @@ validate_incorrect_missing_deps = beautifulsoup4 [tabulate==0.8.9] [tabulate==0.8.10] +[tempora==4.1.1] + [time-machine==2.12.0] [time-machine==2.13.0] @@ -1800,6 +1837,7 @@ python_versions = <3.11 [urllib3==1.26.13] [urllib3==1.26.14] [urllib3==1.26.16] +[urllib3==1.26.18] [urllib3==2.0.2] [urllib3==2.0.3] [urllib3==2.0.4] @@ -1821,6 +1859,8 @@ python_versions = <3.11 [virtualenv==20.24.7] [virtualenv==20.25.0] +[vultr==1.0.1] + [watchdog==2.1.9] [watchdog==3.0.0] @@ -1881,7 +1921,10 @@ custom_prebuild = prebuild/libxmlsec1-macos [yarl==1.9.2] [yarl==1.9.4] +[zc-lockfile==2.0] + [zipp==3.5.0] [zipp==3.8.1] +[zipp==3.12.0] [zstandard==0.18.0] From cdcf11fd1f703dc761f980c5ba970b84d78da06c Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Mon, 5 Feb 2024 23:45:09 -0800 Subject: [PATCH 02/12] oh --- packages.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/packages.ini b/packages.ini index a59b0f87..57e34783 100644 --- a/packages.ini +++ b/packages.ini @@ -720,6 +720,7 @@ brew_requires = libmaxminddb [more-itertools==10.1.0] [msgpack==1.0.2] +python_versions = <3.11 [msgpack==1.0.4] [msgpack-types==0.2.0] From ed56730dc051a3e22be07586172235c7ec1ff60e Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Mon, 5 Feb 2024 23:56:37 -0800 Subject: [PATCH 03/12] ah its not supported on 3.11 --- packages.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/packages.ini b/packages.ini index 57e34783..5d4d6d9c 100644 --- a/packages.ini +++ b/packages.ini @@ -1108,6 +1108,7 @@ apt_requires = libyaml-dev brew_requires = libyaml [pyzmq==23.2.0] +python_versions = <3.11 [pyzmq==24.0.0] [rb==1.9.0] From 3bf24d62657d6fc0f5ea9de3e591997fe997ea99 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 6 Feb 2024 00:25:27 -0800 Subject: [PATCH 04/12] fix and add tests --- README.md | 2 +- build.py | 5 ++--- tests/build_test.py | 21 ++++++++++++++++----- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index b2a93acb..adba0509 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ source files: ```ini [salt==3006.6] likely_binary_ignore = - pkg/smartos/esky/sodium_grabber.c + salt-3006.6/pkg/old/smartos/esky/sodium_grabber.c ``` ### python_versions diff --git a/build.py b/build.py index da1717b9..cc417137 100644 --- a/build.py +++ b/build.py @@ -500,7 +500,7 @@ def _prefix_path(*parts: str) -> str: env.update(before) -def _likely_binary(sdist: str, likely_binary_ignore: tuple[str, ...]]) -> str | None: +def _likely_binary(sdist: str, likely_binary_ignore: tuple[str, ...]) -> str | None: if sdist.endswith(".zip"): with zipfile.ZipFile(sdist) as zipf: names = zipf.namelist() @@ -528,8 +528,7 @@ def _likely_binary(sdist: str, likely_binary_ignore: tuple[str, ...]]) -> str | if "/test/" in name or "/tests/" in name: continue - print(name) - if name in package.likely_binary_ignore: + if name in likely_binary_ignore: continue _, ext = os.path.splitext(name) diff --git a/tests/build_test.py b/tests/build_test.py index 2c882398..67e0d24b 100644 --- a/tests/build_test.py +++ b/tests/build_test.py @@ -58,6 +58,7 @@ def test_package_default(): apt_requires=(), brew_requires=(), custom_prebuild=(), + likely_binary_ignore=(), python_versions=SpecifierSet(), ) @@ -83,6 +84,7 @@ def test_package_parses_split_values(): apt_requires=("pkg-config", "libxslt1-dev"), brew_requires=("pkg-config", "libxml"), custom_prebuild=("prebuild/crc32c", "deadbeef"), + likely_binary_ignore=(), python_versions=SpecifierSet(), ) @@ -238,7 +240,7 @@ def test_likely_binary_zip(tmp_path): zipf.open("a-1/src/_ext.c", "w").close() zipf.open("a-1/src/_ext.pyx", "w").close() - reason = build._likely_binary(str(filename)) + reason = build._likely_binary(str(filename), ()) assert reason == "sdist contains files with these extensions: .c, .pyx" @@ -248,7 +250,7 @@ def test_likely_binary_tgz(tmp_path): tarf.addfile(tarfile.TarInfo("a-1/src/ext.py")) tarf.addfile(tarfile.TarInfo("a-1/src/_ext.c")) - reason = build._likely_binary(str(filename)) + reason = build._likely_binary(str(filename), ()) assert reason == "sdist contains files with these extensions: .c" @@ -265,7 +267,7 @@ def test_likely_binary_cffi_zip(tmp_path): b" setup()\n" ) - reason = build._likely_binary(str(filename)) + reason = build._likely_binary(str(filename), ()) assert reason == "sdist setup.py has `cffi_modules`" @@ -284,7 +286,7 @@ def test_likely_binary_cffi_tar(tmp_path): tar_info.size = len(bio.getvalue()) tarf.addfile(tar_info, bio) - reason = build._likely_binary(str(filename)) + reason = build._likely_binary(str(filename), ()) assert reason == "sdist setup.py has `cffi_modules`" @@ -294,7 +296,16 @@ def test_likely_binary_ignores_test_files(tmp_path): tarf.addfile(tarfile.TarInfo("a-1/test/_ext.pyd")) tarf.addfile(tarfile.TarInfo("a-1/tests/_ext.c")) - reason = build._likely_binary(str(filename)) + reason = build._likely_binary(str(filename), ()) + assert reason is None + + +def test_likely_binary_ignore(tmp_path): + filename = tmp_path.joinpath("a-1.tar.gz") + with tarfile.open(filename, "w:gz") as tarf: + tarf.addfile(tarfile.TarInfo("a-1/foo/bar.c")) + + reason = build._likely_binary(str(filename), ("a-1/foo/bar.c",)) assert reason is None From c9242250dcb421803533cd81f4439651c7ce3f0e Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 6 Feb 2024 00:28:10 -0800 Subject: [PATCH 05/12] . --- packages.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages.ini b/packages.ini index 5d4d6d9c..423fa634 100644 --- a/packages.ini +++ b/packages.ini @@ -1182,6 +1182,8 @@ python_versions = <3.11 [salt==3006.6] python_versions = <3.11 +likely_binary_ignore = + salt-3006.6/pkg/old/smartos/esky/sodium_grabber.c [selenium==4.3.0] [selenium==4.4.3] From 0c038e438b463a9ea4974fce621bb9383379a480 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 6 Feb 2024 00:31:09 -0800 Subject: [PATCH 06/12] cherrypy < 3.11 --- packages.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages.ini b/packages.ini index 423fa634..ee493323 100644 --- a/packages.ini +++ b/packages.ini @@ -157,6 +157,7 @@ brew_requires = libffi [cheroot==8.5.2] [cherrypy==18.6.1] +python_versions = <3.11 [click==7.1.2] [click==8.0.4] @@ -1181,9 +1182,8 @@ python_versions = <3.11 [s3transfer==0.10.0] [salt==3006.6] +likely_binary_ignore = salt-3006.6/pkg/old/smartos/esky/sodium_grabber.c python_versions = <3.11 -likely_binary_ignore = - salt-3006.6/pkg/old/smartos/esky/sodium_grabber.c [selenium==4.3.0] [selenium==4.4.3] From b2649a19d400882e7ee58645ae67278d2b9dac01 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 6 Feb 2024 00:47:27 -0800 Subject: [PATCH 07/12] xdd --- tests/validate_test.py | 4 ++++ validate.py | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/tests/validate_test.py b/tests/validate_test.py index 69e2030b..dae961b3 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -4,6 +4,7 @@ import pytest from packaging.tags import parse_tag +from packaging.specifiers import SpecifierSet import validate @@ -14,6 +15,7 @@ def test_info_nothing_supplied(): validate_extras=None, validate_incorrect_missing_deps=(), validate_skip_imports=(), + python_versions=SpecifierSet(), ) assert info == expected @@ -24,12 +26,14 @@ def test_info_all_supplied(): "validate_extras": "d", "validate_incorrect_missing_deps": "six", "validate_skip_imports": "uwsgidecorators", + "python_versions": "<3.11", } ) expected = validate.Info( validate_extras="d", validate_incorrect_missing_deps=("six",), validate_skip_imports=("uwsgidecorators",), + python_versions=SpecifierSet("<3.11"), ) assert info == expected diff --git a/validate.py b/validate.py index ec50ad24..4b261e2f 100644 --- a/validate.py +++ b/validate.py @@ -11,6 +11,7 @@ from collections.abc import Mapping from typing import NamedTuple +from packaging.specifiers import SpecifierSet from packaging.tags import Tag from packaging.utils import parse_wheel_filename from packaging.version import Version @@ -23,15 +24,18 @@ class Info(NamedTuple): validate_extras: str | None validate_incorrect_missing_deps: tuple[str, ...] validate_skip_imports: tuple[str, ...] + python_versions: SpecifierSet @classmethod def from_dct(cls, dct: Mapping[str, str]) -> Info: + python_versions = dct.get("python_versions") or "" return cls( validate_extras=dct.get("validate_extras") or None, validate_incorrect_missing_deps=tuple( dct.get("validate_incorrect_missing_deps", "").split() ), validate_skip_imports=tuple(dct.get("validate_skip_imports", "").split()), + python_versions=SpecifierSet(python_versions), ) @@ -92,6 +96,10 @@ def _validate( info: Info, index_url: str, ) -> None: + # quick hack before i go to bed - py_exe just prepends python to {major}.{minor} + if python[6:] not in info.python_versions: + print(f"skipping validating {python}: {filename}") + return print(f"validating {python}: {filename}") with tempfile.TemporaryDirectory() as tmpdir: print("creating env") From 430e85505ee9e5842602edb1b4b730b8908b4944 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 6 Feb 2024 00:47:55 -0800 Subject: [PATCH 08/12] [skip ci] forgot to pc --- tests/validate_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/validate_test.py b/tests/validate_test.py index dae961b3..1b211288 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -3,8 +3,8 @@ import zipfile import pytest -from packaging.tags import parse_tag from packaging.specifiers import SpecifierSet +from packaging.tags import parse_tag import validate From e5f06e2394f5d340c5150af926e186b01ee7dc92 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 6 Feb 2024 10:17:59 -0800 Subject: [PATCH 09/12] Revert "[skip ci] forgot to pc" This reverts commit 430e85505ee9e5842602edb1b4b730b8908b4944. --- tests/validate_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/validate_test.py b/tests/validate_test.py index 1b211288..dae961b3 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -3,8 +3,8 @@ import zipfile import pytest -from packaging.specifiers import SpecifierSet from packaging.tags import parse_tag +from packaging.specifiers import SpecifierSet import validate From 8651b20dae01cbce5c6512c9dfb7993ed8da808e Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 6 Feb 2024 10:18:14 -0800 Subject: [PATCH 10/12] Revert "xdd" This reverts commit b2649a19d400882e7ee58645ae67278d2b9dac01. --- tests/validate_test.py | 4 ---- validate.py | 8 -------- 2 files changed, 12 deletions(-) diff --git a/tests/validate_test.py b/tests/validate_test.py index dae961b3..69e2030b 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -4,7 +4,6 @@ import pytest from packaging.tags import parse_tag -from packaging.specifiers import SpecifierSet import validate @@ -15,7 +14,6 @@ def test_info_nothing_supplied(): validate_extras=None, validate_incorrect_missing_deps=(), validate_skip_imports=(), - python_versions=SpecifierSet(), ) assert info == expected @@ -26,14 +24,12 @@ def test_info_all_supplied(): "validate_extras": "d", "validate_incorrect_missing_deps": "six", "validate_skip_imports": "uwsgidecorators", - "python_versions": "<3.11", } ) expected = validate.Info( validate_extras="d", validate_incorrect_missing_deps=("six",), validate_skip_imports=("uwsgidecorators",), - python_versions=SpecifierSet("<3.11"), ) assert info == expected diff --git a/validate.py b/validate.py index 4b261e2f..ec50ad24 100644 --- a/validate.py +++ b/validate.py @@ -11,7 +11,6 @@ from collections.abc import Mapping from typing import NamedTuple -from packaging.specifiers import SpecifierSet from packaging.tags import Tag from packaging.utils import parse_wheel_filename from packaging.version import Version @@ -24,18 +23,15 @@ class Info(NamedTuple): validate_extras: str | None validate_incorrect_missing_deps: tuple[str, ...] validate_skip_imports: tuple[str, ...] - python_versions: SpecifierSet @classmethod def from_dct(cls, dct: Mapping[str, str]) -> Info: - python_versions = dct.get("python_versions") or "" return cls( validate_extras=dct.get("validate_extras") or None, validate_incorrect_missing_deps=tuple( dct.get("validate_incorrect_missing_deps", "").split() ), validate_skip_imports=tuple(dct.get("validate_skip_imports", "").split()), - python_versions=SpecifierSet(python_versions), ) @@ -96,10 +92,6 @@ def _validate( info: Info, index_url: str, ) -> None: - # quick hack before i go to bed - py_exe just prepends python to {major}.{minor} - if python[6:] not in info.python_versions: - print(f"skipping validating {python}: {filename}") - return print(f"validating {python}: {filename}") with tempfile.TemporaryDirectory() as tmpdir: print("creating env") From c6f10ae808223c6b0357f361d54aa36dd34d9fbb Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 6 Feb 2024 10:18:21 -0800 Subject: [PATCH 11/12] Revert "cherrypy < 3.11" This reverts commit 0c038e438b463a9ea4974fce621bb9383379a480. --- packages.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages.ini b/packages.ini index ee493323..423fa634 100644 --- a/packages.ini +++ b/packages.ini @@ -157,7 +157,6 @@ brew_requires = libffi [cheroot==8.5.2] [cherrypy==18.6.1] -python_versions = <3.11 [click==7.1.2] [click==8.0.4] @@ -1182,8 +1181,9 @@ python_versions = <3.11 [s3transfer==0.10.0] [salt==3006.6] -likely_binary_ignore = salt-3006.6/pkg/old/smartos/esky/sodium_grabber.c python_versions = <3.11 +likely_binary_ignore = + salt-3006.6/pkg/old/smartos/esky/sodium_grabber.c [selenium==4.3.0] [selenium==4.4.3] From 4a008b7a374a691f15a1687e332353095ddbfa10 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 6 Feb 2024 10:19:00 -0800 Subject: [PATCH 12/12] remove problematic packages --- packages.ini | 7 ------- 1 file changed, 7 deletions(-) diff --git a/packages.ini b/packages.ini index 423fa634..149fe97b 100644 --- a/packages.ini +++ b/packages.ini @@ -156,8 +156,6 @@ brew_requires = libffi [cheroot==8.5.2] -[cherrypy==18.6.1] - [click==7.1.2] [click==8.0.4] [click==8.1.3] @@ -1180,11 +1178,6 @@ python_versions = <3.11 [s3transfer==0.6.1] [s3transfer==0.10.0] -[salt==3006.6] -python_versions = <3.11 -likely_binary_ignore = - salt-3006.6/pkg/old/smartos/esky/sodium_grabber.c - [selenium==4.3.0] [selenium==4.4.3] [selenium==4.11.2]