From f2ddefdb1af9d6b17f40e2435973c1d49f643b22 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Wed, 14 Sep 2022 15:12:16 -0700 Subject: [PATCH 1/2] likely_binary_ignore (cherry picked from commit d44ca602863bfa29ab1b01579f67c8d353404ad8) --- README.md | 14 ++++++++++++++ build.py | 10 ++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0761c8f2..fc51486d 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,20 @@ the script should set up whatever tools are necessary inside only that directory custom_prebuild = prebuild/crc32c 1.1.2 ``` +### likely_binary_ignore + +when the sdist has source files of a compiled language (see BINARY_EXTS), +we would expect compiled things (e.g. binary executables, shared objects) +in the wheel. sometimes this isn't the case, and you can ignore those +source files: + + +```ini +[salt-ssh==3005] +likely_binary_ignore = + pkg/smartos/esky/sodium_grabber.c +``` + ### python_versions some packages are only intended for particular python versions (or don't diff --git a/build.py b/build.py index cce575ba..0b456233 100644 --- a/build.py +++ b/build.py @@ -71,6 +71,7 @@ class Package(NamedTuple): apt_requires: tuple[str, ...] brew_requires: tuple[str, ...] custom_prebuild: tuple[str, ...] + likely_binary_ignore: tuple[str, ...] python_versions: SpecifierSet def satisfied_by( @@ -92,6 +93,7 @@ def make(cls, key: str, val: Mapping[str, str]) -> Package: apt_requires = tuple(dct.pop("apt_requires", "").split()) brew_requires = tuple(dct.pop("brew_requires", "").split()) custom_prebuild = tuple(dct.pop("custom_prebuild", "").split()) + likely_binary_ignore = tuple(dct.pop("likely_binary_ignore", "").split()) python_versions = dct.pop("python_versions", "") # ignore validate-only settings for setting in ( @@ -109,6 +111,7 @@ def make(cls, key: str, val: Mapping[str, str]) -> Package: apt_requires=apt_requires, brew_requires=brew_requires, custom_prebuild=custom_prebuild, + likely_binary_ignore=likely_binary_ignore, python_versions=SpecifierSet(python_versions), ) @@ -497,7 +500,7 @@ def _prefix_path(*parts: str) -> str: env.update(before) -def _likely_binary(sdist: str) -> str | None: +def _likely_binary(package: Package, sdist: str) -> str | None: if sdist.endswith(".zip"): with zipfile.ZipFile(sdist) as zipf: names = zipf.namelist() @@ -525,6 +528,9 @@ def _likely_binary(sdist: str) -> str | None: if "/test/" in name or "/tests/" in name: continue + if any(substr in name for substr in package.likely_binary_ignore): + continue + _, ext = os.path.splitext(name) if ext in BINARY_EXTS: ret.add(ext) @@ -584,7 +590,7 @@ def _build(package: Package, python: Python, dest: str, index_url: str) -> str: (filename,) = os.listdir(build_dir) filename_full = os.path.join(build_dir, filename) - likely_binary_reason = _likely_binary(sdist) + likely_binary_reason = _likely_binary(package, sdist) if likely_binary_reason and not _produced_binary(filename_full): raise SystemExit( f"{package.name}=={package.version} expected binary as " From fe9f34af1ee8269c466741021c8290f8d7a48b4d Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Wed, 14 Sep 2022 15:30:53 -0700 Subject: [PATCH 2/2] adapt and add tests --- tests/build_test.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/tests/build_test.py b/tests/build_test.py index 2c882398..9ae3a08b 100644 --- a/tests/build_test.py +++ b/tests/build_test.py @@ -58,6 +58,7 @@ def test_package_default(): apt_requires=(), brew_requires=(), custom_prebuild=(), + likely_binary_ignore=(), python_versions=SpecifierSet(), ) @@ -83,6 +84,7 @@ def test_package_parses_split_values(): apt_requires=("pkg-config", "libxslt1-dev"), brew_requires=("pkg-config", "libxml"), custom_prebuild=("prebuild/crc32c", "deadbeef"), + likely_binary_ignore=(), python_versions=SpecifierSet(), ) @@ -232,27 +234,30 @@ def test_get_archs_linux_aarch64(): def test_likely_binary_zip(tmp_path): + package = Package.make("a==1", {}) filename = tmp_path.joinpath("a-1.zip") with zipfile.ZipFile(filename, "w") as zipf: zipf.open("a-1/src/ext.py", "w").close() zipf.open("a-1/src/_ext.c", "w").close() zipf.open("a-1/src/_ext.pyx", "w").close() - reason = build._likely_binary(str(filename)) + reason = build._likely_binary(package, str(filename)) assert reason == "sdist contains files with these extensions: .c, .pyx" def test_likely_binary_tgz(tmp_path): + package = Package.make("a==1", {}) filename = tmp_path.joinpath("a-1.tar.gz") with tarfile.open(filename, "w:gz") as tarf: tarf.addfile(tarfile.TarInfo("a-1/src/ext.py")) tarf.addfile(tarfile.TarInfo("a-1/src/_ext.c")) - reason = build._likely_binary(str(filename)) + reason = build._likely_binary(package, str(filename)) assert reason == "sdist contains files with these extensions: .c" def test_likely_binary_cffi_zip(tmp_path): + package = Package.make("a==1", {}) filename = tmp_path.joinpath("a-1.zip") with zipfile.ZipFile(filename, "w") as zipf: with zipf.open("a-1/setup.py", "w") as f: @@ -265,11 +270,12 @@ def test_likely_binary_cffi_zip(tmp_path): b" setup()\n" ) - reason = build._likely_binary(str(filename)) + reason = build._likely_binary(package, str(filename)) assert reason == "sdist setup.py has `cffi_modules`" def test_likely_binary_cffi_tar(tmp_path): + package = Package.make("a==1", {}) filename = tmp_path.joinpath("a-1.tar.gz") with tarfile.open(filename, "w:gz") as tarf: # similar to google-crc32c==1.1.2 @@ -284,17 +290,28 @@ def test_likely_binary_cffi_tar(tmp_path): tar_info.size = len(bio.getvalue()) tarf.addfile(tar_info, bio) - reason = build._likely_binary(str(filename)) + reason = build._likely_binary(package, str(filename)) assert reason == "sdist setup.py has `cffi_modules`" def test_likely_binary_ignores_test_files(tmp_path): + package = Package.make("a==1", {}) filename = tmp_path.joinpath("a-1.tar.gz") with tarfile.open(filename, "w:gz") as tarf: tarf.addfile(tarfile.TarInfo("a-1/test/_ext.pyd")) tarf.addfile(tarfile.TarInfo("a-1/tests/_ext.c")) - reason = build._likely_binary(str(filename)) + reason = build._likely_binary(package, str(filename)) + assert reason is None + + +def test_likely_binary_ignore(tmp_path): + package = Package.make("a==1", {"likely_binary_ignore": "\nfoo/bar.c\n"}) + filename = tmp_path.joinpath("a-1.tar.gz") + with tarfile.open(filename, "w:gz") as tarf: + tarf.addfile(tarfile.TarInfo("a-1/foo/bar.c")) + + reason = build._likely_binary(package, str(filename)) assert reason is None