Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,20 @@ the script should set up whatever tools are necessary inside only that directory
custom_prebuild = prebuild/crc32c 1.1.2
```

### likely_binary_ignore

when the sdist has source files of a compiled language (see BINARY_EXTS),
we would expect compiled things (e.g. binary executables, shared objects)
in the wheel. sometimes this isn't the case, and you can ignore those
source files:


```ini
[salt-ssh==3005]
likely_binary_ignore =
pkg/smartos/esky/sodium_grabber.c
```

### python_versions

some packages are only intended for particular python versions (or don't
Expand Down
10 changes: 8 additions & 2 deletions build.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ class Package(NamedTuple):
apt_requires: tuple[str, ...]
brew_requires: tuple[str, ...]
custom_prebuild: tuple[str, ...]
likely_binary_ignore: tuple[str, ...]
python_versions: SpecifierSet

def satisfied_by(
Expand All @@ -92,6 +93,7 @@ def make(cls, key: str, val: Mapping[str, str]) -> Package:
apt_requires = tuple(dct.pop("apt_requires", "").split())
brew_requires = tuple(dct.pop("brew_requires", "").split())
custom_prebuild = tuple(dct.pop("custom_prebuild", "").split())
likely_binary_ignore = tuple(dct.pop("likely_binary_ignore", "").split())
python_versions = dct.pop("python_versions", "")
# ignore validate-only settings
for setting in (
Expand All @@ -109,6 +111,7 @@ def make(cls, key: str, val: Mapping[str, str]) -> Package:
apt_requires=apt_requires,
brew_requires=brew_requires,
custom_prebuild=custom_prebuild,
likely_binary_ignore=likely_binary_ignore,
python_versions=SpecifierSet(python_versions),
)

Expand Down Expand Up @@ -497,7 +500,7 @@ def _prefix_path(*parts: str) -> str:
env.update(before)


def _likely_binary(sdist: str) -> str | None:
def _likely_binary(package: Package, sdist: str) -> str | None:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it'll be much easier to test if this takes likely_binary_ignore directly rather than the whole Package

if sdist.endswith(".zip"):
with zipfile.ZipFile(sdist) as zipf:
names = zipf.namelist()
Expand Down Expand Up @@ -525,6 +528,9 @@ def _likely_binary(sdist: str) -> str | None:
if "/test/" in name or "/tests/" in name:
continue

if any(substr in name for substr in package.likely_binary_ignore):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think these would be better as exact matches?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok!

continue

_, ext = os.path.splitext(name)
if ext in BINARY_EXTS:
ret.add(ext)
Expand Down Expand Up @@ -584,7 +590,7 @@ def _build(package: Package, python: Python, dest: str, index_url: str) -> str:
(filename,) = os.listdir(build_dir)
filename_full = os.path.join(build_dir, filename)

likely_binary_reason = _likely_binary(sdist)
likely_binary_reason = _likely_binary(package, sdist)
if likely_binary_reason and not _produced_binary(filename_full):
raise SystemExit(
f"{package.name}=={package.version} expected binary as "
Expand Down
27 changes: 22 additions & 5 deletions tests/build_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ def test_package_default():
apt_requires=(),
brew_requires=(),
custom_prebuild=(),
likely_binary_ignore=(),
python_versions=SpecifierSet(),
)

Expand All @@ -83,6 +84,7 @@ def test_package_parses_split_values():
apt_requires=("pkg-config", "libxslt1-dev"),
brew_requires=("pkg-config", "libxml"),
custom_prebuild=("prebuild/crc32c", "deadbeef"),
likely_binary_ignore=(),
python_versions=SpecifierSet(),
)

Expand Down Expand Up @@ -232,27 +234,30 @@ def test_get_archs_linux_aarch64():


def test_likely_binary_zip(tmp_path):
package = Package.make("a==1", {})
filename = tmp_path.joinpath("a-1.zip")
with zipfile.ZipFile(filename, "w") as zipf:
zipf.open("a-1/src/ext.py", "w").close()
zipf.open("a-1/src/_ext.c", "w").close()
zipf.open("a-1/src/_ext.pyx", "w").close()

reason = build._likely_binary(str(filename))
reason = build._likely_binary(package, str(filename))
assert reason == "sdist contains files with these extensions: .c, .pyx"


def test_likely_binary_tgz(tmp_path):
package = Package.make("a==1", {})
filename = tmp_path.joinpath("a-1.tar.gz")
with tarfile.open(filename, "w:gz") as tarf:
tarf.addfile(tarfile.TarInfo("a-1/src/ext.py"))
tarf.addfile(tarfile.TarInfo("a-1/src/_ext.c"))

reason = build._likely_binary(str(filename))
reason = build._likely_binary(package, str(filename))
assert reason == "sdist contains files with these extensions: .c"


def test_likely_binary_cffi_zip(tmp_path):
package = Package.make("a==1", {})
filename = tmp_path.joinpath("a-1.zip")
with zipfile.ZipFile(filename, "w") as zipf:
with zipf.open("a-1/setup.py", "w") as f:
Expand All @@ -265,11 +270,12 @@ def test_likely_binary_cffi_zip(tmp_path):
b" setup()\n"
)

reason = build._likely_binary(str(filename))
reason = build._likely_binary(package, str(filename))
assert reason == "sdist setup.py has `cffi_modules`"


def test_likely_binary_cffi_tar(tmp_path):
package = Package.make("a==1", {})
filename = tmp_path.joinpath("a-1.tar.gz")
with tarfile.open(filename, "w:gz") as tarf:
# similar to google-crc32c==1.1.2
Expand All @@ -284,17 +290,28 @@ def test_likely_binary_cffi_tar(tmp_path):
tar_info.size = len(bio.getvalue())
tarf.addfile(tar_info, bio)

reason = build._likely_binary(str(filename))
reason = build._likely_binary(package, str(filename))
assert reason == "sdist setup.py has `cffi_modules`"


def test_likely_binary_ignores_test_files(tmp_path):
package = Package.make("a==1", {})
filename = tmp_path.joinpath("a-1.tar.gz")
with tarfile.open(filename, "w:gz") as tarf:
tarf.addfile(tarfile.TarInfo("a-1/test/_ext.pyd"))
tarf.addfile(tarfile.TarInfo("a-1/tests/_ext.c"))

reason = build._likely_binary(str(filename))
reason = build._likely_binary(package, str(filename))
assert reason is None


def test_likely_binary_ignore(tmp_path):
package = Package.make("a==1", {"likely_binary_ignore": "\nfoo/bar.c\n"})
filename = tmp_path.joinpath("a-1.tar.gz")
with tarfile.open(filename, "w:gz") as tarf:
tarf.addfile(tarfile.TarInfo("a-1/foo/bar.c"))

reason = build._likely_binary(package, str(filename))
assert reason is None


Expand Down