diff --git a/.github/workflows/kernel_abi_python_release.yaml b/.github/workflows/kernel_abi_python_release.yaml deleted file mode 100644 index c7d9aee9..00000000 --- a/.github/workflows/kernel_abi_python_release.yaml +++ /dev/null @@ -1,260 +0,0 @@ -# This file is autogenerated by maturin v1.13.1 -# To update, run -# -# maturin generate-ci github -m kernel-abi-check/bindings/python/Cargo.toml -o .github/workflows/kernel_abi_python_release.yaml -# -name: CI - -on: - push: - branches: - - main - - master - tags: - - "*" - pull_request: - paths-ignore: - - "docs/**" - - "*.md" - workflow_dispatch: - -permissions: - contents: read - -jobs: - linux: - runs-on: ${{ matrix.platform.runner }} - strategy: - matrix: - platform: - - runner: ubuntu-22.04 - target: x86_64 - - runner: ubuntu-22.04 - target: x86 - - runner: ubuntu-22.04 - target: aarch64 - - runner: ubuntu-22.04 - target: armv7 - - runner: ubuntu-22.04 - target: s390x - - runner: ubuntu-22.04 - target: ppc64le - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: 3.8 - - name: Build wheels - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path kernel-abi-check/bindings/python/Cargo.toml - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - manylinux: auto - - name: Build free-threaded wheels (3.13t) - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path kernel-abi-check/bindings/python/Cargo.toml -i python3.13t - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - manylinux: auto - - name: Build free-threaded wheels (3.14t) - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path kernel-abi-check/bindings/python/Cargo.toml -i python3.14t - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - manylinux: auto - - name: Upload wheels - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: wheels-linux-${{ matrix.platform.target }} - path: dist - - musllinux: - runs-on: ${{ matrix.platform.runner }} - strategy: - matrix: - platform: - - runner: ubuntu-22.04 - target: x86_64 - - runner: ubuntu-22.04 - target: x86 - - runner: ubuntu-22.04 - target: aarch64 - - runner: ubuntu-22.04 - target: armv7 - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: 3.8 - - name: Build wheels - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path kernel-abi-check/bindings/python/Cargo.toml - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - manylinux: musllinux_1_2 - - name: Build free-threaded wheels (3.13t) - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path kernel-abi-check/bindings/python/Cargo.toml -i python3.13t - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - manylinux: musllinux_1_2 - - name: Build free-threaded wheels (3.14t) - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path kernel-abi-check/bindings/python/Cargo.toml -i python3.14t - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - manylinux: musllinux_1_2 - - name: Upload wheels - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: wheels-musllinux-${{ matrix.platform.target }} - path: dist - - windows: - runs-on: ${{ matrix.platform.runner }} - strategy: - matrix: - platform: - - runner: windows-latest - target: x64 - python_arch: x64 - - runner: windows-latest - target: x86 - python_arch: x86 - - runner: windows-11-arm - target: aarch64 - python_arch: arm64 - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: 3.13 - architecture: ${{ matrix.platform.python_arch }} - - name: Build wheels - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path kernel-abi-check/bindings/python/Cargo.toml - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - if: matrix.platform.target != 'aarch64' - with: - python-version: 3.13t - architecture: ${{ matrix.platform.python_arch }} - - name: Build free-threaded wheels (3.13t) - if: matrix.platform.target != 'aarch64' - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path kernel-abi-check/bindings/python/Cargo.toml -i python3.13t - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - if: matrix.platform.target != 'aarch64' - with: - python-version: 3.14t - architecture: ${{ matrix.platform.python_arch }} - - name: Build free-threaded wheels (3.14t) - if: matrix.platform.target != 'aarch64' - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path kernel-abi-check/bindings/python/Cargo.toml -i python3.14t - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - - name: Upload wheels - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: wheels-windows-${{ matrix.platform.target }} - path: dist - - macos: - runs-on: ${{ matrix.platform.runner }} - strategy: - matrix: - platform: - - runner: macos-15-intel - target: x86_64 - - runner: macos-latest - target: aarch64 - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: 3.8 - - name: Build wheels - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path kernel-abi-check/bindings/python/Cargo.toml - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: 3.13t - - name: Build free-threaded wheels (3.13t) - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path kernel-abi-check/bindings/python/Cargo.toml -i python3.13t - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: 3.14t - - name: Build free-threaded wheels (3.14t) - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --manifest-path kernel-abi-check/bindings/python/Cargo.toml -i python3.14t - sccache: ${{ !startsWith(github.ref, 'refs/tags/') }} - - name: Upload wheels - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: wheels-macos-${{ matrix.platform.target }} - path: dist - - sdist: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Build sdist - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 - with: - command: sdist - args: --out dist --manifest-path kernel-abi-check/bindings/python/Cargo.toml - - name: Upload sdist - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: wheels-sdist - path: dist - - release: - name: Release - runs-on: ubuntu-latest - if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} - needs: [linux, musllinux, windows, macos, sdist] - environment: - name: pypi - url: https://pypi.org/p/kernel-abi-check - permissions: - # Use to sign the release artifacts - id-token: write - # Used to upload release artifacts - contents: write - # Used to generate artifact attestation - attestations: write - steps: - - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 - - name: Generate artifact attestation - uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0 - with: - subject-path: "wheels-*/*" - - name: Publish to PyPI - if: ${{ startsWith(github.ref, 'refs/tags/') }} - uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1.50.1 - with: - command: upload - args: --non-interactive --skip-existing wheels-*/* diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index abf7ec46..f737c078 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -18,8 +18,7 @@ jobs: components: rustfmt - name: Cargo fmt (kernel-abi-check) run: | - ( cd kernel-abi-check/kernel-abi-check && cargo fmt --all -- --check ) - ( cd kernel-abi-check/bindings/python && cargo fmt --all -- --check ) + ( cd kernel-abi-check && cargo fmt --all -- --check ) - name: Cargo fmt (kernels-data) run: | ( cd kernels-data && cargo fmt --all -- --check ) @@ -46,8 +45,7 @@ jobs: components: clippy - name: Clippy (kernel-abi-check) run: | - ( cd kernel-abi-check/kernel-abi-check && cargo clippy -- -D warnings ) - ( cd kernel-abi-check/bindings/python && cargo clippy -- -D warnings ) + ( cd kernel-abi-check && cargo clippy -- -D warnings ) - name: Clippy (kernels-data) run: | ( cd kernels-data && cargo clippy -- -D warnings ) diff --git a/.github/workflows/test_kernels.yaml b/.github/workflows/test_kernels.yaml index cf509a6e..8cb8fcb5 100644 --- a/.github/workflows/test_kernels.yaml +++ b/.github/workflows/test_kernels.yaml @@ -84,12 +84,6 @@ jobs: uv pip install einops nvidia-cutlass-dsl uv run pytest tests/test_deps.py - - name: Check kernel check - working-directory: ./kernels - run: | - uv pip install ../kernel-abi-check/bindings/python - uv run kernels check kernels-community/activation - - name: Import check without torch working-directory: ./kernels run: | @@ -104,7 +98,6 @@ jobs: - name: Run command-line tools without Torch working-directory: ./kernels run: | - uv run kernels check kernels-community/activation uv run kernels versions kernels-community/activation # This is done to securely run the coverage test and to post comments even diff --git a/.github/workflows/test_python.yaml b/.github/workflows/test_python.yaml index de57b465..dcf90e59 100644 --- a/.github/workflows/test_python.yaml +++ b/.github/workflows/test_python.yaml @@ -16,31 +16,6 @@ concurrency: cancel-in-progress: true jobs: - kernel-abi-check: - name: kernel-abi-check bindings - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.12"] - - env: - UV_PYTHON_PREFERENCE: only-managed - - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Install uv and set the python version - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 - with: - python-version: ${{ matrix.python-version }} - - - name: Install the project - run: ( cd kernel-abi-check/bindings/python && uv sync --all-extras --dev) - - - name: Run tests - run: ( cd kernel-abi-check/bindings/python && uv run pytest tests ) - kernels-data: name: kernels-data bindings runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index a9dc7cd0..224c64dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,15 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "addr2line" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" -dependencies = [ - "gimli", -] - [[package]] name = "adler2" version = "2.0.1" @@ -169,21 +160,6 @@ dependencies = [ "fs_extra", ] -[[package]] -name = "backtrace" -version = "0.3.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" -dependencies = [ - "addr2line", - "cfg-if 1.0.4", - "libc", - "miniz_oxide", - "object 0.37.3", - "rustc-demangle", - "windows-link", -] - [[package]] name = "base32" version = "0.5.1" @@ -409,33 +385,6 @@ dependencies = [ "cc", ] -[[package]] -name = "color-eyre" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5920befb47832a6d61ee3a3a846565cfa39b331331e68a3b1d1116630f2f26d" -dependencies = [ - "backtrace", - "color-spantrace", - "eyre", - "indenter", - "once_cell", - "owo-colors", - "tracing-error", -] - -[[package]] -name = "color-spantrace" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8b88ea9df13354b55bc7234ebcce36e6ef896aca2e42a15de9e10edce01b427" -dependencies = [ - "once_cell", - "owo-colors", - "tracing-core", - "tracing-error", -] - [[package]] name = "colorchoice" version = "1.0.5" @@ -1042,12 +991,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "gimli" -version = "0.32.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" - [[package]] name = "git-version" version = "0.3.9" @@ -1191,9 +1134,11 @@ dependencies = [ "hf-hub", "indicatif", "itertools 0.13.0", + "kernel-abi-check", "kernels-data", "minijinja", "minijinja-embed", + "object", "rand 0.8.6", "regex", "reqwest", @@ -1666,27 +1611,16 @@ dependencies = [ name = "kernel-abi-check" version = "0.15.0-dev0" dependencies = [ - "clap", - "color-eyre", "cpp_demangle", "eyre", "itertools 0.14.0", - "object 0.36.7", + "object", "once_cell", "serde", "serde_json", "toml", ] -[[package]] -name = "kernel-abi-check-python" -version = "0.15.0-dev0" -dependencies = [ - "kernel-abi-check", - "object 0.36.7", - "pyo3", -] - [[package]] name = "kernels-data" version = "0.15.0-dev0" @@ -2072,15 +2006,6 @@ dependencies = [ "ruzstd", ] -[[package]] -name = "object" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" -dependencies = [ - "memchr", -] - [[package]] name = "once_cell" version = "1.21.4" @@ -2147,12 +2072,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "owo-colors" -version = "4.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" - [[package]] name = "paste" version = "1.0.15" @@ -2627,12 +2546,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rustc-demangle" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" - [[package]] name = "rustc-hash" version = "1.1.0" @@ -3496,16 +3409,6 @@ dependencies = [ "valuable", ] -[[package]] -name = "tracing-error" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db" -dependencies = [ - "tracing", - "tracing-subscriber", -] - [[package]] name = "tracing-log" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 0979c04a..3a1f7926 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,6 @@ [workspace] members = [ - "kernel-abi-check/kernel-abi-check", - "kernel-abi-check/bindings/python", + "kernel-abi-check", "kernel-builder", "kernels-data", "kernels-data/bindings/python", diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index 312745a5..bd27cc40 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -57,8 +57,6 @@ - sections: - local: cli-benchmark title: kernels benchmark - - local: cli-check - title: kernels check - local: cli-versions title: kernels versions - local: cli-lock diff --git a/docs/source/builder-cli.md b/docs/source/builder-cli.md index 893926bb..30f1b6ed 100644 --- a/docs/source/builder-cli.md +++ b/docs/source/builder-cli.md @@ -12,6 +12,7 @@ This document contains the help content for the `kernel-builder` command-line pr * [`kernel-builder build-and-upload`↴](#kernel-builder-build-and-upload) * [`kernel-builder upload`↴](#kernel-builder-upload) * [`kernel-builder check-config`↴](#kernel-builder-check-config) +* [`kernel-builder check-abi`↴](#kernel-builder-check-abi) * [`kernel-builder check-builds`↴](#kernel-builder-check-builds) * [`kernel-builder create-pyproject`↴](#kernel-builder-create-pyproject) * [`kernel-builder devshell`↴](#kernel-builder-devshell) @@ -37,6 +38,7 @@ Build Hugging Face Hub kernels * `build-and-upload` — Build the kernel and upload to Hugging Face Hub * `upload` — Upload kernel build artifacts to the Hugging Face Hub * `check-config` — Validate the build.toml file +* `check-abi` — Check the ABI compatibility of a kernel extension * `check-builds` — Validate kernel builds * `create-pyproject` — Generate CMake files for a kernel extension build * `devshell` — Spawn a kernel development shell @@ -188,6 +190,31 @@ Validate the build.toml file +## `kernel-builder check-abi` + +Check the ABI compatibility of a kernel extension + +**Usage:** `kernel-builder check-abi [OPTIONS] [KERNEL_DIR]` + +###### **Arguments:** + +* `` — Directory with kernels + +###### **Options:** + +* `-m`, `--manylinux ` — Manylinux version + + Default value: `manylinux_2_28` +* `--macos ` — macOS version + + Default value: `15.0` +* `-p`, `--python-abi ` — Python ABI version + + Default value: `3.9` +* `--torch-stable-abi ` — Torch stable ABI version + + + ## `kernel-builder check-builds` Validate kernel builds diff --git a/docs/source/cli-check.md b/docs/source/cli-check.md deleted file mode 100644 index 226b802d..00000000 --- a/docs/source/cli-check.md +++ /dev/null @@ -1,65 +0,0 @@ -# kernels check - -Use `kernels check` to verify that a kernel on the Hub meets compliance requirements. - -## What It Checks - -- Python ABI compatibility (default: 3.9) -- Operating system compatibility (macOS 15.0+, manylinux_2_28) - -## Usage - -```bash -kernels check [--revision ] [--macos ] [--manylinux ] [--python-abi ] -``` - -## Installation - -`kernels check` requires an additional dependency: - -```bash -uv pip install kernel-abi-check # or pip install kernel-abi-check -``` - -## Examples - -Check a kernel on the Hub: - -```bash -kernels check kernels-community/flash-attn3 -``` - -Check a specific revision: - -```bash -kernels check kernels-community/flash-attn3 --revision v2 -``` - -Check with custom compatibility requirements: - -```bash -kernels check kernels-community/flash-attn3 --python-abi 3.10 --manylinux manylinux_2_31 -``` - -## Example Output - -```text -Checking variant: torch210-metal-aarch64-darwin - Dynamic library _example_kernel_metal_2juixjwdznbhy.abi3.so: - 🐍 Python ABI 3.9 compatible - 🍏 compatible with macOS 15.0 -Checking variant: torch29-metal-aarch64-darwin - Dynamic library _example_kernel_metal_vtlnpevkb6uum.abi3.so: - 🐍 Python ABI 3.9 compatible - 🍏 compatible with macOS 15.0 -``` - -## Options - -| Option | Default | Description | -| -------------- | ---------------- | ----------------------------------- | -| `--revision` | `main` | Branch, tag, or commit SHA to check | -| `--macos` | `15.0` | Minimum macOS version to require | -| `--manylinux` | `manylinux_2_28` | Manylinux version to require | -| `--python-abi` | `3.9` | Python ABI version to require | - diff --git a/docs/source/kernel-requirements.md b/docs/source/kernel-requirements.md index 64f9c621..96b2a5bd 100644 --- a/docs/source/kernel-requirements.md +++ b/docs/source/kernel-requirements.md @@ -230,14 +230,19 @@ The ABI3 requirement can be checked with the ABI checker (see below). ### ABI checker The manylinux_2_28 and Python ABI 3.9 version requirements can be checked with -[`kernel-abi-check`](https://crates.io/crates/kernel-abi-check): +`kernel-builder check-abi`: ```bash - -$ cargo install kernel-abi-check -$ kernel-abi-check result/relu/_relu_e87e0ca_dirty.abi3.so -🐍 Checking for compatibility with manylinux_2_28 and Python ABI version 3.9 +$ kernel-builder check-abi examples/kernels/relu +🐍 Checking for compatibility with manylinux_2_28 and Python ABI version 3.9: /home/daniel/git/kernels/examples/kernels/relu/result/torch211-cpu-x86_64-linux/_relu_cpu_30dc0ae_dirty.abi3.so +✅ No compatibility issues found +🐍 Checking for compatibility with manylinux_2_28 and Python ABI version 3.9: /home/daniel/git/kernels/examples/kernels/relu/result/torch211-cu126-x86_64-linux/_relu_cuda_30dc0ae_dirty.abi3.so +✅ No compatibility issues found +🐍 Checking for compatibility with manylinux_2_28 and Python ABI version 3.9: /home/daniel/git/kernels/examples/kernels/relu/result/torch211-cu128-x86_64-linux/_relu_cuda_30dc0ae_dirty.abi3.so +✅ No compatibility issues found +🐍 Checking for compatibility with manylinux_2_28 and Python ABI version 3.9: /home/daniel/git/kernels/examples/kernels/relu/result/torch211-cu130-x86_64-linux/_relu_cuda_30dc0ae_dirty.abi3.so ✅ No compatibility issues found +[...] ``` ## Torch extension diff --git a/flake.nix b/flake.nix index de60f41b..4b30a061 100644 --- a/flake.nix +++ b/flake.nix @@ -146,7 +146,6 @@ mkShell { nativeBuildInputs = [ kernel-builder - kernel-abi-check nodejs # For hf-doc-builder. pinact pkg-config @@ -168,7 +167,6 @@ huggingface-hub jax jax-tvm-ffi - kernel-abi-check kernels-data matplotlib mktestdocs @@ -213,7 +211,7 @@ formatter = pkgs.nixfmt-tree; packages = rec { - inherit (buildSet.pkgs) kernel-builder kernel-abi-check; + inherit (buildSet.pkgs) kernel-builder; inherit (buildSet.pkgs.python3.pkgs) kernels; update-build = pkgs.writeShellScriptBin "update-build" '' diff --git a/kernel-abi-check/kernel-abi-check/Cargo.toml b/kernel-abi-check/Cargo.toml similarity index 88% rename from kernel-abi-check/kernel-abi-check/Cargo.toml rename to kernel-abi-check/Cargo.toml index 00bfb315..a04f2864 100644 --- a/kernel-abi-check/kernel-abi-check/Cargo.toml +++ b/kernel-abi-check/Cargo.toml @@ -10,8 +10,6 @@ repository = "https://github.com/huggingface/kernel-builder" [dependencies] -clap = { version = "4", features = ["derive"] } -color-eyre = "0.6" cpp_demangle = "0.5" eyre = "0.6" itertools = "0.14.0" @@ -20,4 +18,3 @@ once_cell = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" toml = "0.8" - diff --git a/kernel-abi-check/bindings/python/Cargo.toml b/kernel-abi-check/bindings/python/Cargo.toml deleted file mode 100644 index a7beddd9..00000000 --- a/kernel-abi-check/bindings/python/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "kernel-abi-check-python" -version = "0.15.0-dev0" -edition = "2024" -description = "Check the ABI of Hub Kernels" -homepage = "https://github.com/huggingface/kernel-builder" -license = "Apache-2.0" -repository = "https://github.com/huggingface/kernel-builder" - -[lib] -name = "kernel_abi_check" -crate-type = ["cdylib"] - -[dependencies] -object = "0.36.7" -pyo3 = { version = "0.26", features = ["abi3", "abi3-py38"] } - -[dependencies.kernel-abi-check] -path = "../../kernel-abi-check" -version = "0.15.0-dev0" diff --git a/kernel-abi-check/bindings/python/MANIFEST.in b/kernel-abi-check/bindings/python/MANIFEST.in deleted file mode 100644 index 00afeadb..00000000 --- a/kernel-abi-check/bindings/python/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -include Cargo.toml -include pyproject.toml -include rust-toolchain -include ../../LICENSE -recursive-include src * diff --git a/kernel-abi-check/bindings/python/kernel_abi_check.pyi b/kernel-abi-check/bindings/python/kernel_abi_check.pyi deleted file mode 100644 index 36449b7f..00000000 --- a/kernel-abi-check/bindings/python/kernel_abi_check.pyi +++ /dev/null @@ -1,215 +0,0 @@ -"""Type stubs for kernel_abi_check module.""" - -from enum import Enum -from typing import List, Union -import os - -__version__: str - -class BinaryFormat(Enum): - """Binary format of an object file.""" - - COFF = "COFF" - """COFF (Common Object File Format)""" - - ELF = "ELF" - """ELF (Executable and Linkable Format)""" - - MACH_O = "MACH_O" - """Mach-O (Mach Object file format)""" - - PE = "PE" - """PE (Portable Executable)""" - - WASM = "WASM" - """WebAssembly""" - - XCOFF = "XCOFF" - """XCOFF (Extended Common Object File Format)""" - - def __str__(self) -> str: ... - def __repr__(self) -> str: ... - -class ObjectFile: - """Object file that can be validated for ABI compatibility.""" - - def __init__(self, filename: os.PathLike[str] | str) -> None: - """Create a new ObjectFile from a path. - - Args: - filename: Path to the object file to analyze - - Raises: - IOError: If the file cannot be opened or read - """ - ... - - def format(self) -> BinaryFormat: - """Get the binary format of this object file. - - Returns: - The binary format of the object file - - Raises: - ValueError: If the object file cannot be parsed or has an unsupported format - """ - ... - - def check_python_abi( - self, abi_version: str - ) -> List[Union[IncompatibleAbi3Symbol, NonAbi3Symbol]]: - """Check Python stable ABI compatibility for this object file. - - Args: - abi_version: Python ABI version string (e.g., "3.8") - - Returns: - List of ABI violations found - - Raises: - ValueError: If the ABI version cannot be parsed or ABI check fails - """ - ... - - def check_manylinux( - self, manylinux_version: str - ) -> List[IncompatibleManylinuxSymbol]: - """Check manylinux compatibility for this object file. - - Args: - manylinux_version: Manylinux version string (e.g., "manylinux_2_17") - - Returns: - List of manylinux violations found - - Raises: - ValueError: If the manylinux check fails - """ - ... - - def check_macos( - self, macos_version: str - ) -> List[Union[MissingMacOSVersion, IncompatibleMacOSVersion]]: - """Check macOS compatibility for this object file. - - Args: - macos_version: macOS version string (e.g., "10.15") - - Returns: - List of macOS violations found - - Raises: - ValueError: If the macOS version cannot be parsed or check fails - """ - ... - -class IncompatibleAbi3Symbol: - """ABI3 symbol that is not compatible with the specified Python ABI version.""" - - def __init__(self, *, name: str, added: str) -> None: - """Create a new IncompatibleAbi3Symbol. - - Args: - name: Name of the symbol - added: Version when this symbol was added to Python - - Raises: - ValueError: If the version string cannot be parsed - TypeError: If positional arguments are used - """ - ... - - @property - def name(self) -> str: - """Name of the symbol.""" - ... - - @property - def version_added(self) -> str: - """Version when this symbol was added to Python.""" - ... - - def __eq__(self, other: object) -> bool: ... - def __ne__(self, other: object) -> bool: ... - def __repr__(self) -> str: ... - -class NonAbi3Symbol: - """Python symbol that is not part of ABI3.""" - - @property - def name(self) -> str: - """Name of the symbol.""" - ... - - def __repr__(self) -> str: ... - -class IncompatibleManylinuxSymbol: - """Symbol that is not allowed by the manylinux version.""" - - def __init__(self, *, name: str, dep: str, version: str) -> None: - """Create a new IncompatibleManylinuxSymbol. - - Args: - name: Name of the symbol - dep: Dependency that contains the symbol - version: Version of the symbol - - Raises: - TypeError: If positional arguments are used - """ - ... - - @property - def name(self) -> str: - """Name of the symbol.""" - ... - - @property - def dep(self) -> str: - """Dependency that contains the symbol.""" - ... - - @property - def version(self) -> str: - """Version of the symbol.""" - ... - - def __eq__(self, other: object) -> bool: ... - def __ne__(self, other: object) -> bool: ... - def __repr__(self) -> str: ... - -class MissingMacOSVersion: - """Object file does not specify minimum OS version.""" - - def __init__(self) -> None: - """Create a new MissingMacOSVersion.""" - ... - - def __eq__(self, other: object) -> bool: ... - def __ne__(self, other: object) -> bool: ... - def __repr__(self) -> str: ... - -class IncompatibleMacOSVersion: - """The minimum OS version of the object file is higher than the - specified macOS version.""" - - def __init__(self, *, version: str) -> None: - """Create a new IncompatibleMacOSVersion. - - Args: - version: Minimum OS version of the object file - - Raises: - ValueError: If the version string cannot be parsed - TypeError: If positional arguments are used - """ - ... - - @property - def version(self) -> str: - """Minimum OS version of the object file.""" - ... - - def __eq__(self, other: object) -> bool: ... - def __ne__(self, other: object) -> bool: ... - def __repr__(self) -> str: ... diff --git a/kernel-abi-check/bindings/python/pyproject.toml b/kernel-abi-check/bindings/python/pyproject.toml deleted file mode 100644 index 3eaef47c..00000000 --- a/kernel-abi-check/bindings/python/pyproject.toml +++ /dev/null @@ -1,26 +0,0 @@ -[project] -name = "kernel-abi-check" -requires-python = '>=3.8' -classifiers = [ - "Programming Language :: Rust", - "Programming Language :: Python :: Implementation :: CPython", -] -dynamic = [ - 'description', - 'readme', - 'version', -] - -[build-system] -requires = ["maturin>=1,<2"] -build-backend = "maturin" - -[dependency-groups] -dev = [ - "pytest>=8", -] - -[tool.maturin] -module-name = "kernel_abi_check" -bindings = 'pyo3' -features = ["pyo3/extension-module"] diff --git a/kernel-abi-check/bindings/python/src/lib.rs b/kernel-abi-check/bindings/python/src/lib.rs deleted file mode 100644 index 94bbf603..00000000 --- a/kernel-abi-check/bindings/python/src/lib.rs +++ /dev/null @@ -1,435 +0,0 @@ -use std::fs; -use std::path::PathBuf; -use std::str::FromStr; - -use kernel_abi_check::{ - MacOSViolation, ManylinuxViolation, PythonAbiViolation, Version, check_macos, check_manylinux, -}; -use object::{BinaryFormat, Object as ObjectTrait}; -use pyo3::Bound as PyBound; -use pyo3::exceptions::PyIOError; -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; -use pyo3::types::PyTuple; - -/// Object file that can be validated. -#[pyclass(name = "ObjectFile")] -struct PyObjectFile { - filename: PathBuf, - data: Vec, -} - -impl PyObjectFile { - fn parse_file(&self) -> PyResult> { - object::File::parse(&*self.data).map_err(|err| { - PyValueError::new_err(format!( - "Cannot parse object file `{}`: {}", - self.filename.to_string_lossy(), - err - )) - }) - } -} - -#[pymethods] -impl PyObjectFile { - /// Create a new `ObjectFile` from a path. - #[new] - fn new(filename: PathBuf) -> PyResult { - let data = fs::read(&filename).map_err(|err| { - PyIOError::new_err(format!( - "Cannot open object file `{}`: {}", - filename.to_string_lossy(), - err - )) - })?; - - Ok(Self { filename, data }) - } - - /// Check Python ABI compatibility for this object file - fn check_python_abi(&self, abi_version: String, py: Python) -> PyResult>> { - let file = self.parse_file()?; - - let python_abi = Version::from_str(&abi_version).map_err(|err| { - PyValueError::new_err(format!( - "Cannot parse Python ABI version `{abi_version}`: {err}", - )) - })?; - - let violations = - kernel_abi_check::check_python_abi(&python_abi, file.format(), file.symbols()) - .map_err(|err| { - PyValueError::new_err(format!( - "Cannot check Python ABI for `{}`: {}", - self.filename.to_string_lossy(), - err - )) - })?; - - let mut result = Vec::new(); - for violation in violations { - let py_violation: Py = match violation { - PythonAbiViolation::IncompatibleAbi3Symbol { name, added } => { - Py::new(py, PyIncompatibleAbi3Symbol { name, added })?.into() - } - PythonAbiViolation::NonAbi3Symbol { name } => { - Py::new(py, PyNonAbi3Symbol { name })?.into() - } - }; - result.push(py_violation); - } - Ok(result) - } - - /// Check manylinux compatibility for this object file - fn check_manylinux(&self, manylinux_version: String, py: Python) -> PyResult>> { - let file = self.parse_file()?; - - let violations = check_manylinux( - &manylinux_version, - file.architecture(), - file.endianness(), - file.symbols(), - ) - .map_err(|err| { - PyValueError::new_err(format!( - "Cannot check manylinux for `{}`: {}", - self.filename.to_string_lossy(), - err - )) - })?; - - let mut result = Vec::new(); - for violation in violations { - let py_violation: Py = match violation { - ManylinuxViolation::Symbol { name, dep, version } => { - Py::new(py, PyIncompatibleManylinuxSymbol { name, dep, version })?.into() - } - }; - result.push(py_violation); - } - Ok(result) - } - - /// Check macOS compatibility for this object file - fn check_macos(&self, macos_version: String, py: Python) -> PyResult>> { - let file = self.parse_file()?; - - let macos_ver = Version::from_str(&macos_version).map_err(|err| { - PyValueError::new_err(format!( - "Cannot parse macOS version `{macos_version}`: {err}", - )) - })?; - - let violations = check_macos(&file, &macos_ver).map_err(|err| { - PyValueError::new_err(format!( - "Cannot check macOS for `{}`: {}", - self.filename.to_string_lossy(), - err - )) - })?; - - let mut result = Vec::new(); - for violation in violations { - let py_violation: Py = match violation { - MacOSViolation::MissingMinOS => Py::new(py, PyMissingMacOSVersion)?.into(), - MacOSViolation::IncompatibleMinOS { version } => { - Py::new(py, PyIncompatibleMacOSVersion { version })?.into() - } - }; - result.push(py_violation); - } - Ok(result) - } - - /// Get the binary format of this object file - fn format(&self) -> PyResult { - let file = self.parse_file()?; - let binary_format = file.format(); - - let py_format = match binary_format { - BinaryFormat::Coff => PyBinaryFormat::Coff, - BinaryFormat::Elf => PyBinaryFormat::Elf, - BinaryFormat::MachO => PyBinaryFormat::MachO, - BinaryFormat::Pe => PyBinaryFormat::Pe, - BinaryFormat::Wasm => PyBinaryFormat::Wasm, - BinaryFormat::Xcoff => PyBinaryFormat::Xcoff, - _ => { - return Err(PyValueError::new_err(format!( - "Unsupported binary format: {binary_format:?}" - ))); - } - }; - - Ok(py_format) - } -} - -/// Binary format of an object file -#[derive(Clone, Debug, Eq, PartialEq)] -#[pyclass(name = "BinaryFormat")] -pub enum PyBinaryFormat { - /// COFF (Common Object File Format) - #[pyo3(name = "COFF")] - Coff, - /// ELF (Executable and Linkable Format) - #[pyo3(name = "ELF")] - Elf, - /// Mach-O (Mach Object file format) - #[pyo3(name = "MACH_O")] - MachO, - /// PE (Portable Executable) - #[pyo3(name = "PE")] - Pe, - /// WebAssembly - #[pyo3(name = "WASM")] - Wasm, - /// XCOFF (Extended Common Object File Format) - #[pyo3(name = "XCOFF")] - Xcoff, -} - -#[pymethods] -impl PyBinaryFormat { - fn __eq__(&self, other: &Self) -> bool { - self == other - } - - fn __ne__(&self, other: &Self) -> bool { - self != other - } - - fn __repr__(&self) -> String { - match self { - PyBinaryFormat::Coff => "BinaryFormat.COFF".to_string(), - PyBinaryFormat::Elf => "BinaryFormat.ELF".to_string(), - PyBinaryFormat::MachO => "BinaryFormat.MACH_O".to_string(), - PyBinaryFormat::Pe => "BinaryFormat.PE".to_string(), - PyBinaryFormat::Wasm => "BinaryFormat.WASM".to_string(), - PyBinaryFormat::Xcoff => "BinaryFormat.XCOFF".to_string(), - } - } - - fn __str__(&self) -> String { - self.__repr__() - } -} - -/// Incompatible ABI3 symbol violation -#[derive(Clone, Debug, Eq, PartialEq)] -#[pyclass(name = "IncompatibleAbi3Symbol")] -struct PyIncompatibleAbi3Symbol { - name: String, - added: Version, -} - -#[pymethods] -impl PyIncompatibleAbi3Symbol { - #[new] - #[pyo3(signature = (*py_args, name, added))] - fn new(py_args: &Bound<'_, PyTuple>, name: String, added: String) -> PyResult { - if !py_args.is_empty() { - return Err(PyErr::new::( - "All arguments must be provided as keyword arguments", - )); - } - let added = Version::from_str(&added).map_err(|err| { - PyValueError::new_err(format!( - "Cannot parse the version the symbol was added `{added}`: {err}", - )) - })?; - Ok(Self { name, added }) - } - - #[getter] - fn name(&self) -> &str { - &self.name - } - - #[getter] - fn version_added(&self) -> String { - self.added.to_string() - } - - fn __eq__(&self, other: &Self) -> bool { - self == other - } - - fn __ne__(&self, other: &Self) -> bool { - self != other - } - - fn __repr__(&self) -> String { - format!( - "IncompatibleAbi3Symbol(name='{}', version_added='{}')", - self.name, self.added - ) - } -} - -/// Non-ABI3 symbol violation -#[pyclass(name = "NonAbi3Symbol")] -struct PyNonAbi3Symbol { - name: String, -} - -#[pymethods] -impl PyNonAbi3Symbol { - #[getter] - fn name(&self) -> &str { - &self.name - } - - fn __repr__(&self) -> String { - format!("NonAbi3Symbol(name='{}')", self.name) - } -} - -/// Manylinux symbol violation -#[derive(Clone, Debug, Eq, PartialEq)] -#[pyclass(name = "IncompatibleManylinuxSymbol")] -struct PyIncompatibleManylinuxSymbol { - name: String, - dep: String, - version: String, -} - -#[pymethods] -impl PyIncompatibleManylinuxSymbol { - #[new] - #[pyo3(signature = (*py_args, name, dep, version))] - fn new( - py_args: &Bound<'_, PyTuple>, - name: String, - dep: String, - version: String, - ) -> PyResult { - if !py_args.is_empty() { - return Err(PyErr::new::( - "All arguments must be provided as keyword arguments", - )); - } - Ok(Self { name, dep, version }) - } - - #[getter] - fn name(&self) -> &str { - &self.name - } - - #[getter] - fn dep(&self) -> &str { - &self.dep - } - - #[getter] - fn version(&self) -> &str { - &self.version - } - - fn __eq__(&self, other: &Self) -> bool { - self == other - } - - fn __ne__(&self, other: &Self) -> bool { - self != other - } - - fn __repr__(&self) -> String { - format!( - "IncompatibleManylinuxSymbol(name='{}', dep='{}', version='{}')", - self.name, self.dep, self.version - ) - } -} - -/// Missing minimum OS version violation -#[derive(Clone, Debug, Eq, PartialEq)] -#[pyclass(name = "MissingMacOSVersion")] -struct PyMissingMacOSVersion; - -#[pymethods] -impl PyMissingMacOSVersion { - #[new] - pub fn new() -> Self { - Self - } - - fn __eq__(&self, other: &Self) -> bool { - self == other - } - - fn __ne__(&self, other: &Self) -> bool { - self != other - } - - fn __repr__(&self) -> String { - "MissingMacOSVersion()".to_string() - } -} - -/// Incompatible minimum OS version violation -#[derive(Clone, Debug, Eq, PartialEq)] -#[pyclass(name = "IncompatibleMacOSVersion")] -struct PyIncompatibleMacOSVersion { - version: Version, -} - -#[pymethods] -impl PyIncompatibleMacOSVersion { - #[new] - #[pyo3(signature = (*py_args, version))] - fn new(py_args: &Bound<'_, PyTuple>, version: String) -> PyResult { - if !py_args.is_empty() { - return Err(PyErr::new::( - "All arguments must be provided as keyword arguments", - )); - } - let version = Version::from_str(&version).map_err(|err| { - PyValueError::new_err(format!( - "Cannot parse the version the symbol was added `{version}`: {err}", - )) - })?; - Ok(Self { version }) - } - - #[getter] - fn version(&self) -> String { - self.version.to_string() - } - - fn __eq__(&self, other: &Self) -> bool { - self == other - } - - fn __ne__(&self, other: &Self) -> bool { - self != other - } - - fn __repr__(&self) -> String { - format!("IncompatibleMacOSVersion(version='{}')", self.version) - } -} - -#[pyo3::pymodule(name = "kernel_abi_check")] -fn kernel_abi_check_py(m: &PyBound<'_, PyModule>) -> PyResult<()> { - m.add_class::()?; - - // Binary format enum - m.add_class::()?; - - // Python ABI violation classes - m.add_class::()?; - m.add_class::()?; - - // Manylinux violation classes - m.add_class::()?; - - // macOS violation classes - m.add_class::()?; - m.add_class::()?; - - m.add("__version__", env!("CARGO_PKG_VERSION"))?; - Ok(()) -} diff --git a/kernel-abi-check/bindings/python/tests/hello-darwin-x86_64.abi3.so b/kernel-abi-check/bindings/python/tests/hello-darwin-x86_64.abi3.so deleted file mode 100755 index 55fff2c4..00000000 Binary files a/kernel-abi-check/bindings/python/tests/hello-darwin-x86_64.abi3.so and /dev/null differ diff --git a/kernel-abi-check/bindings/python/tests/hello-linux-x86_64.abi3.so b/kernel-abi-check/bindings/python/tests/hello-linux-x86_64.abi3.so deleted file mode 100755 index 732acb78..00000000 Binary files a/kernel-abi-check/bindings/python/tests/hello-linux-x86_64.abi3.so and /dev/null differ diff --git a/kernel-abi-check/bindings/python/tests/test_kernel_abi_check.py b/kernel-abi-check/bindings/python/tests/test_kernel_abi_check.py deleted file mode 100644 index 77d8699a..00000000 --- a/kernel-abi-check/bindings/python/tests/test_kernel_abi_check.py +++ /dev/null @@ -1,54 +0,0 @@ -from pathlib import Path - -import pytest - -from kernel_abi_check import ( - BinaryFormat, - IncompatibleAbi3Symbol, - IncompatibleMacOSVersion, - ObjectFile, - IncompatibleManylinuxSymbol, -) - - -@pytest.fixture -def test_dir(): - return Path(__file__).parent - - -def test_macos_shared_lib(test_dir): - o = ObjectFile(test_dir / "hello-darwin-x86_64.abi3.so") - assert o.check_python_abi("3.8") == [] - assert o.check_python_abi("3.5") == [ - IncompatibleAbi3Symbol(name="PyModule_GetNameObject", added="3.7") - ] - - assert o.check_macos("15.0") == [] - assert o.check_macos("10.0") == [IncompatibleMacOSVersion(version="11.3")] - - assert o.format() == BinaryFormat.MACH_O - - -def test_linux_shared_lib(test_dir): - o = ObjectFile(test_dir / "hello-linux-x86_64.abi3.so") - assert o.check_python_abi("3.8") == [] - assert o.check_python_abi("3.5") == [ - IncompatibleAbi3Symbol(name="PyModule_GetNameObject", added="3.7") - ] - - assert o.check_manylinux("manylinux_2_34") == [] - assert o.check_manylinux("manylinux_2_28") == [ - IncompatibleManylinuxSymbol(name="fstat64", dep="GLIBC", version="2.33"), - IncompatibleManylinuxSymbol( - name="pthread_key_create", dep="GLIBC", version="2.34" - ), - IncompatibleManylinuxSymbol( - name="pthread_key_delete", dep="GLIBC", version="2.34" - ), - IncompatibleManylinuxSymbol( - name="pthread_setspecific", dep="GLIBC", version="2.34" - ), - IncompatibleManylinuxSymbol(name="stat64", dep="GLIBC", version="2.33"), - ] - - assert o.format() == BinaryFormat.ELF diff --git a/kernel-abi-check/flake.lock b/kernel-abi-check/flake.lock deleted file mode 100644 index 0f8378d3..00000000 --- a/kernel-abi-check/flake.lock +++ /dev/null @@ -1,82 +0,0 @@ -{ - "nodes": { - "flake-utils": { - "inputs": { - "systems": "systems" - }, - "locked": { - "lastModified": 1731533236, - "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "nixpkgs": { - "locked": { - "lastModified": 1758427187, - "narHash": "sha256-pHpxZ/IyCwoTQPtFIAG2QaxuSm8jWzrzBGjwQZIttJc=", - "owner": "nixos", - "repo": "nixpkgs", - "rev": "554be6495561ff07b6c724047bdd7e0716aa7b46", - "type": "github" - }, - "original": { - "owner": "nixos", - "ref": "nixos-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "root": { - "inputs": { - "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs", - "rust-overlay": "rust-overlay" - } - }, - "rust-overlay": { - "inputs": { - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1758594771, - "narHash": "sha256-loYxdliGF/ytyAorc36Tt/PwBpc2rAfMSJycNxc2oeg=", - "owner": "oxalica", - "repo": "rust-overlay", - "rev": "96722b8da34a7d796668b9a1cbcb7e799cc524b5", - "type": "github" - }, - "original": { - "owner": "oxalica", - "repo": "rust-overlay", - "type": "github" - } - }, - "systems": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/kernel-abi-check/kernel-abi-check/src/lib.rs b/kernel-abi-check/src/lib.rs similarity index 100% rename from kernel-abi-check/kernel-abi-check/src/lib.rs rename to kernel-abi-check/src/lib.rs diff --git a/kernel-abi-check/kernel-abi-check/src/macos.rs b/kernel-abi-check/src/macos.rs similarity index 100% rename from kernel-abi-check/kernel-abi-check/src/macos.rs rename to kernel-abi-check/src/macos.rs diff --git a/kernel-abi-check/kernel-abi-check/src/manylinux/manylinux-policy.json b/kernel-abi-check/src/manylinux/manylinux-policy.json similarity index 100% rename from kernel-abi-check/kernel-abi-check/src/manylinux/manylinux-policy.json rename to kernel-abi-check/src/manylinux/manylinux-policy.json diff --git a/kernel-abi-check/kernel-abi-check/src/manylinux/mod.rs b/kernel-abi-check/src/manylinux/mod.rs similarity index 100% rename from kernel-abi-check/kernel-abi-check/src/manylinux/mod.rs rename to kernel-abi-check/src/manylinux/mod.rs diff --git a/kernel-abi-check/kernel-abi-check/src/python_abi/mod.rs b/kernel-abi-check/src/python_abi/mod.rs similarity index 100% rename from kernel-abi-check/kernel-abi-check/src/python_abi/mod.rs rename to kernel-abi-check/src/python_abi/mod.rs diff --git a/kernel-abi-check/kernel-abi-check/src/python_abi/stable_abi.toml b/kernel-abi-check/src/python_abi/stable_abi.toml similarity index 100% rename from kernel-abi-check/kernel-abi-check/src/python_abi/stable_abi.toml rename to kernel-abi-check/src/python_abi/stable_abi.toml diff --git a/kernel-abi-check/kernel-abi-check/src/torch_stable_abi/mod.rs b/kernel-abi-check/src/torch_stable_abi/mod.rs similarity index 100% rename from kernel-abi-check/kernel-abi-check/src/torch_stable_abi/mod.rs rename to kernel-abi-check/src/torch_stable_abi/mod.rs diff --git a/kernel-abi-check/kernel-abi-check/src/torch_stable_abi/shim_function_versions.txt b/kernel-abi-check/src/torch_stable_abi/shim_function_versions.txt similarity index 100% rename from kernel-abi-check/kernel-abi-check/src/torch_stable_abi/shim_function_versions.txt rename to kernel-abi-check/src/torch_stable_abi/shim_function_versions.txt diff --git a/kernel-abi-check/kernel-abi-check/src/version.rs b/kernel-abi-check/src/version.rs similarity index 100% rename from kernel-abi-check/kernel-abi-check/src/version.rs rename to kernel-abi-check/src/version.rs diff --git a/kernel-builder/Cargo.toml b/kernel-builder/Cargo.toml index a4625352..556d28ab 100644 --- a/kernel-builder/Cargo.toml +++ b/kernel-builder/Cargo.toml @@ -10,7 +10,9 @@ repository = "https://github.com/huggingface/kernels" [dependencies] base32 = "0.5" +kernel-abi-check = { path = "../kernel-abi-check", version = "0.15.0-dev0" } kernels-data = { path = "../kernels-data", version = "0.15.0-dev0" } +object = "0.36.7" clap = { version = "4", features = ["derive"] } clap-markdown = "0.1.5" clap_complete = "4" diff --git a/kernel-abi-check/kernel-abi-check/src/main.rs b/kernel-builder/src/check_abi.rs similarity index 71% rename from kernel-abi-check/kernel-abi-check/src/main.rs rename to kernel-builder/src/check_abi.rs index b3b13a80..198ea0bf 100644 --- a/kernel-abi-check/kernel-abi-check/src/main.rs +++ b/kernel-builder/src/check_abi.rs @@ -1,7 +1,7 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::{collections::BTreeSet, fs}; -use clap::Parser; +use clap::Args; use eyre::{Context, Result}; use object::{File, Object}; @@ -9,13 +9,14 @@ use kernel_abi_check::{ check_macos, check_manylinux, check_python_abi, check_torch_stable_abi, MacOSViolation, ManylinuxViolation, PythonAbiViolation, TorchStableAbiViolation, Version, }; +use walkdir::WalkDir; -/// CLI tool to check library versions -#[derive(Parser, Debug)] -#[command(version, about, long_about = None)] -struct Cli { - /// Python extension library. - object: PathBuf, +use crate::util::{check_or_infer_kernel_dir, discover_variants}; + +#[derive(Args, Debug)] +pub struct CheckAbiArgs { + /// Directory with kernels. + kernel_dir: Option, /// Manylinux version. #[arg(short, long, value_name = "VERSION", default_value = "manylinux_2_28")] @@ -34,14 +35,43 @@ struct Cli { torch_stable_abi: Option, } -fn main() -> Result<()> { - // Initialize color_eyre error handling - color_eyre::install()?; +/// Recursively walk a directory and return all file paths. +fn shared_library_iter(dir: &Path) -> impl Iterator { + WalkDir::new(dir) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().is_file()) + .filter(|e| { + e.path() + .extension() + .is_some_and(|ext| ext == "so" || ext == "pyd") + }) + .map(|e| e.into_path()) +} + +pub fn run_check_abi(args: CheckAbiArgs) -> Result<()> { + let kernel_dir = check_or_infer_kernel_dir(args.kernel_dir.as_ref())?; + let kernel_dir = fs::canonicalize(&kernel_dir) + .wrap_err_with(|| format!("Cannot resolve kernel directory `{}`", kernel_dir.display()))?; - // Parse command-line arguments - let args = Cli::parse(); + let mut has_failure = false; + let (_, variants) = discover_variants(&kernel_dir)?; + for variant_path in variants { + for shared_lib_path in shared_library_iter(&variant_path) { + has_failure |= check_shared_library_abi(&shared_lib_path, &args).is_err(); + } + } - let binary_data = fs::read(args.object).context("Cannot open object file")?; + if has_failure { + eyre::bail!("ABI compatibility issues found"); + } + + Ok(()) +} + +fn check_shared_library_abi(path: impl AsRef, args: &CheckAbiArgs) -> Result<()> { + let path = path.as_ref(); + let binary_data = fs::read(path).context("Cannot open object file")?; let file = object::File::parse(&*binary_data).context("Cannot parse object")?; let mut manylinux_violations = BTreeSet::new(); @@ -49,10 +79,11 @@ fn main() -> Result<()> { match file { File::Elf32(_) | File::Elf64(_) => { - // Assume for now that ELF is Linux. eprintln!( - "🐍 Checking for compatibility with {} and Python ABI version {}", - args.manylinux, args.python_abi + "🐍 Checking for compatibility with {} and Python ABI version {}: {}", + args.manylinux, + args.python_abi, + path.to_string_lossy(), ); manylinux_violations = check_manylinux( @@ -65,8 +96,10 @@ fn main() -> Result<()> { } File::MachO32(_) | File::MachO64(_) => { eprintln!( - "🐍 Checking for compatibility with macOS {}, and Python ABI version {}", - args.macos, args.python_abi + "🐍 Checking for compatibility with macOS {}, and Python ABI version {}: {}", + args.macos, + args.python_abi, + path.to_string_lossy(), ); macos_violations = check_macos(&file, &args.macos)?; print_macos_violations(&macos_violations, &args.macos); @@ -79,28 +112,24 @@ fn main() -> Result<()> { let python_abi_violations = check_python_abi(&args.python_abi, file.format(), file.symbols())?; print_python_abi_violations(&python_abi_violations, &args.python_abi); + let mut torch_stable_abi_violations = BTreeSet::new(); + if let Some(torch_stable_abi) = &args.torch_stable_abi { + eprintln!("🔥 Checking for compatibility with Torch stable ABI version {torch_stable_abi}"); + torch_stable_abi_violations = + check_torch_stable_abi(torch_stable_abi, file.format(), file.symbols())?; + print_torch_stable_abi_violations(&torch_stable_abi_violations, torch_stable_abi); + } + if !(manylinux_violations.is_empty() && macos_violations.is_empty() - && python_abi_violations.is_empty()) + && python_abi_violations.is_empty() + && torch_stable_abi_violations.is_empty()) { return Err(eyre::eyre!("Compatibility issues found")); } else { eprintln!("✅ No compatibility issues found"); } - if let Some(torch_stable_abi) = args.torch_stable_abi { - eprintln!("🔥 Checking for compatibility with Torch stable ABI version {torch_stable_abi}"); - let torch_stable_abi_violations = - check_torch_stable_abi(&torch_stable_abi, file.format(), file.symbols())?; - print_torch_stable_abi_violations(&torch_stable_abi_violations, &torch_stable_abi); - - if !torch_stable_abi_violations.is_empty() { - return Err(eyre::eyre!("Torch stable ABI compatibility issues found")); - } else { - eprintln!("✅ No Torch stable ABI compatibility issues found"); - } - } - Ok(()) } @@ -128,7 +157,7 @@ fn print_manylinux_violations( manylinux_version: &str, ) -> Result<()> { if !violations.is_empty() { - eprintln!("\n⛔ Symbols incompatible with `{manylinux_version}` found:\n",); + eprintln!("\n⛔ Symbols incompatible with `{manylinux_version}` found:\n"); for violation in violations { match violation { ManylinuxViolation::Symbol { name, dep, version } => { @@ -169,7 +198,7 @@ fn print_python_abi_violations(violations: &BTreeSet, python .collect::>(); if !newer_abi3_symbols.is_empty() { - eprintln!("\n⛔ Symbols >= Python ABI {python_abi} found:\n",); + eprintln!("\n⛔ Symbols >= Python ABI {python_abi} found:\n"); for violation in newer_abi3_symbols { if let PythonAbiViolation::IncompatibleAbi3Symbol { name, added } = violation { eprintln!("{name}: {added}"); diff --git a/kernel-builder/src/main.rs b/kernel-builder/src/main.rs index 19a8bbf1..3c024cbb 100644 --- a/kernel-builder/src/main.rs +++ b/kernel-builder/src/main.rs @@ -3,6 +3,8 @@ use std::io::{BufWriter, Write}; use std::path::PathBuf; mod card; +mod check_abi; +use check_abi::{run_check_abi, CheckAbiArgs}; use clap::{Args, CommandFactory, Parser, Subcommand}; use clap_complete::Shell; @@ -142,6 +144,9 @@ enum Commands { kernel_dir: Option, }, + /// Check the ABI compatibility of a kernel extension. + CheckAbi(CheckAbiArgs), + /// Validate kernel builds. CheckBuilds { #[arg(name = "KERNEL_DIR")] @@ -380,6 +385,7 @@ fn main() -> Result<()> { check_config(kernel_dir)?; Ok(()) } + Commands::CheckAbi(args) => run_check_abi(args), Commands::CheckBuilds { kernel_dir } => { check_builds(kernel_dir)?; Ok(()) diff --git a/kernel-builder/src/util.rs b/kernel-builder/src/util.rs index d6104820..b18c9704 100644 --- a/kernel-builder/src/util.rs +++ b/kernel-builder/src/util.rs @@ -12,15 +12,16 @@ pub(crate) fn parse_build(kernel_dir: impl AsRef) -> Result { Ok(build_compat.into()) } -pub(crate) fn check_or_infer_kernel_dir(kernel_dir: Option) -> Result { +pub(crate) fn check_or_infer_kernel_dir(kernel_dir: Option>) -> Result { match kernel_dir { Some(kernel_dir) => { + let kernel_dir = kernel_dir.as_ref(); ensure!( kernel_dir.is_dir(), "`{}` is not a directory", kernel_dir.to_string_lossy() ); - Ok(kernel_dir) + Ok(kernel_dir.to_owned()) } None => Ok(current_dir()?), } diff --git a/kernels/pyproject.toml b/kernels/pyproject.toml index dfcf4ac5..f740a442 100644 --- a/kernels/pyproject.toml +++ b/kernels/pyproject.toml @@ -38,7 +38,6 @@ dev = [ ] [project.optional-dependencies] -abi-check = ["kernel-abi-check>=0.6.2,<0.7.0"] benchmark = [ "matplotlib>=3.7.0", "numpy>=2.0.2", diff --git a/kernels/src/kernels/cli/__init__.py b/kernels/src/kernels/cli/__init__.py index b3c3d92b..f3785e70 100644 --- a/kernels/src/kernels/cli/__init__.py +++ b/kernels/src/kernels/cli/__init__.py @@ -18,25 +18,8 @@ def main(): subparsers = parser.add_subparsers(required=True) check_parser = subparsers.add_parser("check", help="Check a kernel for compliance") - check_parser.add_argument("repo_id", type=str, help="The kernel repo ID") - check_parser.add_argument( - "--revision", - type=str, - default="main", - help="The kernel revision (branch, tag, or commit SHA, defaults to 'main')", - ) - check_parser.add_argument("--macos", type=str, help="macOS version", default="15.0") - check_parser.add_argument("--manylinux", type=str, help="Manylinux version", default="manylinux_2_28") - check_parser.add_argument("--python-abi", type=str, help="Python ABI version", default="3.9") - check_parser.set_defaults( - func=lambda args: check_kernel( - macos=args.macos, - manylinux=args.manylinux, - python_abi=args.python_abi, - repo_id=args.repo_id, - revision=args.revision, - ) - ) + check_parser.add_argument("repo_id", type=str, nargs="?") + check_parser.set_defaults(func=_check_moved) download_parser = subparsers.add_parser("download", help="Download locked kernels") download_parser.add_argument( @@ -168,23 +151,12 @@ def default(self, o): return super().default(o) -def check_kernel(*, macos: str, manylinux: str, python_abi: str, repo_id: str, revision: str): - try: - from kernels.cli import check - except ImportError: - print( - "`kernels check` requires the `kernel-abi-check` package: pip install kernel-abi-check", - file=sys.stderr, - ) - sys.exit(1) - - check.check_kernel( - macos=macos, - manylinux=manylinux, - python_abi=python_abi, - repo_id=repo_id, - revision=revision, +def _check_moved(_args): + print( + "`kernels check` has moved to `kernel-builder check-abi`", + file=sys.stderr, ) + sys.exit(1) def run_benchmark(args): diff --git a/kernels/src/kernels/cli/check.py b/kernels/src/kernels/cli/check.py deleted file mode 100644 index 37b5a7a8..00000000 --- a/kernels/src/kernels/cli/check.py +++ /dev/null @@ -1,149 +0,0 @@ -import sys -from pathlib import Path - -from kernel_abi_check import ( # type: ignore[import-not-found] - BinaryFormat, - IncompatibleAbi3Symbol, - IncompatibleMacOSVersion, - IncompatibleManylinuxSymbol, - MissingMacOSVersion, - NonAbi3Symbol, - ObjectFile, -) - -from kernels.utils import CACHE_DIR, _get_hf_api - - -def check_kernel( - *, - macos: str, - manylinux: str, - python_abi: str, - repo_id: str, - revision: str, -): - variants_path = ( - Path( - str( - _get_hf_api().snapshot_download( - repo_id, - repo_type="kernel", - allow_patterns=["build/*"], - cache_dir=CACHE_DIR, - revision=revision, - ) - ) - ) - / "build" - ) - - has_issues = False - for variant_path in variants_path.iterdir(): - if not variant_path.is_dir(): - print( - f"⛔ `build/` must only contain directories, found: {variant_path.name}", - file=sys.stderr, - ) - has_issues = True - continue - - print(f"Checking variant: {variant_path.name}", file=sys.stderr) - - indent = 2 - - for dylib_path in variant_path.rglob("*.so"): - print_with_indent( - indent, - f"Dynamic library {dylib_path.relative_to(variant_path)}:", - ) - - o = ObjectFile(dylib_path) - has_issues |= check_abi3(o, python_abi, indent + 2) - - # TODO: also check operating system - if o.format() == BinaryFormat.ELF: - has_issues |= check_manylinux(o, manylinux, indent + 2) - elif o.format() == BinaryFormat.MACH_O: - has_issues |= check_macos(o, macos, indent + 2) - - if has_issues: - sys.exit(1) - - -def check_abi3(object_file: ObjectFile, python_abi: str, indent: int) -> bool: - has_issues = False - violations = object_file.check_python_abi(python_abi) - if violations != []: - has_issues = True - print_with_indent( - indent, - f"⛔ Found symbols that are incompatible with Python ABI {python_abi}:", - ) - for violation in violations: - if isinstance(violation, IncompatibleAbi3Symbol): - print_with_indent( - indent + 3, - f"{violation.name}: {violation.version_added}", - ) - elif isinstance(violation, NonAbi3Symbol): - print_with_indent( - indent + 3, - f"{violation.name}", - ) - else: - print_with_indent(indent, f"🐍 Python ABI {python_abi} compatible") - - return has_issues - - -def check_macos(object_file: ObjectFile, macos: str, indent: int) -> bool: - has_issues = False - violations = object_file.check_macos(macos) - if violations != []: - has_issues = True - print_with_indent( - indent, - f"⛔ Found incompatibility with macOS {macos}:", - ) - - for violation in violations: - if isinstance(violation, MissingMacOSVersion): - print_with_indent( - indent + 3, - "shared library does not contain macOS version", - ) - elif isinstance(violation, IncompatibleMacOSVersion): - print_with_indent( - indent + 3, - f"shared library requires macOS {violation.version}", - ) - else: - print_with_indent(indent, f"🍏 compatible with macOS {macos}") - - return has_issues - - -def check_manylinux(object_file: ObjectFile, manylinux: str, indent: int) -> bool: - has_issues = False - violations = object_file.check_manylinux(manylinux) - if violations != []: - has_issues = True - print_with_indent( - indent, - f"⛔ Found symbols that are incompatible with {manylinux}:", - ) - - for violation in violations: - if isinstance(violation, IncompatibleManylinuxSymbol): - print_with_indent( - indent + 3, - f"{violation.name}_{violation.dep}: {violation.version}", - ) - else: - print_with_indent(indent, f"🐧 {manylinux} compatible") - - return has_issues - - -def print_with_indent(indent: int, message: str): - print(f"{' ' * indent}{message}", file=sys.stderr) diff --git a/nix-builder/lib/build.nix b/nix-builder/lib/build.nix index 2948af4b..4b49871d 100644 --- a/nix-builder/lib/build.nix +++ b/nix-builder/lib/build.nix @@ -435,7 +435,6 @@ rec { with pkgs; [ kernel-builder - kernel-abi-check ] ++ (pythonNativeCheckInputs python3.pkgs); buildInputs = [ python ]; diff --git a/nix-builder/lib/cache.nix b/nix-builder/lib/cache.nix index 1888fc2a..05f13660 100644 --- a/nix-builder/lib/cache.nix +++ b/nix-builder/lib/cache.nix @@ -24,7 +24,6 @@ allOutputs buildSet.torch ++ lib.concatMap allOutputs buildSet.extension.extraBuildDeps ++ allOutputs kernel-builder - ++ allOutputs kernel-abi-check ++ allOutputs python3.pkgs.einops ++ allOutputs python3.pkgs.jax ++ allOutputs python3.pkgs.jax-tvm-ffi diff --git a/nix-builder/lib/extension/torch/arch.nix b/nix-builder/lib/extension/torch/arch.nix index a7413851..cebee40f 100644 --- a/nix-builder/lib/extension/torch/arch.nix +++ b/nix-builder/lib/extension/torch/arch.nix @@ -14,7 +14,6 @@ cmakeNvccThreadsHook, cuda_nvcc, get-kernel-check, - kernel-abi-check, kernel-layout-check, torch-ops-check, ninja, @@ -164,7 +163,6 @@ stdenv.mkDerivation (prevAttrs: { cmake ninja kernel-builder - kernel-abi-check kernel-layout-check remove-bytecode-hook torch-ops-check diff --git a/nix-builder/lib/extension/tvm-ffi/arch.nix b/nix-builder/lib/extension/tvm-ffi/arch.nix index 7c3b7fe9..4e3271cb 100644 --- a/nix-builder/lib/extension/tvm-ffi/arch.nix +++ b/nix-builder/lib/extension/tvm-ffi/arch.nix @@ -14,7 +14,6 @@ cmakeNvccThreadsHook, cuda_nvcc, get-kernel-check, - kernel-abi-check, kernel-layout-check, ninja, python3, @@ -161,7 +160,6 @@ stdenv.mkDerivation (prevAttrs: { cmake ninja kernel-builder - kernel-abi-check kernel-layout-check remove-bytecode-hook torch-ops-check diff --git a/nix-builder/lib/gen-flake-outputs.nix b/nix-builder/lib/gen-flake-outputs.nix index c00470dd..5409edb3 100644 --- a/nix-builder/lib/gen-flake-outputs.nix +++ b/nix-builder/lib/gen-flake-outputs.nix @@ -269,7 +269,6 @@ in kernels = pkgs.python3.withPackages ( ps: with ps; [ - kernel-abi-check kernels ] ) diff --git a/nix-builder/overlay.nix b/nix-builder/overlay.nix index d7f5a507..7a64a340 100644 --- a/nix-builder/overlay.nix +++ b/nix-builder/overlay.nix @@ -14,8 +14,6 @@ in get-kernel-check = final.callPackage ./pkgs/get-kernel-check { }; - kernel-abi-check = final.callPackage ./pkgs/kernel-abi-check { }; - kernel-layout-check = final.callPackage ./pkgs/kernel-layout-check { }; nvtx = final.callPackage ./pkgs/nvtx { }; @@ -143,8 +141,6 @@ in nvidia-cutlass-dsl-libs = python-self.callPackage ./pkgs/python-modules/nvidia-cutlass-dsl-libs { }; - kernel-abi-check = callPackage ./pkgs/python-modules/kernel-abi-check { }; - kernels = callPackage ./pkgs/python-modules/kernels { }; kernels-data = callPackage ./pkgs/python-modules/kernels-data { }; diff --git a/nix-builder/pkgs/kernel-abi-check/default.nix b/nix-builder/pkgs/kernel-abi-check/default.nix deleted file mode 100644 index 7dc30851..00000000 --- a/nix-builder/pkgs/kernel-abi-check/default.nix +++ /dev/null @@ -1,46 +0,0 @@ -{ - lib, - rustPlatform, -}: - -let - version = - (builtins.fromTOML (builtins.readFile ../../../kernel-abi-check/kernel-abi-check/Cargo.toml)) - .package.version; - cargoFlags = [ - "-p" - "kernel-abi-check" - ]; -in -rustPlatform.buildRustPackage { - inherit version; - pname = "kernel-abi-check"; - - src = - let - sourceFiles = - file: - file.name == "Cargo.toml" - || file.name == "Cargo.lock" - || file.name == "manylinux-policy.json" - || file.hasExt "rs" - || file.name == "shim_function_versions.txt" - || file.name == "stable_abi.toml"; - in - import ../crate-dirs.nix { - inherit lib sourceFiles; - }; - - cargoLock = { - lockFile = ../../../Cargo.lock; - }; - - cargoBuildFlags = cargoFlags; - cargoTestFlags = cargoFlags; - - setupHook = ./kernel-abi-check-hook.sh; - - meta = { - description = "Check glibc and libstdc++ ABI compat"; - }; -} diff --git a/nix-builder/pkgs/kernel-abi-check/kernel-abi-check-hook.sh b/nix-builder/pkgs/kernel-builder/check-kernel-abi-hook.sh similarity index 51% rename from nix-builder/pkgs/kernel-abi-check/kernel-abi-check-hook.sh rename to nix-builder/pkgs/kernel-builder/check-kernel-abi-hook.sh index 3e9293f0..40d80efd 100755 --- a/nix-builder/pkgs/kernel-abi-check/kernel-abi-check-hook.sh +++ b/nix-builder/pkgs/kernel-builder/check-kernel-abi-hook.sh @@ -1,21 +1,18 @@ #!/bin/sh -echo "Sourcing kernel-abi-check-hook.sh" +echo "Sourcing check-kernel-abi-hook.sh" _checkAbiHook() { if [ -z "${doAbiCheck:-}" ]; then echo "Skipping ABI check" else + echo "Checking of ABI compatibility" if [ -z "${torchStableAbiVersion:-}" ]; then - _torchStableAbiFlag="" + kernel-builder check-abi "$out/" else - _torchStableAbiFlag="--torch-stable-abi=${torchStableAbiVersion}" + kernel-builder check-abi --torch-stable-abi="${torchStableAbiVersion}" "$out/" fi - - echo "Checking of ABI compatibility" - find "$out/" -name '*.so' -print0 | \ - xargs -0 -n1 kernel-abi-check ${_torchStableAbiFlag} fi } diff --git a/nix-builder/pkgs/kernel-builder/default.nix b/nix-builder/pkgs/kernel-builder/default.nix index 39fa2ade..2fb34485 100644 --- a/nix-builder/pkgs/kernel-builder/default.nix +++ b/nix-builder/pkgs/kernel-builder/default.nix @@ -26,9 +26,12 @@ rustPlatform.buildRustPackage { file.name == "Cargo.toml" || file.name == "Cargo.lock" || file.name == "flake.nix" + || file.name == "manylinux-policy.json" || file.name == "pyproject.toml" || file.name == "pyproject_universal.toml" || file.name == "python_dependencies.json" + || file.name == "shim_function_versions.txt" + || file.name == "stable_abi.toml" || file.name == ".gitattributes" || file.name == ".gitignore" || (builtins.any file.hasExt [ @@ -76,7 +79,10 @@ rustPlatform.buildRustPackage { installShellCompletion kernel-builder.{bash,fish,zsh} ''; - setupHook = ./check-kernel-build-hook.sh; + setupHooks = [ + ./check-kernel-abi-hook.sh + ./check-kernel-build-hook.sh + ]; meta = { description = "Create cmake build infrastructure from build.toml files"; diff --git a/nix-builder/pkgs/python-modules/kernel-abi-check/default.nix b/nix-builder/pkgs/python-modules/kernel-abi-check/default.nix deleted file mode 100644 index eb4babf9..00000000 --- a/nix-builder/pkgs/python-modules/kernel-abi-check/default.nix +++ /dev/null @@ -1,54 +0,0 @@ -{ - lib, - buildPythonPackage, - rustPlatform, -}: - -let - version = - (builtins.fromTOML (builtins.readFile ../../../../kernel-abi-check/kernel-abi-check/Cargo.toml)) - .package.version; - cargoFlags = [ - "-m" - "kernel-abi-check/bindings/python/Cargo.toml" - ]; -in -buildPythonPackage { - pname = "kernel-abi-check"; - inherit version; - format = "pyproject"; - - src = - let - sourceFiles = - file: - file.name == "Cargo.toml" - || file.name == "Cargo.lock" - || file.name == "manylinux-policy.json" - || file.hasExt "pyi" - || file.name == "pyproject.toml" - || file.hasExt "rs" - || file.name == "shim_function_versions.txt" - || file.name == "stable_abi.toml"; - in - import ../../crate-dirs.nix { - inherit lib sourceFiles; - }; - - cargoDeps = rustPlatform.importCargoLock { - lockFile = ../../../../Cargo.lock; - }; - - maturinBuildFlags = cargoFlags; - - #sourceRoot = "source/bindings/python"; - - build-system = [ - rustPlatform.cargoSetupHook - rustPlatform.maturinBuildHook - ]; - - meta = with lib; { - description = "Check ABI compliance of Hugging Face Hub kernels"; - }; -} diff --git a/nix-builder/pkgs/python-modules/kernels/default.nix b/nix-builder/pkgs/python-modules/kernels/default.nix index 0dfdd91b..0e51c423 100644 --- a/nix-builder/pkgs/python-modules/kernels/default.nix +++ b/nix-builder/pkgs/python-modules/kernels/default.nix @@ -4,7 +4,6 @@ setuptools, huggingface-hub, - kernel-abi-check, kernels-data, pyyaml, tomlkit, @@ -34,7 +33,6 @@ buildPythonPackage { dependencies = [ huggingface-hub - kernel-abi-check kernels-data pyyaml tomlkit diff --git a/scripts/_version_common.py b/scripts/_version_common.py index 52668c9a..d680aefc 100644 --- a/scripts/_version_common.py +++ b/scripts/_version_common.py @@ -21,8 +21,7 @@ REPO_ROOT / "kernels-data" / "Cargo.toml", REPO_ROOT / "kernels-data" / "bindings" / "python" / "Cargo.toml", REPO_ROOT / "kernel-builder" / "Cargo.toml", - REPO_ROOT / "kernel-abi-check" / "kernel-abi-check" / "Cargo.toml", - REPO_ROOT / "kernel-abi-check" / "bindings" / "python" / "Cargo.toml", + REPO_ROOT / "kernel-abi-check" / "Cargo.toml", ]