diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index a060384..2d7efda 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -5,6 +5,10 @@ inputs: rust-target: description: "Rust target triple (e.g., x86_64-unknown-linux-gnu)" required: true + save-cache: + description: "Save caches at the end (set false for PR builds to save GHA cache budget)" + required: false + default: "true" runs: using: "composite" @@ -25,14 +29,17 @@ runs: env: RUSTUP_TOOLCHAIN: ${{ steps.toolchain.outputs.channel }} - # Layers 1+2: zccache (compilation cache + cargo registry) - # Replaces both sccache and Swatinem/rust-cache + # 3-layer caching via zccache (replaces sccache + Swatinem/rust-cache): + # 1. Compilation cache (~/.zccache) — cached .o/.rlib per compilation unit + # 2. Cargo registry — index, crate downloads, git deps + # 3. Target metadata — fingerprints + dep-info; warm restores .rlib/.rmeta - name: Setup zccache uses: zackees/zccache@main with: shared-key: ${{ inputs.rust-target }} + save-cache: ${{ inputs.save-cache }} - # Layer 3: uv cache (Python packages) + # Layer 4: uv cache (Python packages) - name: Setup uv uses: astral-sh/setup-uv@v5 with: @@ -46,6 +53,6 @@ runs: - name: Set build environment shell: bash run: | - echo "RUSTC_WRAPPER=zccache" >> "$GITHUB_ENV" + # RUSTC_WRAPPER=zccache is set by the zccache action above echo "CARGO_BUILD_TARGET=${{ inputs.rust-target }}" >> "$GITHUB_ENV" echo "RUSTUP_TOOLCHAIN=${{ steps.toolchain.outputs.channel }}" >> "$GITHUB_ENV" diff --git a/.github/workflows/_build-and-test.yml b/.github/workflows/_build-and-test.yml index 6d027a8..3daf9ed 100644 --- a/.github/workflows/_build-and-test.yml +++ b/.github/workflows/_build-and-test.yml @@ -13,6 +13,10 @@ on: required: false type: string default: "" + save-cache: + required: false + type: string + default: "true" jobs: build: @@ -25,6 +29,7 @@ jobs: uses: ./.github/actions/setup with: rust-target: ${{ inputs.target }} + save-cache: ${{ inputs.save-cache }} - name: Build wheel shell: bash @@ -80,6 +85,7 @@ jobs: uses: ./.github/actions/setup with: rust-target: ${{ inputs.target }} + save-cache: ${{ inputs.save-cache }} - name: Rust fmt shell: bash @@ -113,6 +119,7 @@ jobs: uses: ./.github/actions/setup with: rust-target: ${{ inputs.target }} + save-cache: ${{ inputs.save-cache }} - name: Rust tests shell: bash diff --git a/.github/workflows/benchmark-cache.yml b/.github/workflows/benchmark-cache.yml new file mode 100644 index 0000000..dc02228 --- /dev/null +++ b/.github/workflows/benchmark-cache.yml @@ -0,0 +1,150 @@ +name: "Benchmark: zccache vs sccache" + +on: + workflow_dispatch: + +concurrency: + group: benchmark-${{ github.ref }} + cancel-in-progress: true + +jobs: + # --- Legacy: sccache + Swatinem/rust-cache --- + sccache-legacy: + name: "sccache + Swatinem/rust-cache (legacy)" + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - name: Read rust-toolchain.toml + id: toolchain + shell: bash + run: | + CHANNEL=$(grep 'channel' rust-toolchain.toml | sed 's/.*= *"\(.*\)"/\1/') + echo "channel=$CHANNEL" >> "$GITHUB_OUTPUT" + + - name: Install Rust toolchain + shell: bash + run: | + rustup toolchain install ${{ steps.toolchain.outputs.channel }} \ + --profile minimal --no-self-update + env: + RUSTUP_TOOLCHAIN: ${{ steps.toolchain.outputs.channel }} + + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.9 + + - name: Setup Swatinem/rust-cache (registry only) + uses: Swatinem/rust-cache@v2 + with: + cache-targets: false + shared-key: benchmark-sccache + + - name: Setup uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Install Python deps + run: uv sync --no-install-project + + - name: "Build (cold — sccache)" + shell: bash + run: | + export SCCACHE_GHA_ENABLED=true + export RUSTC_WRAPPER=sccache + export CARGO_BUILD_TARGET=x86_64-unknown-linux-gnu + export RUSTUP_TOOLCHAIN=${{ steps.toolchain.outputs.channel }} + + echo "::group::Cold build" + START=$(date +%s%3N) + uv run maturin build --release --target x86_64-unknown-linux-gnu --zig --compatibility manylinux2014 + END=$(date +%s%3N) + COLD_MS=$((END - START)) + echo "::endgroup::" + echo "### sccache cold build: ${COLD_MS}ms" >> "$GITHUB_STEP_SUMMARY" + + # Clean build artifacts (but not caches) for warm run + cargo clean + + echo "::group::Warm build" + START=$(date +%s%3N) + uv run maturin build --release --target x86_64-unknown-linux-gnu --zig --compatibility manylinux2014 + END=$(date +%s%3N) + WARM_MS=$((END - START)) + echo "::endgroup::" + echo "### sccache warm build: ${WARM_MS}ms" >> "$GITHUB_STEP_SUMMARY" + + - name: sccache stats + if: always() + run: sccache --show-stats + + # --- Current: zccache (3-layer) --- + zccache-current: + name: "zccache 3-layer (current)" + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - name: Read rust-toolchain.toml + id: toolchain + shell: bash + run: | + CHANNEL=$(grep 'channel' rust-toolchain.toml | sed 's/.*= *"\(.*\)"/\1/') + echo "channel=$CHANNEL" >> "$GITHUB_OUTPUT" + + - name: Install Rust toolchain + shell: bash + run: | + rustup toolchain install ${{ steps.toolchain.outputs.channel }} \ + --profile minimal --no-self-update + env: + RUSTUP_TOOLCHAIN: ${{ steps.toolchain.outputs.channel }} + + - name: Setup zccache + uses: zackees/zccache@main + with: + shared-key: benchmark-zccache + save-cache: "true" + + - name: Setup uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Install Python deps + run: uv sync --no-install-project + + - name: "Build (cold — zccache)" + shell: bash + run: | + export CARGO_BUILD_TARGET=x86_64-unknown-linux-gnu + export RUSTUP_TOOLCHAIN=${{ steps.toolchain.outputs.channel }} + + echo "::group::Cold build" + START=$(date +%s%3N) + uv run maturin build --release --target x86_64-unknown-linux-gnu --zig --compatibility manylinux2014 + END=$(date +%s%3N) + COLD_MS=$((END - START)) + echo "::endgroup::" + echo "### zccache cold build: ${COLD_MS}ms" >> "$GITHUB_STEP_SUMMARY" + + # Clean build artifacts (but not caches) for warm run + cargo clean + + echo "::group::Warm build" + START=$(date +%s%3N) + uv run maturin build --release --target x86_64-unknown-linux-gnu --zig --compatibility manylinux2014 + END=$(date +%s%3N) + WARM_MS=$((END - START)) + echo "::endgroup::" + echo "### zccache warm build: ${WARM_MS}ms" >> "$GITHUB_STEP_SUMMARY" + + - name: zccache stats + if: always() + run: zccache status || true + + - name: Cleanup zccache + if: always() + uses: zackees/zccache/action/cleanup@main diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63ae9a8..bc70139 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,3 +38,5 @@ jobs: runner: ${{ matrix.os }} target: ${{ matrix.target }} maturin-args: ${{ matrix.maturin-args }} + # Only save caches on push to main (skip on PRs to save GHA cache budget) + save-cache: ${{ github.event_name != 'pull_request' }} diff --git a/README.md b/README.md index 53d1522..56fcc5f 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,7 @@ A **GitHub template repository** that provides everything needed to build a Pyth | Rust build | [Maturin](https://github.com/PyO3/maturin) | Correct wheel tags, auditwheel compliance, zero hand-rolling | | Python ABI | [PyO3](https://pyo3.rs/) with `abi3-py310` | One wheel per platform covers all Python 3.10+ | | Python toolchain | [uv](https://github.com/astral-sh/uv) | Fast venv/dependency management, 5s vs 83s for Python setup on Windows | -| Compilation cache | [sccache](https://github.com/mozilla/sccache) | 50%+ hit rate after first run, biggest single Rust build speedup | -| Dependency cache | [Swatinem/rust-cache](https://github.com/Swatinem/rust-cache) | Registry + index only (`cache-targets: false` to avoid sccache overlap) | +| Compilation cache | [zccache](https://github.com/zackees/zccache) | 3-layer caching (~200ms warm vs ~3,200ms sccache), replaces sccache + Swatinem/rust-cache | | Toolchain safety | `_cargo` trampoline | Forces MSVC on Windows, reads pinned version from `rust-toolchain.toml` | | Linux cross-compile | [Zig](https://ziglang.org/) via `maturin --zig` | manylinux2014 compliance without Docker | | Linting | clippy + ruff | Rust + Python in one `./lint` command | @@ -67,7 +66,7 @@ python-rust-build-chain/ │ └── env.py # Cross-platform build environment resolver │ └── .github/ - ├── actions/setup/ # Composite: toolchain + 3-layer cache + ├── actions/setup/ # Composite: toolchain + 4-layer cache (zccache + uv) └── workflows/ ├── ci.yml # Matrix CI (6 platforms, 2 files total) ├── _build-and-test.yml # Reusable: build + lint + test @@ -101,28 +100,32 @@ uv run pytest | Hand-rolled METADATA/WHEEL/RECORD | zccache, fbuild | Maturin handles it | | Version drift across files | fbuild | Single source in `Cargo.toml` workspace | | 25-30 workflow files | running-process, fastled-wasm | Single matrix workflow (2 files) | -| sccache + rust-cache target overlap | running-process | `cache-targets: false` on rust-cache | +| sccache + rust-cache target overlap | running-process | Replaced both with single zccache action | | Hidden maturin compilation in `uv sync` | running-process | `--no-install-project` in CI | | Chocolatey GNU Rust on Windows | fastled-wasm, zccache | `_cargo` trampoline | | No manylinux audit | zccache, fbuild | `auditwheel show` in CI | | Windows ARM `python3.lib` naming | fastled-wasm | Documented workaround in CI | | Toolchain version drift | zccache | Single `rust-toolchain.toml` | -## 3-Layer Caching Strategy +## 4-Layer Caching Strategy ``` Build request - └─> sccache hit? ─── YES ──> use cached object files (50%+ hit rate) + └─> zccache warm ─── target/ metadata restored? ──> cargo skips rustc entirely (~200ms) │ NO - └─> Swatinem/rust-cache ──> cached registry + crate index + └─> zccache compilation cache ──> cached .o/.rlib per unit (~1ms per hit) + └─> zccache registry cache ──> cargo registry index + crate downloads └─> uv cache ──> cached Python packages └─> Full rebuild (rare after first run) ``` -- **Layer 1 — sccache**: Compiler output cache via GitHub Actions cache backend -- **Layer 2 — Swatinem/rust-cache**: Registry + index only (`cache-targets: false`) -- **Layer 3 — uv cache**: Python package cache keyed on `uv.lock` +- **Layer 1 — zccache compilation**: Per-unit object cache (~1ms per warm hit vs ~170ms for sccache) +- **Layer 2 — zccache registry**: Cargo registry index, crate downloads, git deps +- **Layer 3 — zccache target metadata**: Fingerprints + dep-info; `warm` restores .rlib/.rmeta so cargo skips rustc +- **Layer 4 — uv cache**: Python package cache keyed on `uv.lock` + +All three Rust caching layers are handled by a single [`zackees/zccache`](https://github.com/zackees/zccache) action, replacing the old sccache + Swatinem/rust-cache dual setup. PR builds restore caches but skip saving to conserve GHA cache budget. ## Platform Targets diff --git a/ci/env.py b/ci/env.py index 1ed15d0..1bff4ce 100644 --- a/ci/env.py +++ b/ci/env.py @@ -5,7 +5,7 @@ - Reading the pinned toolchain from rust-toolchain.toml - Detecting host target triple - Forcing MSVC on Windows (prevents GNU Rust contamination) -- Finding sccache if available +- Finding zccache/sccache if available - Resolving Visual Studio environment on Windows """ @@ -77,9 +77,12 @@ def build_env() -> dict[str, str]: env["CARGO_BUILD_TARGET"] = triple env["PATH"] = bin_dir + os.pathsep + env.get("PATH", "") - # sccache integration (if available) + # Compilation cache: prefer zccache, fall back to sccache + zccache = shutil.which("zccache") sccache = shutil.which("sccache") - if sccache: + if zccache: + env["RUSTC_WRAPPER"] = zccache + elif sccache: env["RUSTC_WRAPPER"] = sccache # Windows: force MSVC target