diff --git a/.github/actions/setup-prebuild/action.yml b/.github/actions/setup-prebuild/action.yml index 9cfc837b680..6fdd067fab4 100644 --- a/.github/actions/setup-prebuild/action.yml +++ b/.github/actions/setup-prebuild/action.yml @@ -17,9 +17,9 @@ inputs: description: "optional targets override (e.g. wasm32-unknown-unknown)" required: false enable-sccache: - description: "Should sccache be enabled, true by default." + description: "Should sccache be enabled." required: false - default: "true" + default: "false" runs: using: "composite" @@ -46,4 +46,4 @@ runs: toolchain: ${{ inputs.toolchain }} components: ${{ inputs.components }} targets: ${{ inputs.targets }} - enable-sccache: ${{ inputs.enable-sccache }} + enable-sccache: "false" diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml index 4f42d38167a..345a2d08f7e 100644 --- a/.github/actions/setup-rust/action.yml +++ b/.github/actions/setup-rust/action.yml @@ -17,9 +17,9 @@ inputs: description: "optional targets override (e.g. wasm32-unknown-unknown)" required: false enable-sccache: - description: "Should sccache be enabled, true by default." + description: "Should sccache be enabled." required: false - default: "true" + default: "false" runs: using: "composite" diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index 8223ea80a56..aad6caf2fc6 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -46,6 +46,7 @@ jobs: - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: ${{ github.event.pull_request.head.repo.fork == false && 'true' || 'false' }} - name: Install DuckDB run: | diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index fc59a8156ea..13cfa9050ba 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -60,6 +60,7 @@ jobs: - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} - name: Install DuckDB run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54a3960ae28..5e6451d22b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,12 +34,12 @@ jobs: duckdb-ready: name: "DuckDB libraries available in R2" needs: duckdb-mirror - if: ${{ !cancelled() }} + if: "!cancelled()" runs-on: ubuntu-latest timeout-minutes: 5 steps: - name: Verify DuckDB mirror - if: ${{ needs.duckdb-mirror.result == 'failure' }} + if: needs.duckdb-mirror.result == 'failure' run: | echo "DuckDB mirror failed; downstream builds would 404" exit 1 @@ -68,7 +68,7 @@ jobs: name: "Python (lint)" runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=python-lint', github.run_id) + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=python-lint', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 10 steps: @@ -78,6 +78,8 @@ jobs: sccache: s3 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" # Use uvx for ruff to avoid building the Rust extension (saves ~4.5 min) - name: Python Lint - Format run: uvx ruff format --check . @@ -95,7 +97,7 @@ jobs: name: "Python (test)" runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=python-test', github.run_id) + && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=python-test', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 30 env: @@ -108,6 +110,8 @@ jobs: sccache: s3 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" - name: Pytest - Vortex run: | @@ -135,8 +139,7 @@ jobs: python-cuda-test: name: "Python CUDA (test)" if: github.repository == 'vortex-data/vortex' - runs-on: >- - ${{ format('runs-on={0}/runner=gpu/tag=python-cuda-test', github.run_id) }} + runs-on: runs-on=${{ github.run_id }}/runner=gpu/extras=s3-cache/tag=python-cuda-test timeout-minutes: 30 env: RUST_LOG: "info,maturin=off,uv=debug" @@ -150,6 +153,7 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} components: cargo + enable-sccache: "true" - name: Pin rustup proxy to repository toolchain run: | TOOLCHAIN="$(grep '^channel' rust-toolchain.toml | cut -d '"' -f 2)" @@ -171,7 +175,7 @@ jobs: timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-small/image=ubuntu24-full-x64-pre-v2/tag=rust-docs', github.run_id) + && format('runs-on={0}/runner=amd64-small/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-docs', github.run_id) || 'ubuntu-latest' }} steps: - uses: runs-on/action@v2 @@ -180,6 +184,8 @@ jobs: sccache: s3 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" - name: Docs env: # required to make sure docs build for features @@ -196,7 +202,7 @@ jobs: timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner={1}/image=ubuntu24-full-x64-pre-v2/tag={2}', github.run_id, matrix.config.runner, matrix.config.name) + && format('runs-on={0}/runner={1}/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag={2}', github.run_id, matrix.config.runner, matrix.config.name) || 'ubuntu-latest' }} env: # disable lints for build, they will be caught in Rust lint job. @@ -228,8 +234,10 @@ jobs: sccache: s3 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" - name: Install wasm32 target - if: ${{ matrix.config.target == 'wasm32-unknown-unknown' }} + if: matrix.config.target == 'wasm32-unknown-unknown' run: rustup target add wasm32-unknown-unknown - uses: ./.github/actions/check-rebuild with: @@ -244,7 +252,7 @@ jobs: timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-min-deps', github.run_id) + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-min-deps', github.run_id) || 'ubuntu-latest' }} steps: - uses: runs-on/action@v2 @@ -253,6 +261,8 @@ jobs: sccache: s3 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" - run: cargo minimal-versions check --direct --workspace --ignore-private rust-lint: @@ -261,7 +271,7 @@ jobs: timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=rust-lint', github.run_id) + && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-lint', github.run_id) || 'ubuntu-latest' }} steps: - uses: runs-on/action@v2 @@ -270,6 +280,8 @@ jobs: sccache: s3 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" - name: Install nightly for fmt run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rustfmt - name: Rust Lint - Format @@ -339,7 +351,7 @@ jobs: timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-lint-no-default', github.run_id) + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-lint-no-default', github.run_id) || 'ubuntu-latest' }} steps: - uses: runs-on/action@v2 @@ -348,28 +360,21 @@ jobs: sccache: s3 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" - name: Rust Lint - Clippy No Default Features shell: bash run: | cargo hack --no-dev-deps --ignore-private clippy --profile ci --no-default-features -- -D warnings - rust-test-other: - name: "Rust tests (${{ matrix.os }})" + rust-test-windows: + name: "Rust tests (windows-x64)" needs: duckdb-ready timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - include: - - os: windows-x64 - runner: runs-on=${{ github.run_id }}/pool=windows-x64-pre - fallback_runner: windows-latest - - os: linux-arm64 - runner: runs-on=${{ github.run_id }}/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/tag=rust-test-linux-arm64 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && matrix.runner - || matrix.fallback_runner }} + && format('runs-on={0}/pool=windows-x64-pre/extras=s3-cache', github.run_id) + || 'windows-latest' }} steps: - uses: runs-on/action@v2 if: github.repository == 'vortex-data/vortex' @@ -377,12 +382,12 @@ jobs: sccache: s3 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - name: Setup (Windows) - if: matrix.os == 'windows-x64' run: | echo "C:\rust\cargo\bin" >> $env:GITHUB_PATH - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" - name: Rust Tests (Windows) - if: matrix.os == 'windows-x64' run: | cargo nextest run --cargo-profile ci --locked --workspace --all-features --no-fail-fast ` --exclude vortex-bench ` @@ -393,12 +398,33 @@ jobs: --exclude lance-bench --exclude datafusion-bench --exclude random-access-bench ` --exclude compress-bench --exclude xtask --exclude vortex-datafusion ` --exclude gpu-scan-cli --exclude vortex-sqllogictest - - name: Rust Tests (Other) - if: matrix.os != 'windows-x64' + + - name: Alert incident.io + if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/develop' + uses: ./.github/actions/alert-incident-io + with: + api-key: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }} + alert-title: "Rust tests (windows-x64) failed on develop" + deduplication-key: ci-rust-test-windows-x64-failure + + rust-test-linux-arm64: + name: "Rust tests (linux-arm64)" + needs: duckdb-ready + if: github.repository == 'vortex-data/vortex' + timeout-minutes: 30 + runs-on: runs-on=${{ github.run_id }}/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/extras=s3-cache/tag=rust-test-linux-arm64 + steps: + - uses: runs-on/action@v2 + with: + sccache: s3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" + - name: Rust Tests run: | cargo nextest run --cargo-profile ci --locked --workspace --all-features --no-fail-fast --exclude vortex-bench --exclude xtask --exclude vortex-sqllogictest - uses: ./.github/actions/check-rebuild - if: matrix.os != 'windows-x64' with: command: "cargo test --profile ci --locked --workspace --all-features --no-run --exclude vortex-bench --exclude xtask --exclude vortex-sqllogictest" @@ -407,14 +433,14 @@ jobs: uses: ./.github/actions/alert-incident-io with: api-key: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }} - alert-title: "Rust tests (${{ matrix.os }}) failed on develop" - deduplication-key: ci-rust-test-${{ matrix.os }}-failure + alert-title: "Rust tests (linux-arm64) failed on develop" + deduplication-key: ci-rust-test-linux-arm64-failure build-java: name: "Java" runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/pool=amd64-medium-pre-v2/tag=java', github.run_id) + && format('runs-on={0}/pool=amd64-medium-pre-v2/extras=s3-cache/tag=java', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 30 steps: @@ -424,6 +450,8 @@ jobs: sccache: s3 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" - run: ./gradlew javadoc working-directory: ./java - run: ./gradlew check @@ -451,7 +479,7 @@ jobs: timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=cxx-build', github.run_id) + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=cxx-build', github.run_id) || 'ubuntu-latest' }} steps: - uses: runs-on/action@v2 @@ -460,6 +488,8 @@ jobs: sccache: s3 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" - name: Build and run C++ unit tests run: | mkdir -p vortex-cxx/build @@ -480,7 +510,7 @@ jobs: needs: duckdb-ready runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=sql-logic-test', github.run_id) + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=sql-logic-test', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 30 steps: @@ -490,6 +520,8 @@ jobs: sccache: s3 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" - name: Run sqllogictest tests run: | ./vortex-sqllogictest/slt/tpch/generate_data.sh @@ -523,7 +555,7 @@ jobs: name: "Check generated source files are up to date" runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=generated-files', github.run_id) + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=generated-files', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 30 steps: @@ -533,6 +565,8 @@ jobs: sccache: s3 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" - name: Install nightly for cbindgen macro expansion run: rustup toolchain install $NIGHTLY_TOOLCHAIN - name: "regenerate all .fbs/.proto Rust code" @@ -563,7 +597,7 @@ jobs: timeout-minutes: 10 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=cxx-build', github.run_id) + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=cxx-build', github.run_id) || 'ubuntu-latest' }} steps: - uses: runs-on/action@v2 @@ -572,6 +606,8 @@ jobs: sccache: s3 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" - name: "regenerate FFI header file" run: | cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi diff --git a/.github/workflows/claude-write.yml b/.github/workflows/claude-write.yml index acf5add785c..2b1f73d6d51 100644 --- a/.github/workflows/claude-write.yml +++ b/.github/workflows/claude-write.yml @@ -187,8 +187,6 @@ jobs: - name: Setup Rust toolchain uses: ./.github/actions/setup-rust - with: - enable-sccache: "false" - name: Install uv uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 diff --git a/.github/workflows/close-fixed-fuzzer-issues.yml b/.github/workflows/close-fixed-fuzzer-issues.yml index ee203553ac7..9030809f5cd 100644 --- a/.github/workflows/close-fixed-fuzzer-issues.yml +++ b/.github/workflows/close-fixed-fuzzer-issues.yml @@ -27,7 +27,7 @@ jobs: target: [file_io, array_ops, compress_roundtrip] runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=arm64-medium/disk=large/tag=fuzzer-cleanup-{1}', github.run_id, matrix.target) + && format('runs-on={0}/runner=arm64-medium/disk=large/extras=s3-cache/tag=fuzzer-cleanup-{1}', github.run_id, matrix.target) || 'ubuntu-latest' }} timeout-minutes: 30 steps: @@ -42,6 +42,7 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} toolchain: ${{ env.NIGHTLY_TOOLCHAIN }} + enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} - name: Install llvm uses: aminya/setup-cpp@v1 diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 2d04d8d35f3..8c55eb71f7c 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -58,7 +58,7 @@ jobs: timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=bench-codspeed-{1}', github.run_id, matrix.shard) + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=bench-codspeed-{1}', github.run_id, matrix.shard) || 'ubuntu-latest' }} steps: - uses: runs-on/action@v2 @@ -69,6 +69,8 @@ jobs: - name: Setup benchmark environment run: sudo bash scripts/setup-benchmark.sh - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} - uses: ./.github/actions/system-info - run: | sudo apt-get update @@ -98,7 +100,7 @@ jobs: name: "Build Codspeed CUDA benchmarks" timeout-minutes: 30 runs-on: >- - runs-on=${{ github.run_id }}/family=g5/cpu=8/image=ubuntu24-gpu-x64/tag=bench-codspeed-cuda-build + runs-on=${{ github.run_id }}/family=g5/cpu=8/image=ubuntu24-gpu-x64/extras=s3-cache/tag=bench-codspeed-cuda-build steps: - uses: runs-on/action@v2 with: @@ -107,6 +109,7 @@ jobs: - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: "true" - name: Install Codspeed uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 with: @@ -142,7 +145,7 @@ jobs: name: "Benchmark with Codspeed (CUDA Shard #${{ matrix.shard }} - ${{ matrix.name }})" timeout-minutes: 30 runs-on: >- - runs-on=${{ github.run_id }}/family=g5/cpu=8/image=ubuntu24-gpu-x64/tag=bench-codspeed-cuda-${{ matrix.shard }} + runs-on=${{ github.run_id }}/family=g5/cpu=8/image=ubuntu24-gpu-x64/extras=s3-cache/tag=bench-codspeed-cuda-${{ matrix.shard }} steps: - uses: runs-on/action@v2 with: @@ -151,6 +154,7 @@ jobs: - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: "true" - name: Display NVIDIA SMI details run: | nvidia-smi diff --git a/.github/workflows/compat-gen-upload.yml b/.github/workflows/compat-gen-upload.yml index 8dd4cd1999e..bfc3c580307 100644 --- a/.github/workflows/compat-gen-upload.yml +++ b/.github/workflows/compat-gen-upload.yml @@ -25,7 +25,7 @@ jobs: dry-run: runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=compat-gen-dry-run', github.run_id) + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=compat-gen-dry-run', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 30 permissions: @@ -42,6 +42,8 @@ jobs: with: fetch-depth: 0 - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 @@ -76,7 +78,7 @@ jobs: if: inputs.confirm_upload == 'yes' runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=compat-gen-upload', github.run_id) + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=compat-gen-upload', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 30 environment: compat-upload @@ -92,6 +94,8 @@ jobs: with: fetch-depth: 0 - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 diff --git a/.github/workflows/compat-validation.yml b/.github/workflows/compat-validation.yml index 8732bdfbf6d..60741d50620 100644 --- a/.github/workflows/compat-validation.yml +++ b/.github/workflows/compat-validation.yml @@ -25,7 +25,7 @@ jobs: compat-test: runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=compat-validation', github.run_id) + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=compat-validation', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 30 steps: @@ -35,6 +35,8 @@ jobs: sccache: s3 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} - name: Run compat tests run: | MODE="${{ inputs.mode || 'last' }}" diff --git a/.github/workflows/cuda.yaml b/.github/workflows/cuda.yaml index 2bd9afbbdf1..004ca7d3a49 100644 --- a/.github/workflows/cuda.yaml +++ b/.github/workflows/cuda.yaml @@ -54,7 +54,7 @@ jobs: needs.changes.outputs.run-cuda-san == 'true' name: "CUDA build & lint" timeout-minutes: 30 - runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-build + runs-on: runs-on=${{ github.run_id }}/runner=gpu/extras=s3-cache/tag=cuda-build steps: - uses: runs-on/action@v2 with: @@ -63,6 +63,7 @@ jobs: - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: "true" - uses: ./.github/actions/check-rebuild with: command: >- @@ -88,7 +89,7 @@ jobs: needs.changes.outputs.run-cuda-san == 'true' name: "CUDA tests" timeout-minutes: 30 - runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-tests + runs-on: runs-on=${{ github.run_id }}/runner=gpu/extras=s3-cache/tag=cuda-tests steps: - uses: runs-on/action@v2 with: @@ -102,6 +103,7 @@ jobs: - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: "true" - name: Install uv uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 with: @@ -144,7 +146,7 @@ jobs: needs.changes.outputs.run-cuda-san == 'true' name: "CUDA tests (${{ matrix.sanitizer }})" timeout-minutes: 30 - runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-sanitizer + runs-on: runs-on=${{ github.run_id }}/runner=gpu/extras=s3-cache/tag=cuda-test-sanitizer strategy: fail-fast: false matrix: @@ -171,6 +173,7 @@ jobs: - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: "true" - name: Build tests run: cargo test --profile ci --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu --no-run - name: "CUDA - ${{ matrix.sanitizer }}" @@ -185,7 +188,7 @@ jobs: needs.changes.outputs.run-cuda-san == 'true' name: "CUDA tests (cudf)" timeout-minutes: 30 - runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-cudf + runs-on: runs-on=${{ github.run_id }}/runner=gpu/extras=s3-cache/tag=cuda-test-cudf steps: - uses: runs-on/action@v2 with: @@ -199,6 +202,7 @@ jobs: - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: "true" - name: Build cudf test library run: cargo build --profile ci --locked -p vortex-test-e2e-cuda --target x86_64-unknown-linux-gnu - name: Download and run cudf-test-harness diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1a0d376f319..cee26f919d6 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -23,7 +23,6 @@ jobs: - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} - enable-sccache: "false" - name: Set up JDK 17 uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5 with: diff --git a/.github/workflows/fuzz-coverage.yml b/.github/workflows/fuzz-coverage.yml index fd6ca8aed1b..88bfb7d01b7 100644 --- a/.github/workflows/fuzz-coverage.yml +++ b/.github/workflows/fuzz-coverage.yml @@ -18,7 +18,7 @@ jobs: timeout-minutes: 120 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=arm64-medium/disk=large', github.run_id) + && format('runs-on={0}/runner=arm64-medium/disk=large/extras=s3-cache', github.run_id) || 'ubuntu-latest' }} steps: - uses: runs-on/action@v2 @@ -33,6 +33,7 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} toolchain: ${{ env.NIGHTLY_TOOLCHAIN }} components: clippy, rustfmt, llvm-tools + enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} - name: Ensure llvm-tools are installed run: | diff --git a/.github/workflows/fuzzer-fix-automation.yml b/.github/workflows/fuzzer-fix-automation.yml index 7b8e7012511..0db67516346 100644 --- a/.github/workflows/fuzzer-fix-automation.yml +++ b/.github/workflows/fuzzer-fix-automation.yml @@ -86,7 +86,6 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} toolchain: ${{ env.NIGHTLY_TOOLCHAIN }} - enable-sccache: "false" - name: Install llvm uses: aminya/setup-cpp@v1 diff --git a/.github/workflows/minimize_fuzz_corpus_workflow.yml b/.github/workflows/minimize_fuzz_corpus_workflow.yml index 00f9ba73b84..b0c2fc58d50 100644 --- a/.github/workflows/minimize_fuzz_corpus_workflow.yml +++ b/.github/workflows/minimize_fuzz_corpus_workflow.yml @@ -36,7 +36,7 @@ jobs: name: "Minimize ${{ inputs.fuzz_target }}" runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=arm64-medium/disk=large/tag={1}-minimize', github.run_id, inputs.fuzz_target) + && format('runs-on={0}/runner=arm64-medium/disk=large/extras=s3-cache/tag={1}-minimize', github.run_id, inputs.fuzz_target) || 'ubuntu-latest' }} timeout-minutes: 240 steps: @@ -57,6 +57,7 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} toolchain: ${{ env.NIGHTLY_TOOLCHAIN }} + enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} - name: Install cargo-fuzz uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 3de9ac5c81d..bded979fdf8 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -39,7 +39,6 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} targets: ${{ matrix.target.target }} - enable-sccache: "false" - name: Install cargo-edit uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 @@ -124,7 +123,6 @@ jobs: - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} - enable-sccache: "false" - run: cargo build --release --package vortex-jni - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: @@ -140,9 +138,9 @@ jobs: matrix: include: - target: x86_64-unknown-linux-gnu - runner: runs-on=${{ github.run_id }}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=prepare-java-linux-amd64 + runner: runs-on=${{ github.run_id }}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=prepare-java-linux-amd64 - target: aarch64-unknown-linux-gnu - runner: runs-on=${{ github.run_id }}/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/tag=prepare-java-linux-arm64 + runner: runs-on=${{ github.run_id }}/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/extras=s3-cache/tag=prepare-java-linux-arm64 runs-on: ${{ matrix.runner }} steps: - uses: runs-on/action@v2 @@ -157,6 +155,8 @@ jobs: distribution: "corretto" java-version: "17" - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 - name: Install cargo-zigbuild uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 diff --git a/.github/workflows/publish-dry-runs.yml b/.github/workflows/publish-dry-runs.yml index 49f539a19e4..77ed1a15526 100644 --- a/.github/workflows/publish-dry-runs.yml +++ b/.github/workflows/publish-dry-runs.yml @@ -35,7 +35,6 @@ jobs: - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} - enable-sccache: "false" - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 - name: Install uv uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 @@ -70,9 +69,9 @@ jobs: matrix: include: - target: x86_64-unknown-linux-gnu - runner: runs-on=${{ github.run_id }}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=check-java-publish-build-amd64 + runner: runs-on=${{ github.run_id }}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=check-java-publish-build-amd64 - target: aarch64-unknown-linux-gnu - runner: runs-on=${{ github.run_id }}/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/tag=check-java-publish-build-arm64 + runner: runs-on=${{ github.run_id }}/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/extras=s3-cache/tag=check-java-publish-build-arm64 runs-on: ${{ matrix.runner }} steps: - uses: runs-on/action@v2 @@ -87,6 +86,8 @@ jobs: distribution: "corretto" java-version: "17" - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 - name: Install cargo-zigbuild uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 @@ -105,7 +106,7 @@ jobs: timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-xsmall/image=ubuntu24-full-x64-pre-v2/tag=rust-publish-dry-run', github.run_id) + && format('runs-on={0}/runner=amd64-xsmall/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-publish-dry-run', github.run_id) || 'ubuntu-latest' }} steps: - uses: runs-on/action@v2 @@ -114,6 +115,8 @@ jobs: sccache: s3 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} - name: Install cargo-edit uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 with: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index aba77190690..fbc76722d1c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -29,7 +29,6 @@ jobs: - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} - enable-sccache: "false" - name: Install cargo-edit uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 with: diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index dcedec293c3..eb933f8a0c8 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -31,11 +31,11 @@ jobs: # cargo-zigbuild with glibc 2.31 pinning, matching the Python wheel # and JNI builds. - target: aarch64-unknown-linux-gnu - runner: runs-on=${{ github.run_id }}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=release-vx-aarch64-linux + runner: runs-on=${{ github.run_id }}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=release-vx-aarch64-linux fallback_runner: ubuntu-24.04 archive: tgz - target: x86_64-unknown-linux-gnu - runner: runs-on=${{ github.run_id }}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=release-vx-amd64-linux + runner: runs-on=${{ github.run_id }}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=release-vx-amd64-linux fallback_runner: ubuntu-24.04 archive: tgz steps: @@ -52,7 +52,7 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} targets: ${{ matrix.target }} - enable-sccache: ${{ contains(matrix.target, 'linux') && 'true' || 'false' }} + enable-sccache: ${{ github.repository == 'vortex-data/vortex' && contains(matrix.target, 'linux') && 'true' || 'false' }} - name: Setup Zig if: contains(matrix.target, 'linux') diff --git a/.github/workflows/run-fuzzer.yml b/.github/workflows/run-fuzzer.yml index e563dbefbee..35effdc6e59 100644 --- a/.github/workflows/run-fuzzer.yml +++ b/.github/workflows/run-fuzzer.yml @@ -64,7 +64,7 @@ jobs: timeout-minutes: 370 # 6 hours 10 minutes runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner={1}/disk=large/tag={2}-fuzz', github.run_id, inputs.runner, inputs.fuzz_name || inputs.fuzz_target) + && format('runs-on={0}/runner={1}/disk=large/extras=s3-cache/tag={2}-fuzz', github.run_id, inputs.runner, inputs.fuzz_name || inputs.fuzz_target) || 'ubuntu-latest' }} outputs: crashes_found: ${{ steps.check.outputs.crashes_found }} @@ -82,6 +82,7 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} toolchain: ${{ env.NIGHTLY_TOOLCHAIN }} + enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} - name: Install llvm uses: aminya/setup-cpp@v1 diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index b8c2d08f64f..0f9fdcb9eb1 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -541,6 +541,7 @@ jobs: - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: ${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && 'true' || 'false' }} - name: Install uv uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 with: diff --git a/.github/workflows/wasm-fuzz.yml b/.github/workflows/wasm-fuzz.yml index d61b3b60598..30a688447f9 100644 --- a/.github/workflows/wasm-fuzz.yml +++ b/.github/workflows/wasm-fuzz.yml @@ -33,7 +33,6 @@ jobs: toolchain: ${{ env.NIGHTLY_TOOLCHAIN }} targets: "wasm32-wasip1" components: "rust-src" - enable-sccache: "false" - name: Build WASM fuzz target run: | diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index d03a2b45d14..7ba67e2b245 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -48,7 +48,6 @@ jobs: - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} - enable-sccache: "false" - name: Install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh working-directory: . @@ -75,7 +74,6 @@ jobs: - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} - enable-sccache: "false" - name: Install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh working-directory: .