From d14034feeabe35726432d5f10a890130acb453e2 Mon Sep 17 00:00:00 2001 From: Leah Date: Wed, 4 Jan 2023 19:08:45 +0100 Subject: [PATCH] fix github actions rust caching --- .../actions/build-debug-turborepo/action.yml | 28 ++- .github/actions/setup-go/action.yml | 1 + .github/actions/setup-node/action.yml | 14 +- .github/actions/setup-rust/action.yml | 55 +++++ .github/actions/setup-rust/matchers.json | 44 ++++ .../workflows/bench-turbopack-scheduled.yml | 55 ++--- .../bench-turbotrace-against-node-nft.yml | 36 +-- .github/workflows/build_rust.yml | 26 +-- .github/workflows/bump-swc.yml | 20 +- .../dispatch-next-dev-custom-bench.yml | 32 +-- .../dispatch-next-dev-matrix-bench.yml | 36 +-- .github/workflows/merge-oss.yml | 20 -- .github/workflows/publish-docs-rs.yml | 11 +- .github/workflows/publish-npm.yml | 15 +- .github/workflows/test.yml | 209 +++++------------- 15 files changed, 292 insertions(+), 310 deletions(-) create mode 100644 .github/actions/setup-rust/action.yml create mode 100644 .github/actions/setup-rust/matchers.json delete mode 100644 .github/workflows/merge-oss.yml diff --git a/.github/actions/build-debug-turborepo/action.yml b/.github/actions/build-debug-turborepo/action.yml index 3785b70fb53df..309cfd45bfcdc 100644 --- a/.github/actions/build-debug-turborepo/action.yml +++ b/.github/actions/build-debug-turborepo/action.yml @@ -12,32 +12,36 @@ runs: steps: - name: "Setup Node" uses: ./.github/actions/setup-node + - name: "Setup Go" uses: ./.github/actions/setup-go with: github-token: ${{ inputs.github-token }} - - name: "Setup Rust toolchain" - uses: actions-rs/toolchain@v1 - if: ${{ inputs.target != 'windows' }} + + - name: "Setup Rust" + uses: ./.github/actions/setup-rust + with: + shared-cache-key: turborepo-debug-build + save-cache: true + - name: "Set Windows default host to MingW" if: ${{ inputs.target == 'windows' }} shell: bash - run: rustup set default-host x86_64-pc-windows-gnu && rustup show - - name: "Setup Rust Cache" - uses: Swatinem/rust-cache@v2 - with: - key: debug-${{ inputs.target }} + run: | + rustup set default-host x86_64-pc-windows-gnu && rustup show + - name: Build Turborepo + working-directory: ./cli shell: bash run: | - cd cli make turbo - cd .. + - name: Strip Turborepo binary + if: ${{ inputs.target == 'windows' }} shell: bash run: strip target/debug/turbo.exe - if: ${{ inputs.target == 'windows' }} + - name: Strip Turborepo binary + if: ${{ inputs.target != 'windows' }} shell: bash run: strip target/debug/turbo - if: ${{ inputs.target != 'windows' }} diff --git a/.github/actions/setup-go/action.yml b/.github/actions/setup-go/action.yml index 33f62e2d8dd99..bb9a90001d9dd 100644 --- a/.github/actions/setup-go/action.yml +++ b/.github/actions/setup-go/action.yml @@ -4,6 +4,7 @@ inputs: github-token: description: "GitHub token. You can pass secrets.GITHUB_TOKEN" required: true + runs: using: "composite" steps: diff --git a/.github/actions/setup-node/action.yml b/.github/actions/setup-node/action.yml index 14ef8d62e46f2..b1c8aebb2f814 100644 --- a/.github/actions/setup-node/action.yml +++ b/.github/actions/setup-node/action.yml @@ -4,7 +4,7 @@ inputs: enable-corepack: description: "Control turning on corepack." required: false - default: true + default: "true" extra-flags: description: "Extra flags to pass to the pnpm install." required: false @@ -12,7 +12,8 @@ inputs: package-install: description: "Don't run the install step." required: false - default: true + default: "true" + runs: using: "composite" steps: @@ -28,21 +29,16 @@ runs: cache: pnpm - name: Configure corepack + if: ${{ inputs.enable-corepack == 'true' }} + shell: bash # Forcibly upgrade our available version of corepack. # The bundled version in node 16 has known issues. # Prepends the npm bin dir so that it is always first. - shell: bash run: | npm install --force --global corepack@latest npm config get prefix >> $GITHUB_PATH corepack enable - - name: Enable corepack - if: ${{ inputs.enable-corepack == 'true' }} - shell: bash - run: | - corepack enable - - name: pnpm install id: install if: ${{ inputs.package-install == 'true' }} diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml new file mode 100644 index 0000000000000..3d1d10acc3c4a --- /dev/null +++ b/.github/actions/setup-rust/action.yml @@ -0,0 +1,55 @@ +name: "Turbo Rust Setup" +description: "Sets up the Rust toolchain for CI" +inputs: + target: + description: "Compilation target" + required: true + + targets: + description: "Comma-separated list of target triples to install for this toolchain" + required: false + components: + description: "Comma-separated list of components to be additionally installed" + required: false + + shared-cache-key: + description: "A cache key that is used instead of the automatic `job`-based key, and is stable over multiple jobs." + required: false + cache-key: + description: "An additional cache key that is added alongside the automatic `job`-based cache key and can be used to further differentiate jobs." + required: false + save-cache: + description: "Determiners whether the cache should be saved. If `false`, the cache is only restored." + required: false + default: "false" + +runs: + using: "composite" + steps: + - name: "Get toolchain version from file" + id: file + shell: bash + run: echo "toolchain=$(cat ./rust-toolchain)" >> $GITHUB_OUTPUT + + - name: "Setup Rust toolchain" + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ steps.file.outputs.toolchain }} + targets: ${{ inputs.targets }} + components: ${{ inputs.components }} + + - name: "Set Windows default host to MingW" + if: ${{ inputs.target == 'windows' }} + shell: bash + run: rustup set default-host x86_64-pc-windows-gnu && rustup show + + - name: "Setup Rust Cache" + uses: Swatinem/rust-cache@v2 + with: + shared-key: ${{ inputs.shared-cache-key }} + key: ${{ inputs.cache-key }} + save-if: ${{ github.ref == 'refs/heads/main' && inputs.save-cache || "false" }} + + - name: "Add problem matcher" + shell: bash + run: echo "::add-matcher::.github/actions/setup-rust/matchers.json" diff --git a/.github/actions/setup-rust/matchers.json b/.github/actions/setup-rust/matchers.json new file mode 100644 index 0000000000000..5e5b9db60ed13 --- /dev/null +++ b/.github/actions/setup-rust/matchers.json @@ -0,0 +1,44 @@ +{ + "problemMatcher": [ + { + "owner": "cargo-common", + "pattern": [ + { + "regexp": "^(warning|warn|error)(:?\\[(\\S*)\\])?: (.*)$", + "severity": 1, + "code": 2, + "message": 3 + }, + { + "regexp": "^\\s+-->\\s(\\S+):(\\d+):(\\d+)$", + "file": 1, + "line": 2, + "column": 3 + } + ] + }, + { + "owner": "cargo-test", + "pattern": [ + { + "regexp": "^.*panicked\\s+at\\s+'(.*)',\\s+(.*):(\\d+):(\\d+)$", + "message": 1, + "file": 2, + "line": 3, + "column": 4 + } + ] + }, + { + "owner": "cargo-fmt", + "pattern": [ + { + "regexp": "^(Diff in (\\S+)) at line (\\d+):", + "message": 1, + "file": 2, + "line": 3 + } + ] + } + ] +} diff --git a/.github/workflows/bench-turbopack-scheduled.yml b/.github/workflows/bench-turbopack-scheduled.yml index 0e5f3bfd2850f..fc5a8c1d64b16 100644 --- a/.github/workflows/bench-turbopack-scheduled.yml +++ b/.github/workflows/bench-turbopack-scheduled.yml @@ -24,42 +24,48 @@ jobs: # This measures Next.js 13 - name: next 13 cache_key: next-dev - args: -p next-dev -- "(Turbopack SSR/1000|Next.js 13)" + cargo_args: -p next-dev + run_args: '"(Turbopack SSR/1000|Next.js 13)"' TURBOPACK_BENCH_COUNTS: 100,500,1000 TURBOPACK_BENCH_BUNDLERS: all # This measures Next.js 12 - name: next 12 cache_key: next-dev - args: -p next-dev -- "(Turbopack SSR/1000|Next.js 12)" + cargo_args: -p next-dev + run_args: '"(Turbopack SSR/1000|Next.js 12)"' TURBOPACK_BENCH_COUNTS: 100,500,1000 TURBOPACK_BENCH_BUNDLERS: all # This measures Next.js 11 - name: next 11 cache_key: next-dev - args: -p next-dev -- "(Turbopack SSR/1000|Next.js 11)" + cargo_args: -p next-dev + run_args: '"(Turbopack SSR/1000|Next.js 11)"' TURBOPACK_BENCH_COUNTS: 100,500,1000 TURBOPACK_BENCH_BUNDLERS: all # This measures Vite - name: vite cache_key: next-dev - args: -p next-dev -- "(Turbopack CSR/1000|Vite)" + cargo_args: -p next-dev + run_args: '"(Turbopack CSR/1000|Vite)"' TURBOPACK_BENCH_COUNTS: 100,500,1000 TURBOPACK_BENCH_BUNDLERS: all # This measures Parcel - name: parcel cache_key: next-dev - args: -p next-dev -- "(Turbopack CSR/1000|Parcel)" + cargo_args: -p next-dev + run_args: '"(Turbopack CSR/1000|Parcel)"' TURBOPACK_BENCH_COUNTS: 100,500,1000 TURBOPACK_BENCH_BUNDLERS: all # This measures Webpack - name: webpack cache_key: next-dev - args: -p next-dev -- "(Turbopack CSR/1000|Webpack)" + cargo_args: -p next-dev + run_args: '"(Turbopack CSR/1000|Webpack)"' TURBOPACK_BENCH_COUNTS: 100,500,1000 TURBOPACK_BENCH_BUNDLERS: all @@ -71,29 +77,28 @@ jobs: - uses: ./.github/actions/setup-node - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust with: - key: benchmark-${{ matrix.bench.cache_key }} + shared-key: benchmark-${{ matrix.bench.cache_key }} - - name: Clear benchmarks + - name: Clear potentially cached benchmarks run: rm -rf target/criterion + - name: Build benchmarks + timeout-minutes: 120 + run: cargo bench --no-run ${{ matrix.bench.cargo_args }} + - name: Run cargo bench - uses: actions-rs/cargo@v1 timeout-minutes: 180 env: TURBOPACK_BENCH_BUNDLERS: ${{ matrix.bench.TURBOPACK_BENCH_BUNDLERS }} TURBOPACK_BENCH_COUNTS: ${{ matrix.bench.TURBOPACK_BENCH_COUNTS }} - with: - command: bench - args: ${{ matrix.bench.args }} + run: cargo bench ${{ matrix.bench.cargo_args }} -- ${{ matrix.bench.run_args }} - name: Install critcmp - if: always() - uses: actions-rs/cargo@v1 + uses: baptiste0928/cargo-install@v1 with: - command: install - args: critcmp + crate: critcmp - name: Compare results if: always() @@ -110,10 +115,6 @@ jobs: name: ${{ matrix.bench.name }} path: raw.json - # This avoids putting this data into the rust-cache - - name: Clear benchmarks - run: rm -rf target/criterion - commit_results: name: Commit benchmark-data needs: bench @@ -122,20 +123,24 @@ jobs: - name: Get current date id: date run: | - echo "::set-output name=year::$(date +'%Y')" - echo "::set-output name=month::$(date +'%m')" - echo "::set-output name=date::$(date +'%s')" - echo "::set-output name=pretty::$(date +'%Y-%m-%d %H:%M')" + echo "year=$(date +'%Y')" >> $GITHUB_OUTPUT + echo "month=$(date +'%m')" >> $GITHUB_OUTPUT + echo "date=$(date +'%s')" >> $GITHUB_OUTPUT + echo "pretty=$(date +'%Y-%m-%d %H:%M')" >> $GITHUB_OUTPUT + - name: Checkout benchmark-data uses: actions/checkout@v3 with: ref: benchmark-data + - name: Download benchmark data uses: actions/download-artifact@v3 with: path: data/${{ steps.date.outputs.year }}/${{ steps.date.outputs.month }}/ubuntu-latest-8-core/${{ steps.date.outputs.date }}-${{ github.sha }} + - name: Git pull run: git pull --depth=1 --no-tags origin benchmark-data + - name: Push data to branch uses: stefanzweifel/git-auto-commit-action@v4 with: diff --git a/.github/workflows/bench-turbotrace-against-node-nft.yml b/.github/workflows/bench-turbotrace-against-node-nft.yml index dd013a0f04126..26f3a978e9e68 100644 --- a/.github/workflows/bench-turbotrace-against-node-nft.yml +++ b/.github/workflows/bench-turbotrace-against-node-nft.yml @@ -26,37 +26,37 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - with: - key: bench_nft - uses: ./.github/actions/setup-node with: package-install: false - - name: Install dependencies - working-directory: crates/turbopack/tests/node-file-trace - run: | - echo 'node-linker = "hoisted"' > .npmrc - pnpm install -r --side-effects-cache false + - uses: ./.github/actions/setup-rust - uses: ./.github/actions/setup-go with: github-token: "${{ secrets.GITHUB_TOKEN }}" + - name: Install hoisted npm dependencies + working-directory: crates/turbopack/tests/node-file-trace + run: | + echo 'node-linker = "hoisted"' > .npmrc + pnpm install -r --side-effects-cache false + - name: Install nextest - run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin - shell: bash + uses: taiki-e/install-action@nextest + + - name: Build benchmark + timeout-minutes: 60 + run: | + cargo nextest run --no-run --release \ + -p turbopack --features bench_against_node_nft bench_against_node_nft - name: Run benchmark - uses: actions-rs/cargo@v1 timeout-minutes: 60 - with: - command: nextest - args: run -j 1 --no-fail-fast -r -p turbopack --features bench_against_node_nft bench_against_node_nft + run: | + cargo nextest run --test-threads 1 --no-fail-fast --release \ + -p turbopack --features bench_against_node_nft bench_against_node_nft - name: Show benchmark result - uses: actions-rs/cargo@v1 - with: - command: xtask - args: nft-bench-result + run: cargo xtask nft-bench-result diff --git a/.github/workflows/build_rust.yml b/.github/workflows/build_rust.yml index f1404879f55f1..cbc4644d7ea9b 100644 --- a/.github/workflows/build_rust.yml +++ b/.github/workflows/build_rust.yml @@ -37,7 +37,7 @@ jobs: container-setup: "apt-get update && apt-get install -y curl" target: "aarch64-unknown-linux-gnu" lib-cache-key: turbo-lib-cross-${{ inputs.release_branch }} - rustflags: 'RUSTFLAGS="-C linker=aarch64-linux-gnu-gcc"' + rustflags: "-C linker=aarch64-linux-gnu-gcc" setup: "apt-get install -y build-essential clang-5.0 lldb-5.0 llvm-5.0-dev libclang-5.0-dev gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu" - host: windows-latest target: x86_64-pc-windows-gnu @@ -57,24 +57,9 @@ jobs: if: ${{ matrix.settings.container-setup }} run: ${{ matrix.settings.container-setup }} - - name: Install - uses: actions-rs/toolchain@v1 + - uses: ./.github/actions/setup-rust with: - profile: minimal - override: true - target: ${{ matrix.settings.target }} - - # - name: Cache cargo registry - # uses: actions/cache@v3 - # with: - # path: ~/.cargo/registry - # key: ${{ matrix.settings.target }}-cargo-registry - - # - name: Cache cargo index - # uses: actions/cache@v3 - # with: - # path: ~/.cargo/git - # key: ${{ matrix.settings.target }}-cargo-index + targets: ${{ matrix.settings.target }} - name: Download artifact id: download-artifact @@ -111,13 +96,16 @@ jobs: # with: # name: ${{ matrix.settings.lib-cache-key }} # path: cli/libturbo + - name: Build Setup shell: bash if: ${{ matrix.settings.setup }} run: ${{ matrix.settings.setup }} - name: Build - run: ${{ matrix.settings.rustflags }} cargo build --release -p turbo --target ${{ matrix.settings.target }} + env: + RUSTFLAGS: ${{ matrix.settings.rustflags }} + run: cargo build --release -p turbo --target ${{ matrix.settings.target }} - name: Upload Artifacts uses: actions/upload-artifact@v3 diff --git a/.github/workflows/bump-swc.yml b/.github/workflows/bump-swc.yml index a7fab43fd361a..b1663f6520d04 100644 --- a/.github/workflows/bump-swc.yml +++ b/.github/workflows/bump-swc.yml @@ -15,22 +15,22 @@ jobs: with: submodules: true - - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - profile: minimal override: true - - uses: Swatinem/rust-cache@v2 + - name: Install cargo-edit + uses: baptiste0928/cargo-install@v1 + with: + crate: cargo-edit + version: "0.11.6" + + - name: Install cargo-outdated + uses: baptiste0928/cargo-install@v1 with: - shared-key: "gha-cargo-upgrade" - cache-on-failure: true + crate: cargo-outdated + version: "0.11.1" - name: Run cargo upgrade uses: kwonoj/gha-cargo-upgrade@latest diff --git a/.github/workflows/dispatch-next-dev-custom-bench.yml b/.github/workflows/dispatch-next-dev-custom-bench.yml index cc0cbfd38d373..61413d1b3e7e0 100644 --- a/.github/workflows/dispatch-next-dev-custom-bench.yml +++ b/.github/workflows/dispatch-next-dev-custom-bench.yml @@ -27,29 +27,29 @@ jobs: - uses: ./.github/actions/setup-node - - uses: Swatinem/rust-cache@v2 + - uses: ./.github/actions/setup-rust with: - key: benchmark-next-dev + shared-cache-key: benchmark-next-dev + save-cache: "false" - name: Clear benchmarks run: rm -rf target/criterion + - name: Build benchmarks + timeout-minutes: 120 + run: cargo bench --no-run -p next-dev + - name: Run cargo bench - uses: actions-rs/cargo@v1 timeout-minutes: 120 env: TURBOPACK_BENCH_BUNDLERS: all TURBOPACK_BENCH_COUNTS: ${{ github.event.inputs.counts }} - with: - command: bench - args: -p next-dev -- "${{ github.event.inputs.filter }}" + run: cargo bench -p next-dev -- "${{ github.event.inputs.filter }}" - name: Install critcmp - if: always() - uses: actions-rs/cargo@v1 + uses: baptiste0928/cargo-install@v1 with: - command: install - args: critcmp + crate: critcmp - name: Compare results if: always() @@ -79,20 +79,24 @@ jobs: - name: Get current date id: date run: | - echo "::set-output name=year::$(date +'%Y')" - echo "::set-output name=month::$(date +'%m')" - echo "::set-output name=date::$(date +'%s')" - echo "::set-output name=pretty::$(date +'%Y-%m-%d %H:%M')" + echo "year=$(date +'%Y')" >> $GITHUB_OUTPUT + echo "month=$(date +'%m')" >> $GITHUB_OUTPUT + echo "date=$(date +'%s')" >> $GITHUB_OUTPUT + echo "pretty=$(date +'%Y-%m-%d %H:%M')" >> $GITHUB_OUTPUT + - name: Checkout benchmark-data uses: actions/checkout@v3 with: ref: benchmark-data + - name: Download benchmark data uses: actions/download-artifact@v3 with: path: data/${{ steps.date.outputs.year }}/${{ steps.date.outputs.month }}/ubuntu-latest/${{ steps.date.outputs.date }}-${{ github.sha }} + - name: Git pull run: git pull --depth=1 --no-tags origin benchmark-data + - name: Push data to branch uses: stefanzweifel/git-auto-commit-action@v4 with: diff --git a/.github/workflows/dispatch-next-dev-matrix-bench.yml b/.github/workflows/dispatch-next-dev-matrix-bench.yml index be7d2529b73ee..a510f3153739f 100644 --- a/.github/workflows/dispatch-next-dev-matrix-bench.yml +++ b/.github/workflows/dispatch-next-dev-matrix-bench.yml @@ -41,31 +41,35 @@ jobs: - uses: actions/checkout@v3 - uses: ./.github/actions/setup-node + - uses: ./.github/actions/setup-rust + with: + shared-cache-key: benchmark-next-dev - uses: Swatinem/rust-cache@v2 with: - key: benchmark-next-dev + shared-key: benchmark-next-dev + save-if: "false" - name: Clear benchmarks run: rm -rf target/criterion + - name: Build benchmarks + timeout-minutes: 120 + run: cargo bench --no-run -p next-dev + - name: Run cargo bench - uses: actions-rs/cargo@v1 timeout-minutes: 120 env: TURBOPACK_BENCH_BUNDLERS: all TURBOPACK_BENCH_COUNTS: 1000,${{ github.event.inputs.count }} - with: - command: bench - # Turbopack with 1000 modules is benchmarked in every run - # to create a baseline result for normalization (should the runners performance vary between jobs) - args: -p next-dev -- "(Turbopack ${{ matrix.bench.type }}/1000|${{ matrix.bench.name }} ${{ matrix.bench.type }}/${{ github.event.inputs.count }})" + # Turbopack with 1000 modules is benchmarked in every run + # to create a baseline result for normalization (should the runners performance vary between jobs) + run: cargo bench -p next-dev -- "(Turbopack ${{ matrix.bench.type }}/1000|${{ matrix.bench.name }} ${{ matrix.bench.type }}/${{ github.event.inputs.count }})" - name: Install critcmp - if: always() - uses: actions-rs/cargo@v1 + uses: baptiste0928/cargo-install@v1 with: - command: install + crate: critcmp args: critcmp - name: Compare results @@ -96,20 +100,24 @@ jobs: - name: Get current date id: date run: | - echo "::set-output name=year::$(date +'%Y')" - echo "::set-output name=month::$(date +'%m')" - echo "::set-output name=date::$(date +'%s')" - echo "::set-output name=pretty::$(date +'%Y-%m-%d %H:%M')" + echo "year=$(date +'%Y')" >> $GITHUB_OUTPUT + echo "month=$(date +'%m')" >> $GITHUB_OUTPUT + echo "date=$(date +'%s')" >> $GITHUB_OUTPUT + echo "pretty=$(date +'%Y-%m-%d %H:%M')" >> $GITHUB_OUTPUT + - name: Checkout benchmark-data uses: actions/checkout@v3 with: ref: benchmark-data + - name: Download benchmark data uses: actions/download-artifact@v3 with: path: data/${{ steps.date.outputs.year }}/${{ steps.date.outputs.month }}/ubuntu-latest-8-core/${{ steps.date.outputs.date }}-${{ github.sha }} + - name: Git pull run: git pull --depth=1 --no-tags origin benchmark-data + - name: Push data to branch uses: stefanzweifel/git-auto-commit-action@v4 with: diff --git a/.github/workflows/merge-oss.yml b/.github/workflows/merge-oss.yml deleted file mode 100644 index 8ef93cb828c43..0000000000000 --- a/.github/workflows/merge-oss.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Scheduled Merge OSS `main` Action -on: - schedule: - - cron: "15 * * * *" - -jobs: - merge-upstream-to-main: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: main # set the branch to merge to - fetch-depth: 0 - - name: Merge Upstream - uses: exions/merge-upstream@v1 - with: - upstream: vercel/turbo # set the upstream repo - upstream-branch: main # set the upstream branch to merge from - branch: main # set the branch to merge to diff --git a/.github/workflows/publish-docs-rs.yml b/.github/workflows/publish-docs-rs.yml index 793f35f33952a..19bb86b9fc317 100644 --- a/.github/workflows/publish-docs-rs.yml +++ b/.github/workflows/publish-docs-rs.yml @@ -10,13 +10,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - uses: actions-rs/cargo@v1 - with: - command: doc - args: --no-deps + - uses: ./.github/actions/setup-rust + + - run: cargo doc --no-deps + - uses: actions/upload-artifact@v3 with: name: docs_rs diff --git a/.github/workflows/publish-npm.yml b/.github/workflows/publish-npm.yml index a050342e83941..3868d8c00de51 100644 --- a/.github/workflows/publish-npm.yml +++ b/.github/workflows/publish-npm.yml @@ -13,15 +13,7 @@ jobs: - uses: actions/checkout@v3 - uses: ./.github/actions/setup-node - - name: Cache cargo registry - uses: actions/cache@v3 - # Don't care the Cargo.lock, just cache it - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: publish-npm-cargo-cache + - uses: ./.github/actions/setup-rust - name: Build packages run: pnpx turbo@canary run build:ts @@ -30,7 +22,4 @@ jobs: run: echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN_ELEVATED }}" > ~/.npmrc - name: Publish - uses: actions-rs/cargo@v1 - with: - command: xtask - args: workspace --publish + run: cargo xtask workspace --publish diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 09eb778f70b25..a7289d8923195 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -288,34 +288,8 @@ jobs: TURBO_REMOTE_ONLY: true run: pnpm -- turbo run run-example -- "${{ matrix.example }}" "${{ matrix.manager }}" - rust_prepare: - name: Check rust crates - runs-on: ubuntu-latest-16-core-oss - needs: determine_jobs - if: needs.determine_jobs.outputs.rust == 'true' - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Rust - uses: actions-rs/toolchain@v1 - - - name: Setup Go - uses: ./.github/actions/setup-go - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - - - name: Run cargo check - uses: actions-rs/cargo@v1 - with: - command: check - args: --workspace --all-targets - rust_lint: - needs: [determine_jobs, rust_prepare] + needs: determine_jobs if: needs.determine_jobs.outputs.rust == 'true' name: Rust linting runs-on: ubuntu-latest-16-core-oss @@ -324,24 +298,16 @@ jobs: uses: actions/checkout@v3 - name: Setup Rust - uses: actions-rs/toolchain@v1 + uses: ./.github/actions/setup-rust with: - target: wasm32-unknown-unknown + components: clippy,rustfmt + save-cache: true - name: Setup Go uses: ./.github/actions/setup-go with: github-token: "${{ secrets.GITHUB_TOKEN }}" - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - - - name: Install cargo clippy - run: rustup component add clippy - - - name: Install cargo rustfmt - run: rustup component add rustfmt - - name: Run cargo feature check for next-binding run: | # Known set of feature combinations we need to support @@ -354,16 +320,12 @@ jobs: cargo check -p next-binding --features __swc_core_binding_wasm,__swc_core_binding_wasm_plugin,__feature_mdx_rs --target wasm32-unknown-unknown - name: Run cargo clippy - uses: actions-rs/cargo@v1 - with: - command: clippy - args: --workspace --all-targets + run: | + cargo clippy --workspace --all-targets - name: Run cargo fmt check - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --check + run: | + cargo fmt --check - name: Count clippy warnings run: | @@ -425,13 +387,11 @@ jobs: command: check licenses - name: Run cargo check release - uses: actions-rs/cargo@v1 - with: - command: check - args: --workspace --all-targets --release + run: | + cargo check --workspace --all-targets --release rust_test: - needs: [determine_jobs, rust_prepare] + needs: determine_jobs if: needs.determine_jobs.outputs.rust == 'true' strategy: fail-fast: false @@ -459,24 +419,13 @@ jobs: uses: actions/checkout@v3 - name: Setup Rust - uses: actions-rs/toolchain@v1 + uses: ./.github/actions/setup-rust + with: + save-cache: true - name: Setup Node.js uses: ./.github/actions/setup-node - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - - - name: Start Redis - uses: supercharge/redis-github-action@1.4.0 - with: - redis-version: 6 - if: matrix.os.name == 'ubuntu' - - - name: Set Redis URL - run: echo "BULL_REDIS_CONNECTION=redis://127.0.0.1:6379" >> $GITHUB_ENV - if: matrix.os.name == 'ubuntu' - - name: Prepare toolchain on Windows run: | pnpx node-gyp install @@ -492,26 +441,16 @@ jobs: run: npm install - name: Install nextest - id: nextest-install - continue-on-error: true - run: curl -LsSf https://get.nexte.st/latest/${{ matrix.os.nextest }} | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin - shell: bash - - - name: Install nextest (second try) - if: steps.nextest-install.outcome == 'failure' - run: curl -LsSf https://get.nexte.st/latest/${{ matrix.os.nextest }} | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin - shell: bash + uses: taiki-e/install-action@nextest - name: Run nextest - uses: actions-rs/cargo@v1 timeout-minutes: 120 - with: - command: nextest + run: | # We exclude turbo as it requires linking Go and all logic resides in turborepo-lib - args: run --workspace --release --no-fail-fast --exclude turbo + cargo nextest run --workspace --release --no-fail-fast --exclude turbo rust_test_bench: - needs: [determine_jobs, rust_prepare] + needs: determine_jobs if: needs.determine_jobs.outputs.rust == 'true' strategy: fail-fast: false @@ -536,11 +475,14 @@ jobs: git config --global core.autocrlf false git config --global core.eol lf if: matrix.os.name == 'windows' + - name: Checkout uses: actions/checkout@v3 - name: Setup Rust - uses: actions-rs/toolchain@v1 + uses: ./.github/actions/setup-rust + with: + save-cache: true - name: Setup Go uses: ./.github/actions/setup-go @@ -552,33 +494,26 @@ jobs: with: node-version: 16 - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - - name: Run cargo test on benchmarks - uses: actions-rs/cargo@v1 timeout-minutes: 120 env: TURBOPACK_BENCH_COUNTS: "100" TURBOPACK_BENCH_PROGRESS: "1" - with: - command: test - args: --benches --release + run: | + cargo test --benches --release - name: Run cargo test on benchmarks for other bundlers - uses: actions-rs/cargo@v1 if: needs.determine_jobs.outputs.rust_bench == 'true' timeout-minutes: 120 env: TURBOPACK_BENCH_COUNTS: "100" TURBOPACK_BENCH_PROGRESS: "1" TURBOPACK_BENCH_BUNDLERS: "others" - with: - command: test - args: --benches --release + run: | + cargo test --benches --release rust_build_release: - needs: [determine_jobs, rust_prepare] + needs: determine_jobs if: needs.determine_jobs.outputs.rust == 'true' && needs.determine_jobs.outputs.push == 'true' strategy: fail-fast: false @@ -599,11 +534,12 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Install target - uses: actions-rs/toolchain@v1 + - name: Setup Rust + uses: ./.github/actions/setup-rust with: - target: ${{ matrix.os.target }} - default: true + targets: ${{ matrix.os.target }} + cache-key: ${{ matrix.os.target }} + save-cache: true - name: Install musl tools run: | @@ -620,38 +556,25 @@ jobs: rustup component add llvm-tools-preview if: matrix.os.target == 'x86_64-pc-windows-msvc' - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - with: - key: release-${{ matrix.os.target }} - - name: Build node-file-trace - uses: actions-rs/cargo@v1 - with: - command: build - args: --release -p node-file-trace --target ${{ matrix.os.target }} if: matrix.os.target != 'x86_64-pc-windows-msvc' + run: | + cargo build --release -p node-file-trace --target ${{ matrix.os.target }} - name: Build node-file-trace - uses: actions-rs/cargo@v1 - with: - command: xwin - args: build --release -p node-file-trace -p next-dev --target ${{ matrix.os.target }} if: matrix.os.target == 'x86_64-pc-windows-msvc' + run: | + cargo xwin build --release -p node-file-trace --target ${{ matrix.os.target }} - name: Build next-dev (native-tls) - uses: actions-rs/cargo@v1 - with: - command: build - args: --release -p next-dev --target ${{ matrix.os.target }} if: matrix.os.target != 'x86_64-pc-windows-msvc' && matrix.os.target != 'x86_64-unknown-linux-musl' + run: | + cargo build --release -p next-dev --target ${{ matrix.os.target }} - name: Build next-dev (rustls-tls) - uses: actions-rs/cargo@v1 - with: - command: build - args: --release -p next-dev --target ${{ matrix.os.target }} --no-default-features --features cli,custom_allocator,rustls-tls if: matrix.os.target == 'x86_64-unknown-linux-musl' + run: | + cargo build --release -p next-dev --target ${{ matrix.os.target }} --no-default-features --features cli,custom_allocator,rustls-tls - uses: actions/upload-artifact@v3 with: @@ -663,7 +586,7 @@ jobs: target/${{ matrix.os.target }}/release/next-dev.exe rust_bench_pr: - needs: [determine_jobs, rust_prepare] + needs: determine_jobs if: needs.determine_jobs.outputs.rust == 'true' && github.event_name == 'pull_request' name: Benchmark and compare Turbopack performance runs-on: ubuntu-latest-16-core-oss @@ -686,8 +609,10 @@ jobs: - name: Setup Node.js uses: ./.github/actions/setup-node - - name: Rust Cache - uses: Swatinem/rust-cache@v2 + - name: Setup Rust + uses: ./.github/actions/setup-rust + with: + shared-cache-key: benchmark-${{ matrix.bench.cache_key }} - name: Benchmark and compare with base branch uses: sokra/criterion-compare-action@main @@ -697,7 +622,7 @@ jobs: cwd: crates/next-dev rust_bench: - needs: [determine_jobs, rust_prepare] + needs: determine_jobs if: (needs.determine_jobs.outputs.rust == 'true' && needs.determine_jobs.outputs.push == 'true') || needs.determine_jobs.outputs.rust_bench == 'true' strategy: fail-fast: false @@ -744,38 +669,31 @@ jobs: - name: Setup Node.js uses: ./.github/actions/setup-node - - name: Rust Cache - uses: Swatinem/rust-cache@v2 + - name: Setup Rust + uses: ./.github/actions/setup-rust with: - key: benchmark-${{ matrix.bench.cache_key }} + shared-cache-key: benchmark-${{ matrix.bench.cache_key }} + save-cache: true - name: Clear benchmarks run: rm -rf target/criterion - name: Compile cargo benches - uses: actions-rs/cargo@v1 env: TURBOPACK_BENCH_COUNTS: ${{ matrix.bench.TURBOPACK_BENCH_COUNTS }} - with: - command: bench - args: --no-run ${{ matrix.bench.args }} + run: cargo bench --no-run ${{ matrix.bench.args }} - name: Run cargo bench - uses: actions-rs/cargo@v1 timeout-minutes: 180 env: TURBOPACK_BENCH_COUNTS: ${{ matrix.bench.TURBOPACK_BENCH_COUNTS }} TURBOPACK_BENCH_PROGRESS: 1 - with: - command: bench - args: ${{ matrix.bench.args }} + run: cargo bench ${{ matrix.bench.args }} - name: Install critcmp - if: always() - uses: actions-rs/cargo@v1 + uses: baptiste0928/cargo-install@v1 with: - command: install - args: critcmp + crate: critcmp - name: Compare results if: always() @@ -805,10 +723,10 @@ jobs: - name: Get current date id: date run: | - echo "::set-output name=year::$(date +'%Y')" - echo "::set-output name=month::$(date +'%m')" - echo "::set-output name=date::$(date +'%s')" - echo "::set-output name=pretty::$(date +'%Y-%m-%d %H:%M')" + echo "year=$(date +'%Y')" >> $GITHUB_OUTPUT + echo "month=$(date +'%m')" >> $GITHUB_OUTPUT + echo "date=$(date +'%s')" >> $GITHUB_OUTPUT + echo "pretty=$(date +'%Y-%m-%d %H:%M')" >> $GITHUB_OUTPUT - name: Checkout benchmark-data uses: actions/checkout@v3 @@ -836,7 +754,7 @@ jobs: commit_message: Benchmark result for ${{ steps.date.outputs.pretty }} (${{ github.sha }}) next_js_integration: - needs: [determine_jobs, rust_prepare, rust_build_release] + needs: [determine_jobs, rust_build_release] if: needs.determine_jobs.outputs.rust == 'true' name: Next.js integration test runs-on: ubuntu-latest-8-core-oss @@ -947,7 +865,6 @@ jobs: - go_lint - go_unit - go_examples - - rust_prepare - rust_lint - rust_test - rust_test_bench @@ -964,7 +881,7 @@ jobs: steps: - name: Check cancellation id: cancelled - if: needs.rust_prepare.result == 'cancelled' || needs.rust_lint.result == 'cancelled' || needs.rust_test.result == 'cancelled' || needs.rust_test_bench.result == 'cancelled' || needs.rust_build_release.result == 'cancelled' || needs.node_lint.result == 'cancelled' || needs.rust_bench_commit.result == 'cancelled' + if: needs.rust_lint.result == 'cancelled' || needs.rust_test.result == 'cancelled' || needs.rust_test_bench.result == 'cancelled' || needs.rust_build_release.result == 'cancelled' || needs.node_lint.result == 'cancelled' || needs.rust_bench_commit.result == 'cancelled' run: echo "cancelled=true" >> $GITHUB_OUTPUT - name: Start summary @@ -996,12 +913,6 @@ jobs: echo "- Go Examples" >> failures.md exit 1 - - name: Rust Check failed - if: always() && needs.rust_prepare.result != 'success' && needs.rust_prepare.result != 'skipped' - run: | - echo "- Rust checks" >> failures.md - exit 1 - - name: Rust Linting failed if: always() && needs.rust_lint.result != 'success' && needs.rust_lint.result != 'skipped' run: |