diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 6f3de4dca8c8..369d403e27fb 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -150,17 +150,14 @@ jobs: set -e && apt update && apt install -y pkg-config xz-utils && - wget https://nodejs.org/dist/v18.12.0/node-v18.12.0-linux-x64.tar.xz -O node.tar.xz && - tar -xf node.tar.xz && - cd node-* && cp -r ./{bin,include,lib,share} /usr/local/ && cd .. && rustup toolchain install "${RUST_TOOLCHAIN}" && rustup default "${RUST_TOOLCHAIN}" && rustup target add x86_64-unknown-linux-gnu && - npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable && pnpm -v && node -v && + npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" && unset CC_x86_64_unknown_linux_gnu && unset CC && - turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target x86_64-unknown-linux-gnu && - strip packages/next-swc/native/next-swc.*.node && - objdump -T packages/next-swc/native/next-swc.*.node | grep GLIBC_ + cd packages/next-swc && npm run build-native-release -- --target x86_64-unknown-linux-gnu && + strip native/next-swc.*.node && + objdump -T native/next-swc.*.node | grep GLIBC_ - host: - 'self-hosted' @@ -172,16 +169,14 @@ jobs: docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-alpine build: >- set -e && - rm -rfv /usr/local/bin/{node,nodejs,npm,npx,corepack} && - apk update && apk upgrade && - apk add --no-cache libc6-compat pkgconfig nodejs-current npm && - rm -rfv /usr/local/bin/{node,nodejs,npm,npx} && + apk update && + apk add --no-cache libc6-compat pkgconfig && rustup toolchain install "${RUST_TOOLCHAIN}" && rustup default "${RUST_TOOLCHAIN}" && rustup target add x86_64-unknown-linux-musl && - npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable && pnpm -v && node -v && - turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target x86_64-unknown-linux-musl && - strip packages/next-swc/native/next-swc.*.node + npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" && + cd packages/next-swc && npm run build-native-release -- --target x86_64-unknown-linux-musl && + strip native/next-swc.*.node - host: - 'self-hosted' @@ -195,18 +190,15 @@ jobs: set -e && apt update && apt install -y pkg-config xz-utils && - wget https://nodejs.org/dist/v18.12.0/node-v18.12.0-linux-x64.tar.xz -O node.tar.xz && - tar -xf node.tar.xz && - cd node-* && cp -r ./{bin,include,lib,share} /usr/local/ && cd .. && export JEMALLOC_SYS_WITH_LG_PAGE=16 && rustup toolchain install "${RUST_TOOLCHAIN}" && rustup default "${RUST_TOOLCHAIN}" && rustup target add aarch64-unknown-linux-gnu && - npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable && + npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" && export CC_aarch64_unknown_linux_gnu=/usr/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-gcc && - turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target aarch64-unknown-linux-gnu --features plugin,rustls-tls,tracing/release_max_level_info && - llvm-strip -x packages/next-swc/native/next-swc.*.node && - objdump -T packages/next-swc/native/next-swc.*.node | grep GLIBC_ + cd packages/next-swc && npm run build-native-release -- --target aarch64-unknown-linux-gnu --features plugin,rustls-tls,tracing/release_max_level_info && + llvm-strip -x native/next-swc.*.node && + objdump -T native/next-swc.*.node | grep GLIBC_ - host: - 'self-hosted' @@ -218,16 +210,15 @@ jobs: docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-alpine build: >- set -e && - rm -rfv /usr/local/bin/{node,nodejs,npm,npx,corepack} && - apk update && apk upgrade && - apk add --no-cache libc6-compat pkgconfig nodejs-current npm && + apk update && + apk add --no-cache libc6-compat pkgconfig && export JEMALLOC_SYS_WITH_LG_PAGE=16 && - npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable && + npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" && rustup toolchain install "${RUST_TOOLCHAIN}" && rustup default "${RUST_TOOLCHAIN}" && rustup target add aarch64-unknown-linux-musl && - turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target aarch64-unknown-linux-musl && - llvm-strip -x packages/next-swc/native/next-swc.*.node + cd packages/next-swc && npm run build-native-release -- --target aarch64-unknown-linux-musl && + llvm-strip -x native/next-swc.*.node name: stable - ${{ matrix.settings.target }} - node@16 runs-on: ${{ matrix.settings.host }} @@ -269,17 +260,33 @@ jobs: run: ${{ matrix.settings.setup }} if: ${{ matrix.settings.setup }} - - name: Build in docker + # we only need custom caching for docker builds + # as they are on an older Node.js version and have + # issues with turbo caching + - name: pull build cache + if: ${{ matrix.settings.docker }} + run: node ./scripts/pull-turbo-cache.js ${{ matrix.settings.target }} + + - name: check build exists if: ${{ matrix.settings.docker }} + run: if [ -f packages/next-swc/native/next-swc.*.node ]; then echo "BUILD_EXISTS=yes" >> $GITHUB_OUTPUT; else echo "BUILD_EXISTS=no" >> $GITHUB_OUTPUT; fi + id: build-exists + + - name: Build in docker + if: ${{ matrix.settings.docker && steps.build-exists.outputs.BUILD_EXISTS == 'no' }} run: docker run -v "/var/run/docker.sock":"/var/run/docker.sock" -e RUST_TOOLCHAIN -e RUST_BACKTRACE -e NAPI_CLI_VERSION -e CARGO_TERM_COLOR -e CARGO_INCREMENTAL -e CARGO_PROFILE_RELEASE_LTO -e CARGO_REGISTRIES_CRATES_IO_PROTOCOL -e TURBO_API -e TURBO_TEAM -e TURBO_TOKEN -e TURBO_VERSION -e TURBO_REMOTE_ONLY -v ${{ env.HOME }}/.cargo/git:/root/.cargo/git -v ${{ env.HOME }}/.cargo/registry:/root/.cargo/registry -v ${{ github.workspace }}:/build -w /build --entrypoint=bash ${{ matrix.settings.docker }} -c "${{ matrix.settings.build }}" + - name: cache build + if: ${{ matrix.settings.docker && steps.build-exists.outputs.BUILD_EXISTS == 'no' }} + run: pnpm turbo run cache-build-native --force -- ${{ matrix.settings.target }} + - name: 'Build' run: ${{ matrix.settings.build }} if: ${{ !matrix.settings.docker }} - name: 'check build cache status' id: check-did-build - run: if [[ ! -z $(ls .turbo/runs) ]]; then echo "DID_BUILD=yup" >> $GITHUB_OUTPUT; fi + run: if [[ ! -z $(ls packages/next-swc/native) ]]; then echo "DID_BUILD=yup" >> $GITHUB_OUTPUT; fi # Trying to upload metrics for the Turbopack to datadog's CI pipeline execution - name: 'Collect turbopack build metrics' diff --git a/scripts/pull-turbo-cache.js b/scripts/pull-turbo-cache.js new file mode 100644 index 000000000000..45486139db0c --- /dev/null +++ b/scripts/pull-turbo-cache.js @@ -0,0 +1,32 @@ +#!/usr/bin/env node +// @ts-check + +const { execSync } = require('child_process') + +;(async function () { + const target = process.argv[process.argv.length - 1] + + const turboResult = execSync( + `pnpm turbo run cache-build-native --dry=json -- ${target}` + ).toString() + + const turboData = JSON.parse(turboResult) + + const task = turboData.tasks.find((t) => t.command !== '') + + if (!task) { + console.warn(`Failed to find related turbo task`, turboResult) + return + } + + // pull cache if it was available + if (task.cache.local || task.cache.remote) { + const pullResult = execSync( + `pnpm turbo run cache-build-native -- ${target}` + ).toString() + console.log(pullResult) + } else { + console.warn(`No turbo cache was available, continuing...`) + console.warn(task) + } +})()