Move turbo outside of build for docker swc builds

.github/workflows/build_and_deploy.yml

@@ -150,17 +150,14 @@ jobs:
               set -e &&
               apt update &&
               apt install -y pkg-config xz-utils &&
-              wget https://nodejs.org/dist/v18.12.0/node-v18.12.0-linux-x64.tar.xz -O node.tar.xz &&
-              tar -xf node.tar.xz &&
-              cd node-* && cp -r ./{bin,include,lib,share} /usr/local/ && cd .. &&
               rustup toolchain install "${RUST_TOOLCHAIN}" &&
               rustup default "${RUST_TOOLCHAIN}" &&
               rustup target add x86_64-unknown-linux-gnu &&
-              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable && pnpm -v && node -v &&
+              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" &&
               unset CC_x86_64_unknown_linux_gnu && unset CC &&
-              turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target x86_64-unknown-linux-gnu &&
-              strip packages/next-swc/native/next-swc.*.node &&
-              objdump -T packages/next-swc/native/next-swc.*.node | grep GLIBC_
+              cd packages/next-swc && npm run build-native-release -- --target x86_64-unknown-linux-gnu &&
+              strip native/next-swc.*.node &&
+              objdump -T native/next-swc.*.node | grep GLIBC_
 
           - host:
               - 'self-hosted'
@@ -172,16 +169,14 @@ jobs:
             docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-alpine
             build: >-
               set -e &&
-              rm -rfv /usr/local/bin/{node,nodejs,npm,npx,corepack} &&
-              apk update && apk upgrade &&
-              apk add --no-cache libc6-compat pkgconfig nodejs-current npm &&
-              rm -rfv /usr/local/bin/{node,nodejs,npm,npx} &&
+              apk update &&
+              apk add --no-cache libc6-compat pkgconfig &&
               rustup toolchain install "${RUST_TOOLCHAIN}" &&
               rustup default "${RUST_TOOLCHAIN}" &&
               rustup target add x86_64-unknown-linux-musl &&
-              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable && pnpm -v && node -v &&
-              turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target x86_64-unknown-linux-musl &&
-              strip packages/next-swc/native/next-swc.*.node
+              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" &&
+              cd packages/next-swc && npm run build-native-release -- --target x86_64-unknown-linux-musl &&
+              strip native/next-swc.*.node
 
           - host:
               - 'self-hosted'
@@ -195,18 +190,15 @@ jobs:
               set -e &&
               apt update &&
               apt install -y pkg-config xz-utils &&
-              wget https://nodejs.org/dist/v18.12.0/node-v18.12.0-linux-x64.tar.xz -O node.tar.xz &&
-              tar -xf node.tar.xz &&
-              cd node-* && cp -r ./{bin,include,lib,share} /usr/local/ && cd .. &&
               export JEMALLOC_SYS_WITH_LG_PAGE=16 &&
               rustup toolchain install "${RUST_TOOLCHAIN}" &&
               rustup default "${RUST_TOOLCHAIN}" &&
               rustup target add aarch64-unknown-linux-gnu &&
-              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable &&
+              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" &&
               export CC_aarch64_unknown_linux_gnu=/usr/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-gcc &&
-              turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target aarch64-unknown-linux-gnu --features plugin,rustls-tls,tracing/release_max_level_info &&
-              llvm-strip -x packages/next-swc/native/next-swc.*.node &&
-              objdump -T packages/next-swc/native/next-swc.*.node | grep GLIBC_
+              cd packages/next-swc && npm run build-native-release -- --target aarch64-unknown-linux-gnu --features plugin,rustls-tls,tracing/release_max_level_info &&
+              llvm-strip -x native/next-swc.*.node &&
+              objdump -T native/next-swc.*.node | grep GLIBC_
 
           - host:
               - 'self-hosted'
@@ -218,16 +210,15 @@ jobs:
             docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-alpine
             build: >-
               set -e &&
-              rm -rfv /usr/local/bin/{node,nodejs,npm,npx,corepack} &&
-              apk update && apk upgrade &&
-              apk add --no-cache libc6-compat pkgconfig nodejs-current npm &&
+              apk update &&
+              apk add --no-cache libc6-compat pkgconfig &&
               export JEMALLOC_SYS_WITH_LG_PAGE=16 &&
-              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable &&
+              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" &&
               rustup toolchain install "${RUST_TOOLCHAIN}" &&
               rustup default "${RUST_TOOLCHAIN}" &&
               rustup target add aarch64-unknown-linux-musl &&
-              turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target aarch64-unknown-linux-musl &&
-              llvm-strip -x packages/next-swc/native/next-swc.*.node
+              cd packages/next-swc && npm run build-native-release -- --target aarch64-unknown-linux-musl &&
+              llvm-strip -x native/next-swc.*.node
 
     name: stable - ${{ matrix.settings.target }} - node@16
     runs-on: ${{ matrix.settings.host }}
@@ -269,17 +260,33 @@ jobs:
         run: ${{ matrix.settings.setup }}
         if: ${{ matrix.settings.setup }}
 
-      - name: Build in docker
+      # we only need custom caching for docker builds
+      # as they are on an older Node.js version and have
+      # issues with turbo caching
+      - name: pull build cache
+        if: ${{ matrix.settings.docker }}
+        run: node ./scripts/pull-turbo-cache.js ${{ matrix.settings.target }}
+
+      - name: check build exists
         if: ${{ matrix.settings.docker }}
+        run: if [ -f packages/next-swc/native/next-swc.*.node ]; then echo "BUILD_EXISTS=yes" >> $GITHUB_OUTPUT; else echo "BUILD_EXISTS=no" >> $GITHUB_OUTPUT; fi
+        id: build-exists
+
+      - name: Build in docker
+        if: ${{ matrix.settings.docker && steps.build-exists.outputs.BUILD_EXISTS == 'no' }}
         run: docker run -v "/var/run/docker.sock":"/var/run/docker.sock"  -e RUST_TOOLCHAIN -e RUST_BACKTRACE -e NAPI_CLI_VERSION -e CARGO_TERM_COLOR -e CARGO_INCREMENTAL -e CARGO_PROFILE_RELEASE_LTO -e CARGO_REGISTRIES_CRATES_IO_PROTOCOL -e TURBO_API -e TURBO_TEAM -e TURBO_TOKEN -e TURBO_VERSION -e TURBO_REMOTE_ONLY -v ${{ env.HOME }}/.cargo/git:/root/.cargo/git -v ${{ env.HOME }}/.cargo/registry:/root/.cargo/registry -v ${{ github.workspace }}:/build -w /build --entrypoint=bash ${{ matrix.settings.docker }} -c "${{ matrix.settings.build }}"
 
+      - name: cache build
+        if: ${{ matrix.settings.docker && steps.build-exists.outputs.BUILD_EXISTS == 'no' }}
+        run: pnpm turbo run cache-build-native --force -- ${{ matrix.settings.target }}
+
       - name: 'Build'
         run: ${{ matrix.settings.build }}
         if: ${{ !matrix.settings.docker }}
 
       - name: 'check build cache status'
         id: check-did-build
-        run: if [[ ! -z $(ls .turbo/runs) ]]; then echo "DID_BUILD=yup" >> $GITHUB_OUTPUT; fi
+        run: if [[ ! -z $(ls packages/next-swc/native) ]]; then echo "DID_BUILD=yup" >> $GITHUB_OUTPUT; fi
 
       # Trying to upload metrics for the Turbopack to datadog's CI pipeline execution
       - name: 'Collect turbopack build metrics'

scripts/pull-turbo-cache.js

@@ -0,0 +1,32 @@
+#!/usr/bin/env node
+// @ts-check
+
+const { execSync } = require('child_process')
+
+;(async function () {
+  const target = process.argv[process.argv.length - 1]
+
+  const turboResult = execSync(
+    `pnpm turbo run cache-build-native --dry=json -- ${target}`
+  ).toString()
+
+  const turboData = JSON.parse(turboResult)
+
+  const task = turboData.tasks.find((t) => t.command !== '<NONEXISTENT>')
+
+  if (!task) {
+    console.warn(`Failed to find related turbo task`, turboResult)
+    return
+  }
+
+  // pull cache if it was available
+  if (task.cache.local || task.cache.remote) {
+    const pullResult = execSync(
+      `pnpm turbo run cache-build-native -- ${target}`
+    ).toString()
+    console.log(pullResult)
+  } else {
+    console.warn(`No turbo cache was available, continuing...`)
+    console.warn(task)
+  }
+})()