Apply publish step optimizations (#43620)

ijjk · web-flow · commit da29109e6abc · 2022-12-01T21:02:23.000-08:00
Follow-up to #32337 this removes the un-necessary step where we fetch all of the tags which requires pulling a lot of un-necessary git history inflating cache size and publish times. The only reason these tags were needing to be fetched is due to an issue in how the `actions/checkout` step works (actions/checkout#882). This reduces the publish times by at least 4 minutes by removing the tags fetching step https://github.com/vercel/next.js/actions/runs/3598569786/jobs/6061449995#step:16:14 As a further optimization this adds concurrency to the `npm publish` calls themselves to hopefully reduce time spent there as well.
diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
@@ -84,11 +84,10 @@ jobs:
       - run: pnpm install
       - run: pnpm run build
       - run: node run-tests.js --timings --write-timings -g 1/1
-      - run: node ./scripts/fetch-tags.mjs ${{ github.sha }}
 
       - id: check-release
         run: |
-          if [[ $(git describe --exact-match 2> /dev/null || :) = v* ]];
+          if [[ $(node ./scripts/check-is-release.js 2> /dev/null || :) = v* ]];
             then
               echo "::set-output name=IS_RELEASE::true"
             else
@@ -893,7 +892,7 @@ jobs:
 
       - run: npm i -g pnpm@${PNPM_VERSION}
       - run: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc
-      - run: ./scripts/publish-native.js $GITHUB_REF
+      - run: ./scripts/publish-native.js
       - run: ./scripts/publish-release.js
 
   testDeployE2E:
diff --git a/scripts/check-is-release.js b/scripts/check-is-release.js
@@ -1,6 +1,6 @@
-import { execSync } from 'child_process'
-import execa from 'execa'
-;(async () => {
+const { execSync } = require('child_process')
+
+const checkIsRelease = async () => {
   let commitId = process.argv[2] || ''
 
   // parse only the last string which should be version if
@@ -14,21 +14,13 @@ import execa from 'execa'
   const versionString = commitMsg.split(' ').pop().trim()
   const publishMsgRegex = /^v\d{1,}\.\d{1,}\.\d{1,}(-\w{1,}\.\d{1,})?$/
 
-  console.log({ commitId, commitMsg, versionString })
-
   if (publishMsgRegex.test(versionString)) {
-    console.log('publish commit, fetching tags')
-
-    const result = await execa(
-      'git',
-      ['fetch', '--depth=1', 'origin', '+refs/tags/*:refs/tags/*'],
-      {
-        stdio: ['ignore', 'inherit', 'inherit'],
-      }
-    )
-
-    process.exit(result.exitCode)
+    console.log(versionString)
+    process.exit(0)
   } else {
-    console.log('not publish commit')
+    console.log('not publish commit', { commitId, commitMsg, versionString })
+    process.exit(1)
   }
-})()
+}
+
+checkIsRelease()
diff --git a/scripts/publish-native.js b/scripts/publish-native.js
@@ -1,112 +1,132 @@
 #!/usr/bin/env node
 
 const path = require('path')
-const { readFile, readdir, writeFile } = require('fs/promises')
+const execa = require('execa')
 const { copy } = require('fs-extra')
-const { execSync } = require('child_process')
+const { Sema } = require('async-sema')
+const { readFile, readdir, writeFile } = require('fs/promises')
 
 const cwd = process.cwd()
 
 ;(async function () {
   try {
+    const publishSema = new Sema(2)
+
     let version = JSON.parse(
       await readFile(path.join(cwd, 'lerna.json'))
     ).version
-    let gitref = process.argv.slice(2)[0]
 
     // Copy binaries to package folders, update version, and publish
     let nativePackagesDir = path.join(cwd, 'packages/next-swc/crates/napi/npm')
     let platforms = (await readdir(nativePackagesDir)).filter(
       (name) => !name.startsWith('.')
     )
 
-    for (let platform of platforms) {
-      try {
-        let binaryName = `next-swc.${platform}.node`
-        await copy(
-          path.join(cwd, 'packages/next-swc/native', binaryName),
-          path.join(nativePackagesDir, platform, binaryName)
-        )
-        let pkg = JSON.parse(
-          await readFile(path.join(nativePackagesDir, platform, 'package.json'))
-        )
-        pkg.version = version
-        await writeFile(
-          path.join(nativePackagesDir, platform, 'package.json'),
-          JSON.stringify(pkg, null, 2)
-        )
-        execSync(
-          `npm publish ${path.join(
-            nativePackagesDir,
-            platform
-          )} --access public ${
-            gitref.includes('canary') ? ' --tag canary' : ''
-          }`
-        )
-      } catch (err) {
-        // don't block publishing other versions on single platform error
-        console.error(`Failed to publish`, platform)
+    await Promise.all(
+      platforms.map(async (platform) => {
+        await publishSema.acquire()
 
-        if (
-          err.message &&
-          err.message.includes(
-            'You cannot publish over the previously published versions'
+        try {
+          let binaryName = `next-swc.${platform}.node`
+          await copy(
+            path.join(cwd, 'packages/next-swc/native', binaryName),
+            path.join(nativePackagesDir, platform, binaryName)
+          )
+          let pkg = JSON.parse(
+            await readFile(
+              path.join(nativePackagesDir, platform, 'package.json')
+            )
+          )
+          pkg.version = version
+          await writeFile(
+            path.join(nativePackagesDir, platform, 'package.json'),
+            JSON.stringify(pkg, null, 2)
           )
-        ) {
-          console.error('Ignoring already published error', platform)
-        } else {
-          // throw err
+          await execa(
+            `npm publish ${path.join(
+              nativePackagesDir,
+              platform
+            )} --access public ${
+              version.includes('canary') ? ' --tag canary' : ''
+            }`,
+            { stdio: 'inherit' }
+          )
+        } catch (err) {
+          // don't block publishing other versions on single platform error
+          console.error(`Failed to publish`, platform)
+
+          if (
+            err.message &&
+            err.message.includes(
+              'You cannot publish over the previously published versions'
+            )
+          ) {
+            console.error('Ignoring already published error', platform)
+          } else {
+            // throw err
+          }
+        } finally {
+          publishSema.release()
         }
-      }
-      // lerna publish in next step sill fail if git status is not clean
-      execSync(
-        `git update-index --skip-worktree ${path.join(
-          nativePackagesDir,
-          platform,
-          'package.json'
-        )}`
-      )
-    }
+        // lerna publish in next step sill fail if git status is not clean
+        await execa(
+          `git update-index --skip-worktree ${path.join(
+            nativePackagesDir,
+            platform,
+            'package.json'
+          )}`,
+          { stdio: 'inherit' }
+        )
+      })
+    )
 
     // Update name/version of wasm packages and publish
     let wasmDir = path.join(cwd, 'packages/next-swc/crates/wasm')
-    for (let wasmTarget of ['web', 'nodejs']) {
-      let wasmPkg = JSON.parse(
-        await readFile(path.join(wasmDir, `pkg-${wasmTarget}/package.json`))
-      )
-      wasmPkg.name = `@next/swc-wasm-${wasmTarget}`
-      wasmPkg.version = version
 
-      await writeFile(
-        path.join(wasmDir, `pkg-${wasmTarget}/package.json`),
-        JSON.stringify(wasmPkg, null, 2)
-      )
+    await Promise.all(
+      ['web', 'nodejs'].map(async (wasmTarget) => {
+        await publishSema.acquire()
 
-      try {
-        execSync(
-          `npm publish ${path.join(
-            wasmDir,
-            `pkg-${wasmTarget}`
-          )} --access public ${
-            gitref.includes('canary') ? ' --tag canary' : ''
-          }`
+        let wasmPkg = JSON.parse(
+          await readFile(path.join(wasmDir, `pkg-${wasmTarget}/package.json`))
         )
-      } catch (err) {
-        // don't block publishing other versions on single platform error
-        console.error(`Failed to publish`, wasmTarget)
+        wasmPkg.name = `@next/swc-wasm-${wasmTarget}`
+        wasmPkg.version = version
 
-        if (
-          err.message &&
-          err.message.includes(
-            'You cannot publish over the previously published versions'
+        await writeFile(
+          path.join(wasmDir, `pkg-${wasmTarget}/package.json`),
+          JSON.stringify(wasmPkg, null, 2)
+        )
+
+        try {
+          await execa(
+            `npm publish ${path.join(
+              wasmDir,
+              `pkg-${wasmTarget}`
+            )} --access public ${
+              version.includes('canary') ? ' --tag canary' : ''
+            }`,
+            { stdio: 'inherit' }
           )
-        ) {
-          console.error('Ignoring already published error', wasmTarget)
-        } else {
-          // throw err
+        } catch (err) {
+          // don't block publishing other versions on single platform error
+          console.error(`Failed to publish`, wasmTarget)
+
+          if (
+            err.message &&
+            err.message.includes(
+              'You cannot publish over the previously published versions'
+            )
+          ) {
+            console.error('Ignoring already published error', wasmTarget)
+          } else {
+            // throw err
+          }
+        } finally {
+          publishSema.release()
         }
-      }
-    }
+      })
+    )
 
     // Update optional dependencies versions
     let nextPkg = JSON.parse(
@@ -122,7 +142,9 @@ const cwd = process.cwd()
       JSON.stringify(nextPkg, null, 2)
     )
     // lerna publish in next step will fail if git status is not clean
-    execSync('git update-index --skip-worktree packages/next/package.json')
+    await execa('git update-index --skip-worktree packages/next/package.json', {
+      stdio: 'inherit',
+    })
   } catch (err) {
     console.error(err)
     process.exit(1)
diff --git a/scripts/publish-release.js b/scripts/publish-release.js
@@ -2,22 +2,20 @@
 // @ts-check
 
 const path = require('path')
-const { readdir } = require('fs/promises')
+const execa = require('execa')
+const { Sema } = require('async-sema')
 const { execSync } = require('child_process')
-const { readJson } = require('fs-extra')
+const { readJson, readdir } = require('fs-extra')
 
 const cwd = process.cwd()
 
 ;(async function () {
   let isCanary = true
 
-  if (!process.env.NPM_TOKEN) {
-    console.log('No NPM_TOKEN, exiting...')
-    return
-  }
-
   try {
-    const tagOutput = execSync('git describe --exact-match').toString()
+    const tagOutput = execSync(
+      `node ${path.join(__dirname, 'check-is-release.js')}`
+    ).toString()
     console.log(tagOutput)
 
     if (tagOutput.trim().startsWith('v')) {
@@ -34,15 +32,23 @@ const cwd = process.cwd()
   }
   console.log(`Publishing ${isCanary ? 'canary' : 'stable'}`)
 
+  if (!process.env.NPM_TOKEN) {
+    console.log('No NPM_TOKEN, exiting...')
+    return
+  }
+
   const packagesDir = path.join(cwd, 'packages')
   const packageDirs = await readdir(packagesDir)
+  const publishSema = new Sema(2)
 
   const publish = async (pkg, retry = 0) => {
     try {
-      execSync(
+      await publishSema.acquire()
+      await execa(
         `npm publish ${path.join(packagesDir, pkg)} --access public${
           isCanary ? ' --tag canary' : ''
-        }`
+        }`,
+        { stdio: 'inherit' }
       )
     } catch (err) {
       console.error(`Failed to publish ${pkg}`, err)
@@ -66,18 +72,22 @@ const cwd = process.cwd()
         await publish(pkg, retry + 1)
       }
       throw err
+    } finally {
+      publishSema.release()
     }
   }
 
-  for (const packageDir of packageDirs) {
-    const pkgJson = await readJson(
-      path.join(packagesDir, packageDir, 'package.json')
-    )
+  await Promise.all(
+    packageDirs.map(async (packageDir) => {
+      const pkgJson = await readJson(
+        path.join(packagesDir, packageDir, 'package.json')
+      )
 
-    if (pkgJson.private) {
-      console.log(`Skipping private package ${packageDir}`)
-      continue
-    }
-    await publish(packageDir)
-  }
+      if (pkgJson.private) {
+        console.log(`Skipping private package ${packageDir}`)
+        return
+      }
+      await publish(packageDir)
+    })
+  )
 })()
diff --git a/scripts/release-stats.sh b/scripts/release-stats.sh
@@ -1,18 +1,18 @@
 #!/bin/bash
 
-git describe --exact-match
-
-if [[ ! $? -eq 0 ]];then
-  echo "Nothing to publish, exiting.."
-  touch .github/actions/next-stats-action/SKIP_NEXT_STATS.txt
-  exit 0;
+if [[ $(node ./scripts/check-is-release.js 2> /dev/null || :) = v* ]];
+  then
+    echo "Publish occurred, running release stats..."
+  else
+    echo "Not publish commit, exiting..."
+    touch .github/actions/next-stats-action/SKIP_NEXT_STATS.txt
+    exit 0;
 fi
 
 if [[ -z "$NPM_TOKEN" ]];then
   echo "No NPM_TOKEN, exiting.."
   exit 0;
 fi
 
-echo "Publish occurred, running release stats..."
 echo "Waiting 30 seconds to allow publish to finalize"
 sleep 30
