Apply publish step optimizations (#43620)

ijjk · ijjk · commit 6b863fe29450 · 2022-12-01T21:48:51.000-08:00
Follow-up to #32337 this removes the un-necessary step where we fetch all of the tags which requires pulling a lot of un-necessary git history inflating cache size and publish times. The only reason these tags were needing to be fetched is due to an issue in how the `actions/checkout` step works (actions/checkout#882). This reduces the publish times by at least 4 minutes by removing the tags fetching step https://github.com/vercel/next.js/actions/runs/3598569786/jobs/6061449995#step:16:14 As a further optimization this adds concurrency to the `npm publish` calls themselves to hopefully reduce time spent there as well.
diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
@@ -84,11 +84,10 @@ jobs:
       - run: pnpm install
       - run: pnpm run build
       - run: node run-tests.js --timings --write-timings -g 1/1
-      - run: node ./scripts/fetch-tags.mjs ${{ github.sha }}
 
       - id: check-release
         run: |
-          if [[ $(git describe --exact-match 2> /dev/null || :) = v* ]];
+          if [[ $(node ./scripts/check-is-release.js 2> /dev/null || :) = v* ]];
             then
               echo "::set-output name=IS_RELEASE::true"
             else
@@ -893,7 +892,7 @@ jobs:
 
       - run: npm i -g pnpm@${PNPM_VERSION}
       - run: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc
-      - run: ./scripts/publish-native.js $GITHUB_REF
+      - run: ./scripts/publish-native.js
       - run: ./scripts/publish-release.js
 
   testDeployE2E:
diff --git a/scripts/check-is-release.js b/scripts/check-is-release.js
@@ -1,6 +1,6 @@
-import { execSync } from 'child_process'
-import execa from 'execa'
-;(async () => {
+const { execSync } = require('child_process')
+
+const checkIsRelease = async () => {
   let commitId = process.argv[2] || ''
 
   // parse only the last string which should be version if
@@ -14,21 +14,13 @@ import execa from 'execa'
   const versionString = commitMsg.split(' ').pop().trim()
   const publishMsgRegex = /^v\d{1,}\.\d{1,}\.\d{1,}(-\w{1,}\.\d{1,})?$/
 
-  console.log({ commitId, commitMsg, versionString })
-
   if (publishMsgRegex.test(versionString)) {
-    console.log('publish commit, fetching tags')
-
-    const result = await execa(
-      'git',
-      ['fetch', '--depth=1', 'origin', '+refs/tags/*:refs/tags/*'],
-      {
-        stdio: ['ignore', 'inherit', 'inherit'],
-      }
-    )
-
-    process.exit(result.exitCode)
+    console.log(versionString)
+    process.exit(0)
   } else {
-    console.log('not publish commit')
+    console.log('not publish commit', { commitId, commitMsg, versionString })
+    process.exit(1)
   }
-})()
+}
+
+checkIsRelease()
diff --git a/scripts/publish-native.js b/scripts/publish-native.js
@@ -1,112 +1,137 @@
 #!/usr/bin/env node
 
 const path = require('path')
-const { readFile, readdir, writeFile } = require('fs/promises')
+const execa = require('execa')
 const { copy } = require('fs-extra')
-const { execSync } = require('child_process')
+const { Sema } = require('async-sema')
+const { readFile, readdir, writeFile } = require('fs/promises')
 
 const cwd = process.cwd()
 
 ;(async function () {
   try {
+    const publishSema = new Sema(2)
+
     let version = JSON.parse(
       await readFile(path.join(cwd, 'lerna.json'))
     ).version
-    let gitref = process.argv.slice(2)[0]
 
     // Copy binaries to package folders, update version, and publish
     let nativePackagesDir = path.join(cwd, 'packages/next-swc/crates/napi/npm')
     let platforms = (await readdir(nativePackagesDir)).filter(
       (name) => !name.startsWith('.')
     )
 
-    for (let platform of platforms) {
-      try {
-        let binaryName = `next-swc.${platform}.node`
-        await copy(
-          path.join(cwd, 'packages/next-swc/native', binaryName),
-          path.join(nativePackagesDir, platform, binaryName)
-        )
-        let pkg = JSON.parse(
-          await readFile(path.join(nativePackagesDir, platform, 'package.json'))
-        )
-        pkg.version = version
-        await writeFile(
-          path.join(nativePackagesDir, platform, 'package.json'),
-          JSON.stringify(pkg, null, 2)
-        )
-        execSync(
-          `npm publish ${path.join(
-            nativePackagesDir,
-            platform
-          )} --access public ${
-            gitref.includes('canary') ? ' --tag canary' : ''
-          }`
-        )
-      } catch (err) {
-        // don't block publishing other versions on single platform error
-        console.error(`Failed to publish`, platform)
+    await Promise.all(
+      platforms.map(async (platform) => {
+        await publishSema.acquire()
 
-        if (
-          err.message &&
-          err.message.includes(
-            'You cannot publish over the previously published versions'
+        try {
+          let binaryName = `next-swc.${platform}.node`
+          await copy(
+            path.join(cwd, 'packages/next-swc/native', binaryName),
+            path.join(nativePackagesDir, platform, binaryName)
+          )
+          let pkg = JSON.parse(
+            await readFile(
+              path.join(nativePackagesDir, platform, 'package.json')
+            )
           )
-        ) {
-          console.error('Ignoring already published error', platform)
-        } else {
-          // throw err
+          pkg.version = version
+          await writeFile(
+            path.join(nativePackagesDir, platform, 'package.json'),
+            JSON.stringify(pkg, null, 2)
+          )
+          await execa(
+            `npm`,
+            [
+              `publish`,
+              `${path.join(nativePackagesDir, platform)}`,
+              `--access`,
+              `public`,
+              ...(version.includes('canary') ? ['--tag', 'canary'] : []),
+            ],
+            { stdio: 'inherit' }
+          )
+        } catch (err) {
+          // don't block publishing other versions on single platform error
+          console.error(`Failed to publish`, platform, err)
+
+          if (
+            err.message &&
+            err.message.includes(
+              'You cannot publish over the previously published versions'
+            )
+          ) {
+            console.error('Ignoring already published error', platform, err)
+          } else {
+            // throw err
+          }
+        } finally {
+          publishSema.release()
         }
-      }
-      // lerna publish in next step sill fail if git status is not clean
-      execSync(
-        `git update-index --skip-worktree ${path.join(
-          nativePackagesDir,
-          platform,
-          'package.json'
-        )}`
-      )
-    }
+        // lerna publish in next step sill fail if git status is not clean
+        await execa(
+          `git`,
+          [
+            'update-index',
+            '--skip-worktree',
+            `${path.join(nativePackagesDir, platform, 'package.json')}`,
+          ],
+          { stdio: 'inherit' }
+        )
+      })
+    )
 
     // Update name/version of wasm packages and publish
     let wasmDir = path.join(cwd, 'packages/next-swc/crates/wasm')
-    for (let wasmTarget of ['web', 'nodejs']) {
-      let wasmPkg = JSON.parse(
-        await readFile(path.join(wasmDir, `pkg-${wasmTarget}/package.json`))
-      )
-      wasmPkg.name = `@next/swc-wasm-${wasmTarget}`
-      wasmPkg.version = version
 
-      await writeFile(
-        path.join(wasmDir, `pkg-${wasmTarget}/package.json`),
-        JSON.stringify(wasmPkg, null, 2)
-      )
+    await Promise.all(
+      ['web', 'nodejs'].map(async (wasmTarget) => {
+        await publishSema.acquire()
 
-      try {
-        execSync(
-          `npm publish ${path.join(
-            wasmDir,
-            `pkg-${wasmTarget}`
-          )} --access public ${
-            gitref.includes('canary') ? ' --tag canary' : ''
-          }`
+        let wasmPkg = JSON.parse(
+          await readFile(path.join(wasmDir, `pkg-${wasmTarget}/package.json`))
+        )
+        wasmPkg.name = `@next/swc-wasm-${wasmTarget}`
+        wasmPkg.version = version
+
+        await writeFile(
+          path.join(wasmDir, `pkg-${wasmTarget}/package.json`),
+          JSON.stringify(wasmPkg, null, 2)
         )
-      } catch (err) {
-        // don't block publishing other versions on single platform error
-        console.error(`Failed to publish`, wasmTarget)
 
-        if (
-          err.message &&
-          err.message.includes(
-            'You cannot publish over the previously published versions'
+        try {
+          await execa(
+            `npm`,
+            [
+              'publish',
+              `${path.join(wasmDir, `pkg-${wasmTarget}`)}`,
+              '--access',
+              'public',
+              ...(version.includes('canary') ? ['--tag', 'canary'] : []),
+            ],
+            { stdio: 'inherit' }
           )
-        ) {
-          console.error('Ignoring already published error', wasmTarget)
-        } else {
-          // throw err
+        } catch (err) {
+          // don't block publishing other versions on single platform error
+          console.error(`Failed to publish`, wasmTarget, err)
+
+          if (
+            err.message &&
+            err.message.includes(
+              'You cannot publish over the previously published versions'
+            )
+          ) {
+            console.error('Ignoring already published error', wasmTarget)
+          } else {
+            // throw err
+          }
+        } finally {
+          publishSema.release()
         }
-      }
-    }
+      })
+    )
 
     // Update optional dependencies versions
     let nextPkg = JSON.parse(
@@ -122,7 +147,13 @@ const cwd = process.cwd()
       JSON.stringify(nextPkg, null, 2)
     )
     // lerna publish in next step will fail if git status is not clean
-    execSync('git update-index --skip-worktree packages/next/package.json')
+    await execa(
+      'git',
+      ['update-index', '--skip-worktree', 'packages/next/package.json'],
+      {
+        stdio: 'inherit',
+      }
+    )
   } catch (err) {
     console.error(err)
     process.exit(1)
diff --git a/scripts/publish-release.js b/scripts/publish-release.js
@@ -2,22 +2,20 @@
 // @ts-check
 
 const path = require('path')
-const { readdir } = require('fs/promises')
+const execa = require('execa')
+const { Sema } = require('async-sema')
 const { execSync } = require('child_process')
-const { readJson } = require('fs-extra')
+const { readJson, readdir } = require('fs-extra')
 
 const cwd = process.cwd()
 
 ;(async function () {
   let isCanary = true
 
-  if (!process.env.NPM_TOKEN) {
-    console.log('No NPM_TOKEN, exiting...')
-    return
-  }
-
   try {
-    const tagOutput = execSync('git describe --exact-match').toString()
+    const tagOutput = execSync(
+      `node ${path.join(__dirname, 'check-is-release.js')}`
+    ).toString()
     console.log(tagOutput)
 
     if (tagOutput.trim().startsWith('v')) {
@@ -34,15 +32,28 @@ const cwd = process.cwd()
   }
   console.log(`Publishing ${isCanary ? 'canary' : 'stable'}`)
 
+  if (!process.env.NPM_TOKEN) {
+    console.log('No NPM_TOKEN, exiting...')
+    return
+  }
+
   const packagesDir = path.join(cwd, 'packages')
   const packageDirs = await readdir(packagesDir)
+  const publishSema = new Sema(2)
 
   const publish = async (pkg, retry = 0) => {
     try {
-      execSync(
-        `npm publish ${path.join(packagesDir, pkg)} --access public${
-          isCanary ? ' --tag canary' : ''
-        }`
+      await publishSema.acquire()
+      await execa(
+        `npm`,
+        [
+          'publish',
+          `${path.join(packagesDir, pkg)}`,
+          '--access',
+          'public',
+          ...(isCanary ? ['--tag', 'canary'] : []),
+        ],
+        { stdio: 'inherit' }
       )
     } catch (err) {
       console.error(`Failed to publish ${pkg}`, err)
@@ -66,18 +77,22 @@ const cwd = process.cwd()
         await publish(pkg, retry + 1)
       }
       throw err
+    } finally {
+      publishSema.release()
     }
   }
 
-  for (const packageDir of packageDirs) {
-    const pkgJson = await readJson(
-      path.join(packagesDir, packageDir, 'package.json')
-    )
+  await Promise.all(
+    packageDirs.map(async (packageDir) => {
+      const pkgJson = await readJson(
+        path.join(packagesDir, packageDir, 'package.json')
+      )
 
-    if (pkgJson.private) {
-      console.log(`Skipping private package ${packageDir}`)
-      continue
-    }
-    await publish(packageDir)
-  }
+      if (pkgJson.private) {
+        console.log(`Skipping private package ${packageDir}`)
+        return
+      }
+      await publish(packageDir)
+    })
+  )
 })()
diff --git a/scripts/release-stats.sh b/scripts/release-stats.sh
