From f6b79ebbff4c5653165a03acf56b3bc5106057cb Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 6 Oct 2023 01:29:03 +0800 Subject: [PATCH 1/8] chore: replace exec ls with fs.readdir --- .../src/prepare/repo-setup.js | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/actions/next-stats-action/src/prepare/repo-setup.js b/.github/actions/next-stats-action/src/prepare/repo-setup.js index 660a6f4a0b01..d5bdb51f9d90 100644 --- a/.github/actions/next-stats-action/src/prepare/repo-setup.js +++ b/.github/actions/next-stats-action/src/prepare/repo-setup.js @@ -55,6 +55,7 @@ module.exports = (actionInfo) => { async linkPackages({ repoDir, nextSwcVersion }) { const pkgPaths = new Map() const pkgDatas = new Map() + let pkgs try { @@ -104,10 +105,21 @@ module.exports = (actionInfo) => { pkgData.files = [] } pkgData.files.push('native') - require('console').log( - 'using swc binaries: ', - await exec(`ls ${path.join(path.dirname(pkgDataPath), 'native')}`) - ) + + try { + const swcBinariesDirContents = await fs.readdir( + path.join(path.dirname(pkgDataPath), 'native') + ) + require('console').log( + 'using swc binaries: ', + swcBinariesDirContents.join(', ') + ) + } catch (err) { + if (err.code === 'ENOENT') { + require('console').log('swc binaries dir is missing!') + } + throw err + } } if (pkg === 'next') { From 6750c868ef8921ef603035ac86363e26d125da27 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 6 Oct 2023 15:04:19 +0800 Subject: [PATCH 2/8] refactor/chore: prefer native fs over fs-extra, reduce Map#get --- .../src/prepare/repo-setup.js | 115 ++++++++++-------- 1 file changed, 65 insertions(+), 50 deletions(-) diff --git a/.github/actions/next-stats-action/src/prepare/repo-setup.js b/.github/actions/next-stats-action/src/prepare/repo-setup.js index d5bdb51f9d90..5bc22abf0f4a 100644 --- a/.github/actions/next-stats-action/src/prepare/repo-setup.js +++ b/.github/actions/next-stats-action/src/prepare/repo-setup.js @@ -1,5 +1,7 @@ const path = require('path') -const fs = require('fs-extra') +const fse = require('fs-extra') +const fs = require('fs') +const fsp = require('fs/promises') const exec = require('../util/exec') const { remove } = require('fs-extra') const logger = require('../util/logger') @@ -54,12 +56,23 @@ module.exports = (actionInfo) => { }, async linkPackages({ repoDir, nextSwcVersion }) { const pkgPaths = new Map() + + /** + * @typedef {Object} PkgData + * @property {string} pkgDataPath Where the package.json file is located + * @property {string} pkg The folder name of the package + * @property {string} pkgPath The path to the package folder + * @property {any} pkgData The content of package.json + * @property {string} packedPkgPath The npm pack output .tgz file path + */ + + /** @type {Map} */ const pkgDatas = new Map() let pkgs try { - pkgs = await fs.readdir(path.join(repoDir, 'packages')) + pkgs = await fsp.readdir(path.join(repoDir, 'packages')) } catch (err) { if (err.code === 'ENOENT') { require('console').log('no packages to link') @@ -90,25 +103,25 @@ module.exports = (actionInfo) => { pkgPaths.set(name, packedPkgPath) } - for (const pkg of pkgDatas.keys()) { - const { pkgDataPath, pkgData } = pkgDatas.get(pkg) - - for (const pkg of pkgDatas.keys()) { - const { packedPkgPath } = pkgDatas.get(pkg) + for (const [ + pkg, + { pkgDataPath, pkgData, pkgPath }, + ] of pkgDatas.entries()) { + // update the current package dependencies to point to packed tgz path + for (const [pkg, { packedPkgPath }] of pkgDatas.entries()) { if (!pkgData.dependencies || !pkgData.dependencies[pkg]) continue pkgData.dependencies[pkg] = packedPkgPath } // make sure native binaries are included in local linking if (pkg === '@next/swc') { - if (!pkgData.files) { - pkgData.files = [] - } + pkgData.files ||= [] + pkgData.files.push('native') try { - const swcBinariesDirContents = await fs.readdir( - path.join(path.dirname(pkgDataPath), 'native') + const swcBinariesDirContents = await fsp.readdir( + path.join(pkgPath, 'native') ) require('console').log( 'using swc binaries: ', @@ -120,28 +133,27 @@ module.exports = (actionInfo) => { } throw err } - } + } else if (pkg === 'next') { + const nextSwcPkg = pkgDatas.get('@next/swc') - if (pkg === 'next') { console.log('using swc dep', { nextSwcVersion, - nextSwcPkg: pkgDatas.get('@next/swc'), + nextSwcPkg, }) if (nextSwcVersion) { Object.assign(pkgData.dependencies, { '@next/swc-linux-x64-gnu': nextSwcVersion, }) } else { - if (pkgDatas.get('@next/swc')) { - pkgData.dependencies['@next/swc'] = - pkgDatas.get('@next/swc').packedPkgPath + if (nextSwcPkg) { + pkgData.dependencies['@next/swc'] = nextSwcPkg.packedPkgPath } else { pkgData.files.push('native') } } } - await fs.writeFile( + await fsp.writeFile( pkgDataPath, JSON.stringify(pkgData, null, 2), 'utf8' @@ -151,41 +163,44 @@ module.exports = (actionInfo) => { // wait to pack packages until after dependency paths have been updated // to the correct versions await Promise.all( - Array.from(pkgDatas.keys()).map(async (pkgName) => { - const { pkgPath, packedPkgPath } = pkgDatas.get(pkgName) - - let cleanup = null - - if (pkgName === '@next/swc') { - // next-swc uses a gitignore to prevent the committing of native builds but it doesn't - // use files in package.json because it publishes to individual packages based on architecture. - // When we used yarn to pack these packages the gitignore was ignored so the native builds were packed - // however npm does respect gitignore when packing so we need to remove it in this specific case - // to ensure the native builds are packed for use in gh actions and related scripts - await fs.rename( - path.join(pkgPath, 'native/.gitignore'), - path.join(pkgPath, 'disabled-native-gitignore') - ) - cleanup = async () => { - await fs.rename( - path.join(pkgPath, 'disabled-native-gitignore'), - path.join(pkgPath, 'native/.gitignore') + Array.from(pkgDatas.entries()).map( + async ([pkgName, { pkgPath, packedPkgPath }]) => { + let cleanup = null + + if (pkgName === '@next/swc') { + // next-swc uses a gitignore to prevent the committing of native builds but it doesn't + // use files in package.json because it publishes to individual packages based on architecture. + // When we used yarn to pack these packages the gitignore was ignored so the native builds were packed + // however npm does respect gitignore when packing so we need to remove it in this specific case + // to ensure the native builds are packed for use in gh actions and related scripts + await fse.rename( + path.join(pkgPath, 'native/.gitignore'), + path.join(pkgPath, 'disabled-native-gitignore') ) + cleanup = async () => { + await fse.rename( + path.join(pkgPath, 'disabled-native-gitignore'), + path.join(pkgPath, 'native/.gitignore') + ) + } } - } - const { stdout } = await execa('npm', ['pack'], { - cwd: pkgPath, - env: { - ...process.env, - COREPACK_ENABLE_STRICT: '0', - }, - }) - await fs.rename(path.resolve(pkgPath, stdout.trim()), packedPkgPath) - if (cleanup) { - await cleanup() + const { stdout } = await execa('npm', ['pack'], { + cwd: pkgPath, + env: { + ...process.env, + COREPACK_ENABLE_STRICT: '0', + }, + }) + await fse.rename( + path.resolve(pkgPath, stdout.trim()), + packedPkgPath + ) + if (cleanup) { + await cleanup() + } } - }) + ) ) return pkgPaths }, From bec8de7e70459b420171f97a603b7f938d720a4e Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 6 Oct 2023 16:58:17 +0800 Subject: [PATCH 3/8] refactor: replace npm pack w/ pnpm pack --- .../src/prepare/repo-setup.js | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/.github/actions/next-stats-action/src/prepare/repo-setup.js b/.github/actions/next-stats-action/src/prepare/repo-setup.js index 5bc22abf0f4a..b5f8ab64ac5f 100644 --- a/.github/actions/next-stats-action/src/prepare/repo-setup.js +++ b/.github/actions/next-stats-action/src/prepare/repo-setup.js @@ -90,7 +90,7 @@ module.exports = (actionInfo) => { require('console').log(`Skipping ${pkgDataPath}`) continue } - const pkgData = require(pkgDataPath) + const pkgData = await fse.readJSON(pkgDataPath) const { name } = pkgData pkgDatas.set(name, { @@ -165,6 +165,7 @@ module.exports = (actionInfo) => { await Promise.all( Array.from(pkgDatas.entries()).map( async ([pkgName, { pkgPath, packedPkgPath }]) => { + /** @type {null | () => Promise} */ let cleanup = null if (pkgName === '@next/swc') { @@ -173,32 +174,34 @@ module.exports = (actionInfo) => { // When we used yarn to pack these packages the gitignore was ignored so the native builds were packed // however npm does respect gitignore when packing so we need to remove it in this specific case // to ensure the native builds are packed for use in gh actions and related scripts - await fse.rename( - path.join(pkgPath, 'native/.gitignore'), - path.join(pkgPath, 'disabled-native-gitignore') + + const nativeGitignorePath = path.join( + pkgPath, + 'native/.gitignore' + ) + const renamedGitignorePath = path.join( + pkgPath, + 'disabled-native-gitignore' ) + + await fsp.rename(nativeGitignorePath, renamedGitignorePath) cleanup = async () => { - await fse.rename( - path.join(pkgPath, 'disabled-native-gitignore'), - path.join(pkgPath, 'native/.gitignore') - ) + await fsp.rename(renamedGitignorePath, nativeGitignorePath) } } - const { stdout } = await execa('npm', ['pack'], { + const { stdout } = await execa('pnpm', ['pack'], { cwd: pkgPath, env: { ...process.env, COREPACK_ENABLE_STRICT: '0', }, }) - await fse.rename( - path.resolve(pkgPath, stdout.trim()), - packedPkgPath - ) - if (cleanup) { - await cleanup() - } + + return Promise.all([ + fsp.rename(path.resolve(pkgPath, stdout.trim()), packedPkgPath), + cleanup?.(), + ]) } ) ) From 63ad237c15e5916f04277d38f17e07abe4d306e0 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 6 Oct 2023 21:01:00 +0800 Subject: [PATCH 4/8] refactor: remove extra async --- test/lib/create-next-install.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/lib/create-next-install.js b/test/lib/create-next-install.js index 1674921389b4..6ebe3899f58f 100644 --- a/test/lib/create-next-install.js +++ b/test/lib/create-next-install.js @@ -16,8 +16,8 @@ const { linkPackages } = * @param {string} cwd - The project directory where pnpm configuration is set. * @returns {Promise} */ -async function setPnpmResolutionMode(cwd) { - await execa( +function setPnpmResolutionMode(cwd) { + return execa( 'pnpm', ['config', 'set', '--location=project', 'resolution-mode', 'highest'], { @@ -99,8 +99,8 @@ async function createNextInstall({ for (const item of ['package.json', 'packages']) { await rootSpan .traceChild(`copy ${item} to temp dir`) - .traceAsyncFn(async () => { - await fs.copy( + .traceAsyncFn(() => + fs.copy( path.join(origRepoDir, item), path.join(tmpRepoDir, item), { @@ -116,7 +116,7 @@ async function createNextInstall({ }, } ) - }) + ) } pkgPaths = await rootSpan.traceChild('linkPackages').traceAsyncFn(() => @@ -177,7 +177,7 @@ async function createNextInstall({ } else { await rootSpan .traceChild('run generic install command') - .traceAsyncFn(async () => { + .traceAsyncFn(() => { const args = [ 'install', '--strict-peer-dependencies=false', @@ -188,7 +188,7 @@ async function createNextInstall({ args.push('--prefer-offline') } - await execa('pnpm', args, { + return execa('pnpm', args, { cwd: installDir, stdio: ['ignore', 'inherit', 'inherit'], env: process.env, From 541805698bb38a9db94a9f6eb1311e5d8294ce58 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 6 Oct 2023 21:56:44 +0800 Subject: [PATCH 5/8] chore: only use pnpm pack in test --- .../src/prepare/repo-setup.js | 20 +++++++++++-------- test/lib/create-next-install.js | 1 + 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/actions/next-stats-action/src/prepare/repo-setup.js b/.github/actions/next-stats-action/src/prepare/repo-setup.js index b5f8ab64ac5f..f16c27a327cc 100644 --- a/.github/actions/next-stats-action/src/prepare/repo-setup.js +++ b/.github/actions/next-stats-action/src/prepare/repo-setup.js @@ -54,7 +54,7 @@ module.exports = (actionInfo) => { } } }, - async linkPackages({ repoDir, nextSwcVersion }) { + async linkPackages({ repoDir, nextSwcVersion, usePnpmPack = false }) { const pkgPaths = new Map() /** @@ -190,13 +190,17 @@ module.exports = (actionInfo) => { } } - const { stdout } = await execa('pnpm', ['pack'], { - cwd: pkgPath, - env: { - ...process.env, - COREPACK_ENABLE_STRICT: '0', - }, - }) + const { stdout } = await execa( + usePnpmPack ? 'pnpm' : 'npm', + ['pack'], + { + cwd: pkgPath, + env: { + ...process.env, + COREPACK_ENABLE_STRICT: '0', + }, + } + ) return Promise.all([ fsp.rename(path.resolve(pkgPath, stdout.trim()), packedPkgPath), diff --git a/test/lib/create-next-install.js b/test/lib/create-next-install.js index 6ebe3899f58f..00d90d0adf8c 100644 --- a/test/lib/create-next-install.js +++ b/test/lib/create-next-install.js @@ -122,6 +122,7 @@ async function createNextInstall({ pkgPaths = await rootSpan.traceChild('linkPackages').traceAsyncFn(() => linkPackages({ repoDir: tmpRepoDir, + usePnpmPack: true, }) ) } From a9d7f6971ada10e4c7cd4799e4cf386b37a8c031 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 6 Oct 2023 22:47:23 +0800 Subject: [PATCH 6/8] refactor: replace readJson with JSON.parse + readFile --- .../src/prepare/repo-setup.js | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/.github/actions/next-stats-action/src/prepare/repo-setup.js b/.github/actions/next-stats-action/src/prepare/repo-setup.js index f16c27a327cc..7820f32a64c8 100644 --- a/.github/actions/next-stats-action/src/prepare/repo-setup.js +++ b/.github/actions/next-stats-action/src/prepare/repo-setup.js @@ -81,27 +81,29 @@ module.exports = (actionInfo) => { throw err } - for (const pkg of pkgs) { - const pkgPath = path.join(repoDir, 'packages', pkg) - const packedPkgPath = path.join(pkgPath, `${pkg}-packed.tgz`) - - const pkgDataPath = path.join(pkgPath, 'package.json') - if (!fs.existsSync(pkgDataPath)) { - require('console').log(`Skipping ${pkgDataPath}`) - continue - } - const pkgData = await fse.readJSON(pkgDataPath) - const { name } = pkgData - - pkgDatas.set(name, { - pkgDataPath, - pkg, - pkgPath, - pkgData, - packedPkgPath, + await Promise.all( + pkgs.map(async (pkg) => { + const pkgPath = path.join(repoDir, 'packages', pkg) + const packedPkgPath = path.join(pkgPath, `${pkg}-packed.tgz`) + + const pkgDataPath = path.join(pkgPath, 'package.json') + if (fs.existsSync(pkgDataPath)) { + const pkgData = JSON.parse(await fsp.readFile(pkgDataPath)) + const { name } = pkgData + + pkgDatas.set(name, { + pkgDataPath, + pkg, + pkgPath, + pkgData, + packedPkgPath, + }) + pkgPaths.set(name, packedPkgPath) + } else { + require('console').log(`Skipping ${pkgDataPath}`) + } }) - pkgPaths.set(name, packedPkgPath) - } + ) for (const [ pkg, From fec0ae848f99c23f154558d69ae47810108c7cd4 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 6 Oct 2023 23:00:33 +0800 Subject: [PATCH 7/8] chore: all-in pnpm --- .../src/prepare/repo-setup.js | 20 ++++++++----------- test/lib/create-next-install.js | 1 - 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/.github/actions/next-stats-action/src/prepare/repo-setup.js b/.github/actions/next-stats-action/src/prepare/repo-setup.js index 7820f32a64c8..74fd6ea1fc92 100644 --- a/.github/actions/next-stats-action/src/prepare/repo-setup.js +++ b/.github/actions/next-stats-action/src/prepare/repo-setup.js @@ -54,7 +54,7 @@ module.exports = (actionInfo) => { } } }, - async linkPackages({ repoDir, nextSwcVersion, usePnpmPack = false }) { + async linkPackages({ repoDir, nextSwcVersion }) { const pkgPaths = new Map() /** @@ -192,17 +192,13 @@ module.exports = (actionInfo) => { } } - const { stdout } = await execa( - usePnpmPack ? 'pnpm' : 'npm', - ['pack'], - { - cwd: pkgPath, - env: { - ...process.env, - COREPACK_ENABLE_STRICT: '0', - }, - } - ) + const { stdout } = await execa('pnpm', ['pack'], { + cwd: pkgPath, + env: { + ...process.env, + COREPACK_ENABLE_STRICT: '0', + }, + }) return Promise.all([ fsp.rename(path.resolve(pkgPath, stdout.trim()), packedPkgPath), diff --git a/test/lib/create-next-install.js b/test/lib/create-next-install.js index 00d90d0adf8c..6ebe3899f58f 100644 --- a/test/lib/create-next-install.js +++ b/test/lib/create-next-install.js @@ -122,7 +122,6 @@ async function createNextInstall({ pkgPaths = await rootSpan.traceChild('linkPackages').traceAsyncFn(() => linkPackages({ repoDir: tmpRepoDir, - usePnpmPack: true, }) ) } From 64487705a94838fea22f7599aa0ae9b4f0c0642e Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 6 Oct 2023 23:06:09 +0800 Subject: [PATCH 8/8] ci: always run `corepack enable` after `actions/setup-node` --- .github/workflows/build_and_deploy.yml | 1 + .github/workflows/build_and_test.yml | 1 + .github/workflows/code_freeze.yml | 1 + .github/workflows/pull_request_approved.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 44798960f3a6..93648a7ae05c 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -254,6 +254,7 @@ jobs: with: node-version: ${{ env.NODE_LTS_VERSION }} check-latest: true + - run: corepack enable - name: Install Rust uses: ./.github/actions/setup-rust diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 9b18c42aa0ec..1f4d50d639ed 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -58,6 +58,7 @@ jobs: - uses: actions/setup-node@v3 with: node-version: 18 + - run: corepack enable - name: 'Run link checker' run: node ./.github/actions/validate-docs-links/lib env: diff --git a/.github/workflows/code_freeze.yml b/.github/workflows/code_freeze.yml index 513390827cc8..3c4c8b23f6bf 100644 --- a/.github/workflows/code_freeze.yml +++ b/.github/workflows/code_freeze.yml @@ -32,6 +32,7 @@ jobs: with: node-version: 18 check-latest: true + - run: corepack enable - run: git clone https://github.com/vercel/next.js.git --depth=1 . diff --git a/.github/workflows/pull_request_approved.yml b/.github/workflows/pull_request_approved.yml index 4dec0271a80d..77519be02f1a 100644 --- a/.github/workflows/pull_request_approved.yml +++ b/.github/workflows/pull_request_approved.yml @@ -14,6 +14,7 @@ jobs: - uses: actions/setup-node@v3 with: node-version: 18 + - run: corepack enable - name: 'Send notification to Slack' run: node ./.github/actions/pr-approved-open/index.mjs env: