feat: remove qq and parallelize build/pack/promote (#1035)

* feat: remove qq and parallelize build/promote * chore: reformat pjson * refactor: remove redundant objects * refactor: pr feedback * feat: publishes debian packages with qq * test: integration tests in parallel * test: restore debian integration tests * chore: remove ds_store * chore: ignore dsStore * chore: ignore dist * Update src/commands/upload/deb.ts Co-authored-by: Rodrigo Espinosa de los Monteros <1084688+RodEsp@users.noreply.github.com> * fix: cwd on pretarball * refactor: pr feedback * test: integration test for tarball promote * test: revert unnecessary trims * perf: optional xz hashes in parallel * chore: fix pjson format * refactor: remove unused conditional * fix: osslsign command requires "sign" Co-authored-by: Rodrigo Espinosa de los Monteros <1084688+RodEsp@users.noreply.github.com>
oclif · Dec 5, 2022 · db2a858 · db2a858
1 parent a4d258a
commit db2a858
Show file tree

Hide file tree

Showing 28 changed files with 842 additions and 817 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -5,7 +5,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  tests:
+  unit-tests:
     strategy:
       matrix:
         node_version: [lts/-1, lts/*, latest]
@@ -21,10 +21,87 @@ jobs:
       - run: yarn nps lint
       - run: yarn build
       - run: yarn test:unit
+  publish:
+    needs: [unit-tests]
+    strategy:
+      matrix:
+        node_version: [lts/-1, lts/*, latest]
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node_version }}
+          cache: yarn
+      - run: yarn install --network-timeout 600000
+      - run: yarn build
+      - run: yarn mocha test/integration/publish.test.ts
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  win-build:
+    needs: [unit-tests]
+    strategy:
+      matrix:
+        node_version: [lts/-1, lts/*, latest]
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node_version }}
+          cache: yarn
+      - run: yarn install --network-timeout 600000
+      - run: yarn build
       - run: |
           sudo apt-get update
           sudo apt-get install osslsigncode nsis
-      - run: yarn test:integration
+      - run: yarn mocha test/integration/win.test.ts
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  mac-build:
+    needs: [unit-tests]
+    strategy:
+      matrix:
+        node_version: [lts/-1, lts/*, latest]
+      fail-fast: false
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node_version }}
+          cache: yarn
+      - run: yarn install --network-timeout 600000
+      - run: yarn build
+      - run: yarn mocha test/integration/macos.test.ts
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  deb-build:
+    needs: [unit-tests]
+    env:
+      OCLIF_DEB_PRIVATE_KEY: ${{ secrets.OCLIF_DEB_PRIVATE_KEY }}
+    strategy:
+      matrix:
+        node_version: [lts/-1, lts/*, latest]
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node_version }}
+          cache: yarn
+      - run: yarn install --network-timeout 600000
+      - run: yarn build
+      - run: echo -n "$OCLIF_DEB_PRIVATE_KEY" | gpg --import
+
+      - run: yarn mocha test/integration/deb.test.ts
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          OCLIF_DEB_KEY: ${{ secrets.OCLIF_DEB_KEY }}
diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,9 @@
 /coverage
 /coverage.lcov
 /lib
+/dist
 /node_modules
 /tmp
+/test/tmp
 package-lock.json
+**/.DS_Store
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -0,0 +1,15 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "node",
+      "request": "attach",
+      "name": "Attach",
+      "port": 9229,
+      "skipFiles": ["<node_internals>/**"]
+    }
+  ]
+}
diff --git a/package.json b/package.json
@@ -18,9 +18,9 @@
     "find-yarn-workspace-root": "^2.0.0",
     "fs-extra": "^8.1",
     "github-slugger": "^1.5.0",
+    "got": "^11",
     "lodash": "^4.17.21",
     "normalize-package-data": "^3.0.3",
-    "qqjs": "^0.3.11",
     "semver": "^7.3.8",
     "tslib": "^2.3.1",
     "yeoman-environment": "^3.11.1",
@@ -62,12 +62,10 @@
     "typescript": "4.5.5"
   },
   "resolutions": {
-    "colors": "1.4.0",
-    "@oclif/core": "^1.16.1"
+    "colors": "1.4.0"
   },
   "overrides": {
-    "colors": "1.4.0",
-    "@oclif/core": "^1.16.1"
+    "colors": "1.4.0"
   },
   "engines": {
     "node": ">=12.0.0"
@@ -103,7 +101,7 @@
         "debounce": 60
       },
       "node": {
-        "version": "12.12.0"
+        "version": "16.13.2"
       },
       "s3": {
         "bucket": "dfc-data-production",
@@ -142,4 +140,4 @@
     "registry": "https://registry.npmjs.org"
   },
   "types": "lib/index.d.ts"
-}
+}
diff --git a/src/aws.ts b/src/aws.ts
@@ -1,9 +1,9 @@
 import * as CloudFront from 'aws-sdk/clients/cloudfront'
 import * as S3 from 'aws-sdk/clients/s3'
 import * as fs from 'fs-extra'
-import * as qq from 'qqjs'
 
 import {debug as Debug, log} from './log'
+import {prettifyPaths} from './util'
 
 const debug = Debug.new('aws')
 
@@ -65,7 +65,7 @@ export default {
   get s3() {
     return {
       uploadFile: (local: string, options: S3.Types.PutObjectRequest) => new Promise((resolve, reject) => {
-        log('s3:uploadFile', qq.prettifyPaths(local), `s3://${options.Bucket}/${options.Key}`)
+        log('s3:uploadFile', prettifyPaths(local), `s3://${options.Bucket}/${options.Key}`)
         options.Body = fs.createReadStream(local)
         aws.s3.upload(options, err => {
           if (err) reject(err)

diff --git a/src/commands/pack/deb.ts b/src/commands/pack/deb.ts
@@ -1,11 +1,14 @@
 import {Command, Flags} from '@oclif/core'
 import {Interfaces} from '@oclif/core'
-
+import * as fs from 'fs-extra'
 import * as _ from 'lodash'
-import * as qq from 'qqjs'
-
+import * as path from 'path'
 import * as Tarballs from '../../tarballs'
 import {templateShortKey, debVersion, debArch} from '../../upload-util'
+import {exec as execSync} from 'child_process'
+import {promisify} from 'node:util'
+
+const exec = promisify(execSync)
 
 const scripts = {
   /* eslint-disable no-useless-escape */
@@ -59,45 +62,64 @@ export default class PackDeb extends Command {
     const {flags} = await this.parse(PackDeb)
     const buildConfig = await Tarballs.buildConfig(flags.root)
     const {config} = buildConfig
-    await Tarballs.build(buildConfig, {platform: 'linux', pack: false, tarball: flags.tarball})
+    await Tarballs.build(buildConfig, {platform: 'linux', pack: false, tarball: flags.tarball, parallel: true})
     const dist = buildConfig.dist('deb')
-    await qq.emptyDir(dist)
+    await fs.emptyDir(dist)
     const build = async (arch: Interfaces.ArchTypes) => {
+      this.log(`building debian / ${arch}`)
       const target: { platform: 'linux'; arch: Interfaces.ArchTypes} = {platform: 'linux', arch}
       const versionedDebBase = templateShortKey('deb', {bin: config.bin, versionShaRevision: debVersion(buildConfig), arch: debArch(arch) as any})
-      const workspace = qq.join(buildConfig.tmp, 'apt', versionedDebBase.replace('.deb', '.apt'))
-      await qq.rm(workspace)
-      await qq.mkdirp([workspace, 'DEBIAN'])
-      await qq.mkdirp([workspace, 'usr/bin'])
-      await qq.mkdirp([workspace, 'usr/lib'])
-      await qq.mv(buildConfig.workspace(target), [workspace, 'usr/lib', config.dirname])
-      await qq.write([workspace, 'usr/lib', config.dirname, 'bin', config.bin], scripts.bin(config))
-      await qq.write([workspace, 'DEBIAN/control'], scripts.control(buildConfig, debArch(arch)))
-      await qq.chmod([workspace, 'usr/lib', config.dirname, 'bin', config.bin], 0o755)
-      await qq.x(`ln -s "../lib/${config.dirname}/bin/${config.bin}" "${workspace}/usr/bin/${config.bin}"`)
-      await qq.x(`sudo chown -R root "${workspace}"`)
-      await qq.x(`sudo chgrp -R root "${workspace}"`)
-      await qq.x(`dpkg --build "${workspace}" "${qq.join(dist, versionedDebBase)}"`)
+      const workspace = path.join(buildConfig.tmp, 'apt', versionedDebBase.replace('.deb', '.apt'))
+      await fs.remove(workspace)
+      await Promise.all([
+        fs.promises.mkdir(path.join(workspace, 'DEBIAN'), {recursive: true}),
+        fs.promises.mkdir(path.join(workspace, 'usr', 'bin'), {recursive: true}),
+      ])
+      await fs.copy(buildConfig.workspace(target), path.join(workspace, 'usr', 'lib', config.dirname))
+      await Promise.all([
+        // usr/lib/oclif/bin/oclif (the executable)
+        fs.promises.writeFile(path.join(workspace, 'usr', 'lib', config.dirname, 'bin', config.bin), scripts.bin(config), {mode: 0o755}),
+        fs.promises.writeFile(path.join(workspace, 'DEBIAN', 'control'), scripts.control(buildConfig, debArch(arch))),
+      ])
+      // symlink usr/bin/oclif points to usr/lib/oclif/bin/oclif
+      await exec(`ln -s "${path.join('..', 'lib', config.dirname, 'bin', config.bin)}" "${config.bin}"`, {cwd: path.join(workspace, 'usr', 'bin')})
+      await exec(`sudo chown -R root "${workspace}"`)
+      await exec(`sudo chgrp -R root "${workspace}"`)
+      await exec(`dpkg --build "${workspace}" "${path.join(dist, versionedDebBase)}"`)
+      this.log(`finished building debian / ${arch}`)
     }
 
     const arches = _.uniq(buildConfig.targets
     .filter(t => t.platform === 'linux')
     .map(t => t.arch))
-    // eslint-disable-next-line no-await-in-loop
-    for (const a of arches) await build(a)
+    await Promise.all(arches.map(a => build(a)))
+
+    await exec('apt-ftparchive packages . > Packages', {cwd: dist})
+    this.log('debian packages created')
+    await Promise.all([
+      exec('gzip -c Packages > Packages.gz', {cwd: dist}),
+      exec('bzip2 -k Packages', {cwd: dist}),
+      exec('xz -k Packages', {cwd: dist}),
+      packForFTP(buildConfig, config, dist),
+    ])
+
+    this.log('debian packages archived')
 
-    await qq.x('apt-ftparchive packages . > Packages', {cwd: dist})
-    await qq.x('gzip -c Packages > Packages.gz', {cwd: dist})
-    await qq.x('bzip2 -k Packages', {cwd: dist})
-    await qq.x('xz -k Packages', {cwd: dist})
-    const ftparchive = qq.join(buildConfig.tmp, 'apt', 'apt-ftparchive.conf')
-    await qq.write(ftparchive, scripts.ftparchive(config))
-    await qq.x(`apt-ftparchive -c "${ftparchive}" release . > Release`, {cwd: dist})
     const gpgKey = config.scopedEnvVar('DEB_KEY')
     if (gpgKey) {
-      await qq.x(`gpg --digest-algo SHA512 --clearsign -u ${gpgKey} -o InRelease Release`, {cwd: dist})
-      await qq.x(`gpg --digest-algo SHA512 -abs -u ${gpgKey} -o Release.gpg Release`, {cwd: dist})
+      this.log('adding gpg signatures to Release')
+      await exec(`gpg --digest-algo SHA512 --clearsign -u ${gpgKey} -o InRelease Release`, {cwd: dist})
+      await exec(`gpg --digest-algo SHA512 -abs -u ${gpgKey} -o Release.gpg Release`, {cwd: dist})
     }
+
+    this.log('debian packing complete')
   }
 }
 
+async function packForFTP(buildConfig: Tarballs.BuildConfig, config: Interfaces.Config, dist: string) {
+  const ftparchive = path.join(buildConfig.tmp, 'apt', 'apt-ftparchive.conf')
+  await fs.promises.mkdir(path.basename(ftparchive), {recursive: true})
+  await fs.writeFile(ftparchive, scripts.ftparchive(config))
+  await exec(`apt-ftparchive -c "${ftparchive}" release . > Release`, {cwd: dist})
+}
+
diff --git a/src/commands/pack/macos.ts b/src/commands/pack/macos.ts
@@ -1,14 +1,16 @@
 import * as path from 'path'
 
 import * as _ from 'lodash'
-import * as qq from 'qqjs'
-
+import * as fs from 'fs-extra'
 import {Command, Flags} from '@oclif/core'
 import {Interfaces} from '@oclif/core'
 
 import * as Tarballs from '../../tarballs'
 import {templateShortKey} from '../../upload-util'
+import {exec as execSync} from 'child_process'
+import {promisify} from 'node:util'
 
+const exec = promisify(execSync)
 type OclifConfig = {
   macos?: {
     identifier?: string;
@@ -77,7 +79,7 @@ while [ "$1" != "-y" ]; do
 done
 
 echo "Application uninstalling process started"
-# remove link to shorcut file
+# remove link to shortcut file
 find "/usr/local/bin/" -name "${config.bin}" | xargs rm
 ${additionalCLI ? `find "/usr/local/bin/" -name "${additionalCLI}" | xargs rm` : ''}
 if [ $? -eq 0 ]
@@ -156,30 +158,33 @@ the CLI should already exist in a directory named after the CLI that is the root
     if (!c.macos.identifier) this.error('package.json must have oclif.macos.identifier set')
     const macos = c.macos
     const packageIdentifier = macos.identifier
-    await Tarballs.build(buildConfig, {platform: 'darwin', pack: false, tarball: flags.tarball})
-    const scriptsDir = qq.join(buildConfig.tmp, 'macos/scripts')
-    await qq.emptyDir(buildConfig.dist('macos'))
-    const noBundleConfigurationPath = qq.join(buildConfig.tmp, 'macos/no-bundle.plist')
+    await Tarballs.build(buildConfig, {platform: 'darwin', pack: false, tarball: flags.tarball, parallel: true})
+    const scriptsDir = path.join(buildConfig.tmp, 'macos/scripts')
+    await fs.emptyDir(buildConfig.dist('macos'))
+    const noBundleConfigurationPath = path.join(buildConfig.tmp, 'macos', 'no-bundle.plist')
 
     const build = async (arch: Interfaces.ArchTypes) => {
       const templateKey = templateShortKey('macos', {bin: config.bin, version: config.version, sha: buildConfig.gitSha, arch})
       const dist = buildConfig.dist(`macos/${templateKey}`)
       const rootDir = buildConfig.workspace({platform: 'darwin', arch})
       const writeNoBundleConfiguration = async () => {
-        await qq.write(noBundleConfigurationPath, noBundleConfiguration)
-        await qq.chmod(noBundleConfigurationPath, 0o755)
+        await fs.mkdir(path.dirname(noBundleConfigurationPath), {recursive: true})
+        await fs.writeFile(noBundleConfigurationPath, noBundleConfiguration, {mode: 0o755})
       }
+
       const writeScript = async (script: 'preinstall' | 'postinstall' | 'uninstall') => {
-        const path = script === 'uninstall' ? [rootDir, 'bin'] : [scriptsDir]
-        path.push(script)
-        await qq.write(path, scripts[script](config, flags['additional-cli']))
-        await qq.chmod(path, 0o755)
+        const scriptLocation = script === 'uninstall' ? [rootDir, 'bin'] : [scriptsDir]
+        scriptLocation.push(script)
+        await fs.mkdir(path.dirname(path.join(...scriptLocation)), {recursive: true})
+        await fs.writeFile(path.join(...scriptLocation), scripts[script](config, flags['additional-cli']), {mode: 0o755})
       }
 
-      await writeNoBundleConfiguration();
-      await writeScript('preinstall')
-      await writeScript('postinstall')
-      await writeScript('uninstall')
+      await Promise.all([
+        writeNoBundleConfiguration(),
+        writeScript('preinstall'),
+        writeScript('postinstall'),
+        writeScript('uninstall'),
+      ])
       /* eslint-disable array-element-newline */
       const args = [
         '--root', rootDir,
@@ -195,13 +200,12 @@ the CLI should already exist in a directory named after the CLI that is the root
       } else this.debug('Skipping macOS pkg signing')
       if (process.env.OSX_KEYCHAIN) args.push('--keychain', process.env.OSX_KEYCHAIN)
       args.push(dist)
-      await qq.x('pkgbuild', args as string[])
+      await exec(`pkgbuild  ${args.join(' ')}`)
     }
 
     const arches = _.uniq(buildConfig.targets
     .filter(t => t.platform === 'darwin')
     .map(t => t.arch))
-    // eslint-disable-next-line no-await-in-loop
-    for (const a of arches) await build(a)
+    await Promise.all(arches.map(a => build(a)))
   }
 }