fix: add tests for publish

.circleci/config.yml

@@ -32,6 +32,8 @@ jobs:
       - image: oclif/release:0.0.0-8.11.1
   release:
     <<: *test
+    environment:
+      OCLIF_DEV_S3_BUCKET: oclif
     steps:
       - add_ssh_keys
       - checkout

package.json

@@ -9,8 +9,8 @@
   "bugs": "https://github.com/oclif/dev-cli/issues",
   "dependencies": {
     "@heroku-cli/color": "^1.1.3",
-    "@oclif/command": "^1.4.8",
-    "@oclif/config": "^1.4.0",
+    "@oclif/command": "^1.4.9",
+    "@oclif/config": "^1.4.3",
     "@oclif/errors": "^1.0.4",
     "@oclif/plugin-help": "^1.2.3",
     "@oclif/plugin-warn-if-update-available": "^1.3.0",
@@ -20,7 +20,7 @@
     "lodash": "^4.17.5",
     "lodash.template": "^4.4.0",
     "normalize-package-data": "^2.4.0",
-    "qqjs": "^0.2.1",
+    "qqjs": "^0.3.2",
     "require-resolve": "^0.0.2"
   },
   "devDependencies": {
@@ -75,7 +75,7 @@
         ]
       },
       "s3": {
-        "bucket": "oclif",
+        "bucket": "oclif-staging",
         "xz": true
       }
     },

src/commands/publish/s3.ts

@@ -27,29 +27,35 @@ export default class Publish extends Command {
     const {flags} = this.parse(Publish)
     if (process.platform === 'win32') throw new Error('publish:s3 does not function on windows')
     const {channel} = flags
-    const root = path.resolve(flags.root)
+    const root = await qq.pkgDir(path.resolve(flags.root))
+    if (!root) throw new Error(`package root not found in ${path.resolve(flags.root)}`)
 
+    action('building vanilla')
     this.buildConfig = await Tarballs.build(root, channel)
-    const {s3Config, targets, vanilla, xz, dist} = this.buildConfig
+    const {s3Config, targets, vanilla, dist, version} = this.buildConfig
     if (!s3Config.bucket) throw new Error('must set oclif.update.s3.bucket in package.json')
     const S3Options = {
       Bucket: s3Config.bucket,
       ACL: 'public-read',
     }
+    if (targets.length) action('building targets')
     for (let target of targets) await this.uploadNodeBinary(target)
     const ManifestS3Options = {...S3Options, CacheControl: 'max-age=86400', ContentType: 'application/json'}
-    const uploadTarball = async (tarball: {gz: string, xz: string}) => {
+    const uploadTarball = async (tarball: {gz: string, xz?: string}) => {
       const TarballS3Options = {...S3Options, CacheControl: 'max-age=604800'}
       await s3.uploadFile(dist(tarball.gz), {...TarballS3Options, ContentType: 'application/gzip', Key: tarball.gz})
-      if (xz) await s3.uploadFile(dist(tarball.xz), {...TarballS3Options, ContentType: 'application/x-xz', Key: tarball.xz})
+      if (tarball.xz) await s3.uploadFile(dist(tarball.xz), {...TarballS3Options, ContentType: 'application/x-xz', Key: tarball.xz})
     }
+    action('uploading vanilla')
     await uploadTarball(vanilla.tarball)
+    if (targets.length) action('uploading targets')
     for (const target of targets) {
       await uploadTarball(target.keys.tarball)
       await s3.uploadFile(dist(target.keys.manifest), {...ManifestS3Options, Key: target.keys.manifest})
     }
-    action('uploading manifest')
+    action('uploading main manifest')
     await s3.uploadFile(dist(vanilla.manifest), {...ManifestS3Options, Key: vanilla.manifest})
+    action(`published ${version}`)
   }
 
   private async uploadNodeBinary(target: Tarballs.ITarget) {
@@ -61,7 +67,6 @@ export default class Publish extends Command {
       await s3.headObject({Bucket: s3Config.bucket!, Key})
     } catch (err) {
       if (err.code !== 'NotFound') throw err
-      action(`uploading ${key}`)
       let output = dist(key)
       output = await Tarballs.fetchNodeBinary({nodeVersion, platform, arch, output, tmp})
       await qq.x('gzip', ['-f', output])

src/s3.ts

@@ -1,10 +1,14 @@
+// tslint:disable no-console
+
 import * as S3 from 'aws-sdk/clients/s3'
 import * as fs from 'fs-extra'
+import * as qq from 'qqjs'
 
-const debug = require('debug')('@oclif/dev-cli/s3')
+const debug = require('debug')('oclif-dev:s3')
 
 export const uploadFile = (local: string, options: S3.Types.PutObjectRequest) => new Promise((resolve, reject) => {
-  debug('uploadFile', local, `s3://${options.Bucket}/${options.Key}`)
+  console.log('uploadFile', qq.prettifyPaths(local), `s3://${options.Bucket}/${options.Key}`)
+  debug('uploadFile', qq.prettifyPaths(local), `s3://${options.Bucket}/${options.Key}`)
   options.Body = fs.createReadStream(local)
   s3().upload(options, err => {
     if (err) reject(err)
@@ -20,21 +24,21 @@ export const headObject = (options: S3.Types.HeadObjectRequest) => new Promise<S
   })
 })
 
-export const getObject = (options: S3.Types.GetObjectRequest) => new Promise<S3.GetObjectOutput>((resolve, reject) => {
-  debug('getObject', `s3://${options.Bucket}/${options.Key}`)
-  s3().getObject(options, (err, data) => {
-    if (err) reject(err)
-    else resolve(data)
-  })
-})
-
-export const listObjects = (options: S3.Types.ListObjectsV2Request) => new Promise<S3.ListObjectsV2Output>((resolve, reject) => {
-  debug('listObjects', `s3://${options.Bucket}/${options.Prefix}`)
-  s3().listObjectsV2(options, (err, objects) => {
-    if (err) reject(err)
-    else resolve(objects)
-  })
-})
+// export const getObject = (options: S3.Types.GetObjectRequest) => new Promise<S3.GetObjectOutput>((resolve, reject) => {
+//   debug('getObject', `s3://${options.Bucket}/${options.Key}`)
+//   s3().getObject(options, (err, data) => {
+//     if (err) reject(err)
+//     else resolve(data)
+//   })
+// })
+
+// export const listObjects = (options: S3.Types.ListObjectsV2Request) => new Promise<S3.ListObjectsV2Output>((resolve, reject) => {
+//   debug('listObjects', `s3://${options.Bucket}/${options.Prefix}`)
+//   s3().listObjectsV2(options, (err, objects) => {
+//     if (err) reject(err)
+//     else resolve(objects)
+//   })
+// })
 
 export namespace upload {
   export interface Options {

src/tarballs/bin.ts

@@ -43,7 +43,9 @@ if [ -z "\$${redirectedEnvVar}" ] && [ -x "\$BIN_PATH" ] && [[ ! "\$DIR/${t.conf
   fi
   ${redirectedEnvVar}=1 "\$BIN_PATH" "\$@"
 else
-  if [ -x "$(command -v "\$DIR/node")" ]; then
+  if [ -x "$(command -v "\$XDG_DATA_HOME/oclif/node/node-custom")" ]; then
+    NODE="\$XDG_DATA_HOME/oclif/node/node-custom"
+  elif [ -x "$(command -v "\$DIR/node")" ]; then
     NODE="\$DIR/node"
   elif [ -x "$(command -v "\$XDG_DATA_HOME/oclif/node/node-${t.nodeVersion}")" ]; then
     NODE="\$XDG_DATA_HOME/oclif/node/node-${t.nodeVersion}"
@@ -56,7 +58,7 @@ else
   if [ "\$DEBUG" == "*" ]; then
     echo ${binPathEnvVar}="\$DIR/${t.config.bin}" "\$NODE" "\$DIR/run" "\$@"
   fi
-  ${binPathEnvVar}="\$DIR/${t.config.bin}" "\$NODE" "\$@"
+  ${binPathEnvVar}="\$DIR/${t.config.bin}" "\$NODE" "\$DIR/run" "\$@"
 fi
 `)
     await qq.chmod(bin, 0o755)

src/tarballs/build.ts

@@ -2,25 +2,10 @@ import * as path from 'path'
 import * as qq from 'qqjs'
 
 import {writeBinScripts} from './bin'
-import {buildConfig, ITarget} from './config'
+import {buildConfig, ITarget, IVersionManifest} from './config'
 import {log} from './log'
 import {fetchNodeBinary} from './node'
 
-export interface IManifest {
-  version: string
-  channel: string
-  sha256gz: string
-  sha256xz?: string
-}
-
-export interface IVersionManifest extends IManifest {
-  rollout?: number
-  node: {
-    compatible: string
-    recommended: string
-  }
-}
-
 const pack = async (from: string, to: string) => {
   const prevCwd = qq.cwd()
   qq.cd(path.dirname(from))
@@ -34,7 +19,7 @@ const pack = async (from: string, to: string) => {
 
 export async function build(root: string, channel = 'stable'): ReturnType<typeof buildConfig> {
   const t = await buildConfig(root, channel)
-  const {config, baseWorkspace, nodeVersion, version, xz, dist, targetWorkspace, vanilla, targets, updateConfig} = t
+  const {config, baseWorkspace, nodeVersion, version, dist, targetWorkspace, vanilla, targets, updateConfig} = t
   const prevCwd = qq.cwd()
   const packCLI = async () => {
     qq.cd(root)
@@ -55,6 +40,7 @@ export async function build(root: string, channel = 'stable'): ReturnType<typeof
     const pjson = await qq.readJSON('package.json')
     pjson.version = version
     pjson.channel = channel
+    pjson.oclif.update.s3.bucket = t.s3Config.bucket
     await qq.writeJSON('package.json', pjson)
   }
   const addDependencies = async () => {
@@ -94,29 +80,45 @@ export async function build(root: string, channel = 'stable'): ReturnType<typeof
       tmp: qq.join([config.root, 'tmp']),
     })
     await pack(workspace, dist(target.keys.tarball.gz))
-    if (xz) await pack(workspace, dist(target.keys.tarball.xz))
-    const manifest: IManifest = {
+    if (target.keys.tarball.xz) await pack(workspace, dist(target.keys.tarball.xz))
+    target.manifest = {
       version,
       channel,
+      gz: target.urls.tarball.gz,
+      xz: target.urls.tarball.xz,
       sha256gz: await qq.hash('sha256', dist(target.keys.tarball.gz)),
-      sha256xz: xz ? (await qq.hash('sha256', dist(target.keys.tarball.xz))) : undefined,
+      sha256xz: target.keys.tarball.xz ? (await qq.hash('sha256', dist(target.keys.tarball.xz))) : undefined,
     }
-    await qq.writeJSON(dist(target.keys.manifest), manifest)
+    await qq.writeJSON(dist(target.keys.manifest), target.manifest)
   }
   const buildBaseTarball = async () => {
     await pack(baseWorkspace, dist(vanilla.tarball.gz))
-    if (xz) await pack(baseWorkspace, dist(vanilla.tarball.xz))
-    await qq.writeJSON(dist(vanilla.manifest), {
+    if (vanilla.tarball.xz) await pack(baseWorkspace, dist(vanilla.tarball.xz))
+  }
+  const buildBaseManifest = async () => {
+    const manifest: IVersionManifest = {
       version,
       channel,
+      gz: vanilla.urls.gz,
+      xz: vanilla.urls.xz,
       sha256gz: await qq.hash('sha256', dist(vanilla.tarball.gz)),
-      sha256xz: xz ? await qq.hash('sha256', dist(vanilla.tarball.xz)) : undefined,
-      rollout: typeof updateConfig.autoupdate === 'object' && updateConfig.autoupdate.rollout,
+      sha256xz: vanilla.tarball.xz ? await qq.hash('sha256', dist(vanilla.tarball.xz)) : undefined,
+      rollout: (typeof updateConfig.autoupdate === 'object' && updateConfig.autoupdate.rollout) as number,
       node: {
         compatible: config.pjson.engines.node,
         recommended: nodeVersion,
       },
-    } as IVersionManifest)
+      targets: targets.reduce((targets, t) => {
+        targets![`${t.platform}-${t.arch}`] = {
+          gz: t.urls.tarball.gz,
+          xz: t.urls.tarball.xz,
+          sha256gz: t.manifest!.sha256gz,
+          sha256xz: t.manifest!.sha256xz,
+        }
+        return targets
+      }, {} as IVersionManifest['targets'])
+    }
+    await qq.writeJSON(dist(vanilla.manifest), manifest)
   }
   log(`packing ${config.bin} to ${baseWorkspace}`)
   await extractCLI(await packCLI())
@@ -126,6 +128,7 @@ export async function build(root: string, channel = 'stable'): ReturnType<typeof
   await writeBinScripts(t)
   await buildBaseTarball()
   for (let target of targets) await buildTarget(target)
+  await buildBaseManifest()
   qq.cd(prevCwd)
   return t
 }

src/tarballs/config.ts

@@ -2,17 +2,42 @@ import * as Config from '@oclif/config'
 import * as _ from 'lodash'
 import * as path from 'path'
 import * as qq from 'qqjs'
+import {URL} from 'url'
 
 export interface ITarget {
   platform: string
   arch: string
+  urls: {
+    tarball: { gz: string, xz?: string }
+  }
   keys: {
-    tarball: { gz: string, xz: string }
+    tarball: { gz: string, xz?: string }
     manifest: string
   }
+  manifest?: IManifest
+}
+
+export interface IManifest {
+  version: string
+  channel: string
+  gz: string
+  xz?: string
+  sha256gz: string
+  sha256xz?: string
 }
 
-function gitSha(cwd: string, options: {short?: boolean} = {}) {
+export interface IVersionManifest extends IManifest {
+  rollout?: number
+  node: {
+    compatible: string
+    recommended: string
+  }
+  targets?: {
+    [target: string]: Pick<IManifest, 'gz' | 'xz' | 'sha256gz' | 'sha256xz'>
+  }
+}
+
+export function gitSha(cwd: string, options: {short?: boolean} = {}) {
   const args = options.short ? ['rev-parse', '--short', 'HEAD'] : ['rev-parse', 'HEAD']
   return qq.x.stdout('git', args, {cwd})
 }
@@ -29,30 +54,40 @@ export async function buildConfig(root: string, channel: string) {
   const version = channel === 'stable' ? config.version : `${config.version}-${channel}.${_gitSha}`
   const tmp = await Tmp(config)
   const updateConfig = config.pjson.oclif.update
+  const s3Host = updateConfig.s3.host!
   const templateOpts = {
     config,
     channel,
     version,
     name: config.name,
     bin: config.bin,
   }
-  const vanillaTarball = _.template(updateConfig.s3.templates.vanillaTarball)(templateOpts)
+  const vanillaTarball: {gz: string, xz?: string} = {gz: _.template(updateConfig.s3.templates.vanillaTarball)(templateOpts) + '.tar.gz'}
+  const gzUrl = new URL(s3Host)
+  gzUrl.pathname = path.join(gzUrl.pathname, vanillaTarball.gz)
+  const vanillaUrls: {gz: string, xz?: string} = {gz: gzUrl.toString()}
+  const xz = updateConfig.s3.xz
+  if (xz) {
+    vanillaTarball.xz = vanillaTarball.gz.replace(/\.gz$/, '.xz')
+    vanillaUrls.xz = vanillaUrls.gz.replace(/\.gz$/, '.xz')
+  }
   const tConfig = {
     root,
     gitSha: _gitSha,
     config,
     vanilla: {
-      tarball: {gz: vanillaTarball + '.tar.gz', xz: vanillaTarball + '.tar.xz'},
+      tarball: vanillaTarball,
+      urls: vanillaUrls,
       manifest: _.template(updateConfig.s3.templates.vanillaManifest)(templateOpts),
     },
     tmp,
     updateConfig,
     version,
     channel,
+    xz,
     dist: (...args: string[]) => path.join(config.root, 'dist', ...args),
     s3Config: updateConfig.s3,
     gz: updateConfig.s3.gz === false,
-    xz: updateConfig.s3.xz,
     nodeVersion: updateConfig.node.version || process.versions.node,
     baseWorkspace: path.join(config.root, 'tmp', config.bin),
     targetWorkspace(platform: string, arch: string) {
@@ -62,13 +97,24 @@ export async function buildConfig(root: string, channel: string) {
       const [platform, arch] = t.split('-')
       const key = _.template(updateConfig.s3.templates.platformTarball)({...templateOpts, platform, arch})
       const manifest = _.template(updateConfig.s3.templates.platformManifest)({...templateOpts, platform, arch})
+      const keys: ITarget['keys'] = {
+        manifest,
+        tarball: {gz: key + '.tar.gz'},
+      }
+      const gzUrl = new URL(s3Host)
+      gzUrl.pathname = path.join(gzUrl.pathname, keys.tarball.gz)
+      const urls: ITarget['urls'] = {
+        tarball: {gz: gzUrl.toString()}
+      }
+      if (xz) {
+        keys.tarball.xz = keys.tarball.gz.replace(/\.gz$/, '.xz')
+        urls.tarball.xz = urls.tarball.gz.replace(/\.gz$/, '.xz')
+      }
       return {
         platform,
         arch,
-        keys: {
-          manifest,
-          tarball: {gz: key + '.tar.gz', xz: key + '.tar.xz'},
-        },
+        keys,
+        urls,
       }
     }),
   }

test/commands/pack.test.ts

@@ -9,6 +9,10 @@ describe('pack', () => {
   beforeEach(async () => {
     await qq.rm('dist')
   })
+  afterEach(async () => {
+    qq.cd([__dirname, '../..'])
+    await qq.rm('dist')
+  })
 
   skipIfWindows
   .command(['pack'])