feat(cli): various cli improvements (#144)

* chore(nlu-cli): download is a subcommand of lang

* chore(nlu-cli): move default values outside of cli

* chore(nlu-cli): extract yargs parameters in separate files

* feat(nlu-cli): all cli arguments can be passed as env variables

* feat(nlu-cli): config file with schema for both nlu and lang server (#143)

* progress

* feat(nlu-cli): config file with schema for both nlu and lang server

* chore(nlu-server): remove support for NLU_SERVER_CONFIG env var

* chore: write config schemas in appdata cache

* minor fix

* fix github workflows

* fix default nlu server config

* chore: default config now has doc = false

.github/workflows/binary.yml

@@ -27,7 +27,7 @@ jobs:
           mv ./dist/$bin_original_name ./nlu
       - name: Download language models
         run: |
-          ./nlu download --lang en --dim 25
+          ./nlu lang download --lang en --dim 25
       - name: Run Regression Test
         run: |
 

.github/workflows/bitfan.yml

@@ -18,8 +18,8 @@ jobs:
           yarn build
       - name: Download language models
         run: |
-          yarn start download --lang fr --dim 100
-          yarn start download --lang en --dim 100
+          yarn start lang download --lang fr --dim 100
+          yarn start lang download --lang en --dim 100
       - name: Run Regression Test
         run: |
           yarn start lang --dim 100 &

.gitignore

@@ -75,4 +75,4 @@ jspm_packages/
 dist/
 
 
-config.json
+*.config.json

.vscode/launch.json

@@ -13,7 +13,7 @@
       "console": "integratedTerminal",
       "sourceMaps": true,
       "autoAttachChildProcesses": true,
-      "args": ["--config=./config.json"],
+      "args": ["--config=./nlu.config.json"],
       "outFiles": ["${workspaceRoot}/packages/**/*.js", "${workspaceRoot}/node_modules/**/*.js"]
     },
     {

gulpfile.js

@@ -1,21 +1,18 @@
 const gulp = require('gulp')
 const package = require('./scripts/gulp.package')
-const config = require('./scripts/gulp.config')
 const release = require('./scripts/gulp.release')
 
 gulp.task('default', (cb) => {
   console.log(`
       Development Cheat Sheet
       ==================================
       yarn cmd package                      Packages Application in binaries
-      yarn cmd config                       Upsert new NLU server config file
       yarn cmd bump                         Bump version and update change log
       yarn cmd changelog                    Print change log
     `)
   cb()
 })
 
 gulp.task('package', package.package)
-gulp.task('config', config.upsertConfigFile)
 gulp.task('bump', release.bumpVersion)
 gulp.task('changelog', release.printChangeLog)

package.json

@@ -22,9 +22,8 @@
     ]
   },
   "scripts": {
-    "postinstall": "yarn cmd config",
     "cmd": "yarn run gulp",
-    "dev": "cross-env tsnd --transpile-only --watch ./packages/**/*.ts ./packages/nlu-cli/src/index.ts",
+    "dev": "cross-env tsnd --transpile-only --exit-child --watch ./packages/**/*.ts ./packages/nlu-cli/src/index.ts",
     "start": "cross-env node ./packages/nlu-cli/dist/index.js",
     "build": "tsc --build ./tsconfig.json",
     "clean": "yarn workspaces run clean",

packages/bitfan/src/services/bp-provider/stan-provider.ts

@@ -66,7 +66,7 @@ export class StanProvider {
   public async predict(utterances: string[]): Promise<PredictOutput[]> {
     const predOutput = await this._client.predict(APP_ID, this._modelId ?? '', { utterances })
     if (!predOutput.success) {
-      throw new Error(`An error occured at prediction: ${predOutput.error}.`)
+      throw new Error(`An error occured at prediction: ${predOutput.error.message}.`)
     }
     return predOutput.predictions
   }

packages/lang-server/src/config.ts

@@ -0,0 +1,41 @@
+import { LoggerLevel } from '@botpress/logger'
+import path from 'path'
+import { getAppDataPath } from './app-data'
+import { LangServerOptions, DownloadOptions, LangArgv, DownloadArgv } from './typings'
+
+const DEFAULT_LANG_DIR = () => {
+  const appDataPath = getAppDataPath()
+  return path.join(appDataPath, 'embeddings')
+}
+
+const DEFAULT_SERVER_OPTIONS = (): LangServerOptions => ({
+  port: 3100,
+  host: 'localhost',
+  langDir: DEFAULT_LANG_DIR(),
+  authToken: undefined,
+  adminToken: undefined,
+  limit: 0,
+  limitWindow: '1h',
+  metadataLocation: 'https://nyc3.digitaloceanspaces.com/botpress-public/embeddings/index.json',
+  offline: false,
+  dim: 100,
+  domain: 'bp',
+  verbose: LoggerLevel.Info,
+  logFilter: undefined
+})
+
+const DEFAULT_DOWNLOAD_OPTIONS = (lang: string): DownloadOptions => ({
+  langDir: DEFAULT_LANG_DIR(),
+  metadataLocation: 'https://nyc3.digitaloceanspaces.com/botpress-public/embeddings/index.json',
+  dim: 100,
+  domain: 'bp',
+  lang
+})
+
+export const getLangServerConfig = (argv: LangArgv): LangServerOptions => {
+  return { ...DEFAULT_SERVER_OPTIONS(), ...argv }
+}
+
+export const getDownloadConfig = (argv: DownloadArgv): DownloadOptions => {
+  return { ...DEFAULT_DOWNLOAD_OPTIONS(argv.lang), ...argv }
+}

packages/lang-server/src/download.ts

@@ -3,27 +3,19 @@ import { LanguageService } from '@botpress/nlu-engine'
 import cliProgress from 'cli-progress'
 import fse from 'fs-extra'
 import _ from 'lodash'
-import path from 'path'
 
-import { getAppDataPath } from './app-data'
 import DownloadManager from './application/download-manager'
+import { getDownloadConfig } from './config'
+import * as types from './typings'
 
-interface Argv {
-  langDir?: string
-  lang: string
-  dim: number
-  domain: string
-  metadataLocation: string
-}
-
-export default async (options: Argv) => {
+export const download: typeof types.download = async (argv: types.DownloadArgv) => {
+  const options = getDownloadConfig(argv)
   const baseLogger = makeLogger({
     level: LoggerLevel.Info,
     filters: undefined
   })
 
-  const appDataPath = getAppDataPath()
-  const languageDirectory = options.langDir || path.join(appDataPath, 'embeddings')
+  const languageDirectory = options.langDir
   await fse.ensureDir(languageDirectory)
 
   const launcherLogger = baseLogger.sub('Launcher')

packages/lang-server/src/index.ts

@@ -5,10 +5,11 @@ import _ from 'lodash'
 import path from 'path'
 
 import API, { APIOptions } from './api'
-import { getAppDataPath } from './app-data'
 import { LangApplication } from './application'
 import DownloadManager from './application/download-manager'
+import { getLangServerConfig } from './config'
 import { requireJSON } from './require-json'
+import * as types from './typings'
 
 const packageJsonPath = path.resolve(__dirname, '../package.json')
 const packageJson = requireJSON<{ version: string }>(packageJsonPath)
@@ -18,25 +19,9 @@ if (!packageJson) {
 
 const { version: pkgVersion } = packageJson
 
-export { default as download } from './download'
+export { download } from './download'
 export const version = pkgVersion
 
-export interface ArgV {
-  port: number
-  host: string
-  limit: number
-  limitWindow: string
-  langDir?: string
-  authToken?: string
-  adminToken?: string
-  metadataLocation: string
-  offline: boolean
-  dim: number
-  domain: string
-  verbose: number
-  logFilter: string[] | undefined
-}
-
 const wrapLogger = (logger: Logger): EngineLogger => {
   return {
     debug: (msg: string) => logger.debug(msg),
@@ -47,16 +32,14 @@ const wrapLogger = (logger: Logger): EngineLogger => {
   }
 }
 
-export const run = async (options: ArgV) => {
+export const run: typeof types.run = async (argv: types.LangArgv) => {
+  const options = getLangServerConfig(argv)
   const baseLogger = makeLogger({
     level: Number(options.verbose) !== NaN ? Number(options.verbose) : LoggerLevel.Info,
     filters: options.logFilter,
     prefix: 'LANG'
   })
 
-  const appDataPath = getAppDataPath()
-  options.langDir = options.langDir || path.join(appDataPath, 'embeddings')
-
   const launcherLogger = baseLogger.sub('Launcher')
   // Launcher always display
   launcherLogger.configure({

packages/lang-server/src/typings.d.ts

@@ -1,25 +1,29 @@
-export const run: (argv: {
+interface CommonOptions {
+  langDir: string
+  metadataLocation: string
+  dim: number
+  domain: string
+}
+
+export interface LangServerOptions extends CommonOptions {
   port: number
   host: string
   limit: number
   limitWindow: string
-  langDir?: string
   authToken?: string
   adminToken?: string
-  metadataLocation: string
   offline: boolean
-  dim: number
-  domain: string
   verbose: number
-  logFilter: string[] | undefined
-}) => Promise<void>
+  logFilter?: string[]
+}
 
-export const download: (argv: {
-  langDir?: string
+export interface DownloadOptions extends CommonOptions {
   lang: string
-  dim: number
-  domain: string
-  metadataLocation: string
-}) => Promise<void>
+}
+
+export type LangArgv = Partial<LangServerOptions>
+export type DownloadArgv = Partial<CommonOptions> & { lang: string }
 
 export const version: string
+export const run: (argv: LangArgv) => Promise<void>
+export const download: (argv: DownloadArgv) => Promise<void>

packages/nlu-cli/package.json

@@ -10,10 +10,13 @@
     "@botpress/lang-server": "*",
     "@botpress/logger": "*",
     "@botpress/nlu-server": "*",
+    "decamelize": "5.0.1",
+    "json-schema": "^0.4.0",
     "wtfnode": "^0.9.1",
     "yargs": "^17.2.1"
   },
   "devDependencies": {
+    "@types/json-schema": "^7.0.9",
     "@types/node": "^12.13.0",
     "@types/wtfnode": "^0.7.0",
     "@types/yargs": "^17.0.4",

packages/nlu-cli/src/app-data.ts

@@ -0,0 +1,16 @@
+import path from 'path'
+
+export function getAppDataPath() {
+  const homeDir = process.env.APP_DATA_PATH || process.env.HOME || process.env.APPDATA
+  if (homeDir) {
+    if (process.platform === 'darwin') {
+      return path.join(homeDir, 'Library', 'Application Support', 'botpress')
+    }
+
+    return path.join(homeDir, 'botpress')
+  }
+
+  const errorMsg = `Could not determine your HOME directory.
+Please set the environment variable "APP_DATA_PATH", then start Botpress`
+  throw new Error(errorMsg)
+}

packages/nlu-cli/src/config-file/index.ts

@@ -0,0 +1,37 @@
+import fse from 'fs-extra'
+import { validate } from 'json-schema'
+import { YargsSchema, YargsArgv } from '../yargs-utils'
+import { generateSchema } from './schema'
+
+interface WriteConfigFileProps<S extends YargsSchema> {
+  schemaLocation: string
+  fileLocation: string
+  yargSchema: S
+}
+
+interface ReadConfigFileProps<S extends YargsSchema> {
+  fileLocation: string
+  yargSchema: S
+}
+
+export const writeConfigFile = async <S extends YargsSchema>(props: WriteConfigFileProps<S>): Promise<void> => {
+  const { yargSchema, schemaLocation, fileLocation } = props
+  const schema = generateSchema(yargSchema)
+  const jsonConfig = { $schema: schemaLocation }
+  await fse.writeFile(schemaLocation, JSON.stringify(schema, null, 2))
+  await fse.writeFile(fileLocation, JSON.stringify(jsonConfig, null, 2))
+}
+
+export const readConfigFile = async <S extends YargsSchema>(props: ReadConfigFileProps<S>): Promise<YargsArgv<S>> => {
+  const { fileLocation, yargSchema } = props
+  const configFileContent = await fse.readFile(fileLocation, 'utf8')
+  const { $schema, ...parsedConfigFile } = JSON.parse(configFileContent)
+  const schema = generateSchema(yargSchema)
+  const validationResult = validate(parsedConfigFile, schema)
+  const { valid, errors } = validationResult
+  if (!valid) {
+    const errorMsg = errors.map((err) => `${err.property} ${err.message}`).join('\n')
+    throw new Error(errorMsg)
+  }
+  return parsedConfigFile
+}

packages/nlu-cli/src/config-file/schema.ts

@@ -0,0 +1,19 @@
+import { JSONSchema7, JSONSchema7Definition } from 'json-schema'
+import { Dictionary } from 'lodash'
+import { YargsSchema } from '../yargs-utils'
+
+export const generateSchema = (yargSchema: YargsSchema): JSONSchema7 => {
+  const properties: Dictionary<JSONSchema7Definition> = {}
+  for (const param in yargSchema) {
+    const yargProp = yargSchema[param]
+    properties[param] = {
+      type: yargProp.type
+    }
+  }
+
+  const schema: JSONSchema7 = {
+    type: 'object',
+    properties
+  }
+  return schema
+}