feat(nlu-cli): config file with schema for both nlu and lang server

.gitignore

@@ -75,4 +75,4 @@ jspm_packages/
 dist/
 
 
-config.json
+*.config.json

packages/nlu-cli/package.json

@@ -11,10 +11,12 @@
     "@botpress/logger": "*",
     "@botpress/nlu-server": "*",
     "decamelize": "5.0.1",
+    "json-schema": "^0.4.0",
     "wtfnode": "^0.9.1",
     "yargs": "^17.2.1"
   },
   "devDependencies": {
+    "@types/json-schema": "^7.0.9",
     "@types/node": "^12.13.0",
     "@types/wtfnode": "^0.7.0",
     "@types/yargs": "^17.0.4",

packages/nlu-cli/src/app-data.ts

@@ -0,0 +1,16 @@
+import path from 'path'
+
+export function getAppDataPath() {
+  const homeDir = process.env.APP_DATA_PATH || process.env.HOME || process.env.APPDATA
+  if (homeDir) {
+    if (process.platform === 'darwin') {
+      return path.join(homeDir, 'Library', 'Application Support', 'botpress')
+    }
+
+    return path.join(homeDir, 'botpress')
+  }
+
+  const errorMsg = `Could not determine your HOME directory.
+Please set the environment variable "APP_DATA_PATH", then start Botpress`
+  throw new Error(errorMsg)
+}

packages/nlu-cli/src/config-file/index.ts

@@ -0,0 +1,37 @@
+import fse from 'fs-extra'
+import { validate } from 'json-schema'
+import { YargsSchema, YargsArgv } from '../yargs-utils'
+import { generateSchema } from './schema'
+
+interface WriteConfigFileProps<S extends YargsSchema> {
+  schemaLocation: string
+  fileLocation: string
+  yargSchema: S
+}
+
+interface ReadConfigFileProps<S extends YargsSchema> {
+  fileLocation: string
+  yargSchema: S
+}
+
+export const writeConfigFile = async <S extends YargsSchema>(props: WriteConfigFileProps<S>): Promise<void> => {
+  const { yargSchema, schemaLocation, fileLocation } = props
+  const schema = generateSchema(yargSchema)
+  const jsonConfig = { $schema: schemaLocation }
+  await fse.writeFile(schemaLocation, JSON.stringify(schema, null, 2))
+  await fse.writeFile(fileLocation, JSON.stringify(jsonConfig, null, 2))
+}
+
+export const readConfigFile = async <S extends YargsSchema>(props: ReadConfigFileProps<S>): Promise<YargsArgv<S>> => {
+  const { fileLocation, yargSchema } = props
+  const configFileContent = await fse.readFile(fileLocation, 'utf8')
+  const { $schema, ...parsedConfigFile } = JSON.parse(configFileContent)
+  const schema = generateSchema(yargSchema)
+  const validationResult = validate(parsedConfigFile, schema)
+  const { valid, errors } = validationResult
+  if (!valid) {
+    const errorMsg = errors.map((err) => `${err.property} ${err.message}`).join('\n')
+    throw new Error(errorMsg)
+  }
+  return parsedConfigFile
+}

packages/nlu-cli/src/config-file/schema.ts

@@ -0,0 +1,19 @@
+import { JSONSchema7, JSONSchema7Definition } from 'json-schema'
+import { Dictionary } from 'lodash'
+import { YargsSchema } from '../yargs-utils'
+
+export const generateSchema = (yargSchema: YargsSchema): JSONSchema7 => {
+  const properties: Dictionary<JSONSchema7Definition> = {}
+  for (const param in yargSchema) {
+    const yargProp = yargSchema[param]
+    properties[param] = {
+      type: yargProp.type
+    }
+  }
+
+  const schema: JSONSchema7 = {
+    type: 'object',
+    properties
+  }
+  return schema
+}

packages/nlu-cli/src/index.ts

@@ -2,42 +2,112 @@ import './rewire'
 import { run as runLanguageServer, download as downloadLang, version as langServerVersion } from '@botpress/lang-server'
 import { makeLogger } from '@botpress/logger'
 import { run as runNLUServer, version as nluServerVersion } from '@botpress/nlu-server'
+import path from 'path'
 import yargs from 'yargs'
+import { getAppDataPath } from './app-data'
+import { writeConfigFile, readConfigFile } from './config-file'
 import { nluServerParameters, langServerParameters, langDownloadParameters } from './parameters'
-import { parseEnv } from './parameters/parse-env'
+import { parseEnv } from './parse-env'
 
 void yargs
   .version(false)
-  .command(['nlu', '$0'], 'Launch a local standalone nlu server', nluServerParameters, (argv) => {
-    const baseLogger = makeLogger()
-    if (argv.version) {
-      baseLogger.sub('Version').info(nluServerVersion)
-      return
-    }
+  .command(['nlu', '$0'], 'Launch a local standalone nlu server', (yargs) => {
+    return yargs
+      .command(
+        '$0',
+        'Launch a local standalone nlu server',
+        {
+          version: {
+            description: "Prints the NLU Server's version",
+            type: 'boolean',
+            default: false
+          },
+          config: {
+            description: 'Path to your config file. If defined, rest of the CLI arguments are ignored.',
+            type: 'string',
+            alias: 'c'
+          },
+          ...nluServerParameters
+        },
+        async (argv) => {
+          const baseLogger = makeLogger({ prefix: 'NLU' })
+          if (argv.version) {
+            baseLogger.sub('Version').info(nluServerVersion)
+            return
+          }
+          if (argv.config) {
+            const fileArgs = await readConfigFile({
+              fileLocation: argv.config,
+              yargSchema: nluServerParameters
+            })
+            argv = { ...fileArgs, ...argv }
+          }
 
-    argv = parseEnv(nluServerParameters, argv)
-    void runNLUServer(argv).catch((err) => {
-      baseLogger.sub('Exit').attachError(err).critical('NLU Server exits after an error occured.')
-      process.exit(1)
-    })
+          argv = { ...parseEnv(nluServerParameters), ...argv }
+          void runNLUServer(argv).catch((err) => {
+            baseLogger.sub('Exit').attachError(err).critical('NLU Server exits after an error occured.')
+            process.exit(1)
+          })
+        }
+      )
+      .command('init', 'create configuration file in current working directory', {}, (argv) => {
+        const cachePath = getAppDataPath()
+        return writeConfigFile({
+          fileLocation: path.join(process.cwd(), 'nlu.config.json'),
+          schemaLocation: path.join(cachePath, 'nlu.config.schema.json'),
+          yargSchema: nluServerParameters
+        })
+      })
   })
   .command('lang', 'Launch a local language server', (yargs) => {
     const baseLogger = makeLogger({ prefix: 'LANG' })
     return yargs
-      .command('$0', 'Launch a local language server', langServerParameters, (argv) => {
-        if (argv.version) {
-          baseLogger.sub('Version').info(langServerVersion)
-          return
-        }
+      .command(
+        '$0',
+        'Launch a local language server',
+        {
+          version: {
+            description: "Prints the Lang Server's version",
+            type: 'boolean',
+            default: false
+          },
+          config: {
+            description: 'Path to your config file. If defined, rest of the CLI arguments are ignored.',
+            type: 'string',
+            alias: 'c'
+          },
+          ...langServerParameters
+        },
+        async (argv) => {
+          if (argv.version) {
+            baseLogger.sub('Version').info(langServerVersion)
+            return
+          }
+          if (argv.config) {
+            const fileArgs = await readConfigFile({
+              fileLocation: argv.config,
+              yargSchema: langServerParameters
+            })
+            argv = { ...fileArgs, ...argv }
+          }
 
-        argv = parseEnv(langServerParameters, argv)
-        void runLanguageServer(argv).catch((err) => {
-          baseLogger.sub('Exit').attachError(err).critical('Language Server exits after an error occured.')
-          process.exit(1)
+          argv = { ...parseEnv(langServerParameters), ...argv }
+          void runLanguageServer(argv).catch((err) => {
+            baseLogger.sub('Exit').attachError(err).critical('Language Server exits after an error occured.')
+            process.exit(1)
+          })
+        }
+      )
+      .command('init', 'create configuration file in current working directory', {}, (argv) => {
+        const cachePath = getAppDataPath()
+        return writeConfigFile({
+          fileLocation: path.join(process.cwd(), 'lang.config.json'),
+          schemaLocation: path.join(cachePath, 'lang.config.schema.json'),
+          yargSchema: langServerParameters
         })
       })
       .command('download', 'Download a language model for lang and dim', langDownloadParameters, (argv) => {
-        argv = parseEnv(langDownloadParameters, argv)
+        argv = { ...parseEnv(langDownloadParameters), ...argv }
         void downloadLang(argv).catch((err) => {
           baseLogger.sub('Exit').attachError(err).critical('Language Server exits after an error occured.')
           process.exit(1)

packages/nlu-cli/src/parameters/lang-download.ts

@@ -1,4 +1,4 @@
-import { asYargs } from './yargs-utils'
+import { asYargs } from '../yargs-utils'
 
 export const parameters = asYargs({
   langDir: {

packages/nlu-cli/src/parameters/lang-server.ts

@@ -1,11 +1,6 @@
-import { asYargs } from './yargs-utils'
+import { asYargs } from '../yargs-utils'
 
 export const parameters = asYargs({
-  version: {
-    description: "Prints the Lang Server's version",
-    type: 'boolean',
-    default: false
-  },
   port: {
     description: 'The port to listen to',
     type: 'number'

packages/nlu-cli/src/parameters/nlu-server.ts

@@ -1,16 +1,6 @@
-import { asYargs } from './yargs-utils'
+import { asYargs } from '../yargs-utils'
 
 export const parameters = asYargs({
-  version: {
-    description: "Prints the NLU Server's version",
-    type: 'boolean',
-    default: false
-  },
-  config: {
-    description: 'Path to your config file. If defined, rest of the CLI arguments are ignored.',
-    type: 'string',
-    alias: 'c'
-  },
   port: {
     description: 'The port to listen to',
     type: 'number'

packages/nlu-cli/src/parameters/parse-env.ts → packages/nlu-cli/src/parse-env.ts

@@ -1,18 +1,13 @@
 import decamelize from 'decamelize'
 import yargs from 'yargs'
 import yn from 'yn'
-import { YargsParameters } from './yargs-utils'
-
-type Argv<T extends YargsParameters> = yargs.Arguments<yargs.InferredOptionTypes<T>>
-
-const isUndefined = <T>(x: T | undefined): x is undefined => x === undefined
-const isDefined = <T>(x: T | undefined): x is T => x !== undefined
+import { YargsSchema } from './yargs-utils'
 
 const parseSingleEnv = <O extends yargs.Options>(
   yargSchema: O,
   envVarValue: string | undefined
 ): yargs.InferredOptionType<O> | undefined => {
-  if (isUndefined(envVarValue)) {
+  if (envVarValue === undefined) {
     return
   }
 
@@ -42,15 +37,16 @@ const parseSingleEnv = <O extends yargs.Options>(
  * @param yargsSchema the yargs builder parameter that declares what named parameters are required
  * @param argv the filled argv datastructure returned by yargs
  */
-export const parseEnv = <T extends YargsParameters>(yargsSchema: T, argv: Argv<T>): Argv<T> => {
+export const parseEnv = <T extends YargsSchema>(yargsSchema: T): Partial<yargs.InferredOptionTypes<T>> => {
+  const returned: Partial<yargs.InferredOptionTypes<T>> = {}
   for (const param in yargsSchema) {
     const envVarName = decamelize(param, { preserveConsecutiveUppercase: true, separator: '_' }).toUpperCase()
     const envVarValue = process.env[envVarName]
     const schema = yargsSchema[param]
     const parsedEnvValue = parseSingleEnv(schema, envVarValue)
-    if (isUndefined(argv[param]) && isDefined(parsedEnvValue)) {
-      ;(argv as yargs.InferredOptionTypes<T>)[param] = parsedEnvValue
+    if (parsedEnvValue !== undefined) {
+      returned[param] = parsedEnvValue
     }
   }
-  return argv
+  return returned
 }

packages/nlu-cli/src/yargs-utils.ts

@@ -0,0 +1,12 @@
+import { Dictionary } from 'lodash'
+import yargs from 'yargs'
+
+export type YargsOptionType = Exclude<yargs.Options['type'], 'count'>
+export type YargsOption = yargs.Options & { type?: YargsOptionType }
+export type YargsSchema = Dictionary<YargsOption>
+
+export type YargsArgv<T extends YargsSchema> = yargs.Arguments<yargs.InferredOptionTypes<T>>
+
+export const asYargs = <T extends YargsSchema>(x: T): T => {
+  return x
+}

packages/nlu-server/src/bootstrap/config.ts

@@ -1,6 +1,5 @@
 import { LoggerLevel } from '@botpress/logger'
 import bytes from 'bytes'
-import fse from 'fs-extra'
 import { getAppDataPath } from '../app-data'
 import { NLUServerOptions, CommandLineOptions } from '../typings'
 
@@ -26,51 +25,10 @@ const DEFAULT_OPTIONS = (): NLUServerOptions => ({
   maxTraining: 2
 })
 
-export type ConfigSource = 'environment' | 'cli' | 'file'
-
-const readEnvJSONConfig = (): NLUServerOptions | null => {
-  const rawContent = process.env.NLU_SERVER_CONFIG
-  if (!rawContent) {
-    return null
-  }
-  try {
-    const parsedContent = JSON.parse(rawContent)
-    const defaults = DEFAULT_OPTIONS()
-    return { ...defaults, ...parsedContent }
-  } catch {
-    return null
-  }
-}
-
-const readFileConfig = async (configPath: string): Promise<NLUServerOptions> => {
-  try {
-    const rawContent = await fse.readFile(configPath, 'utf8')
-    const parsedContent = JSON.parse(rawContent)
-    const defaults = DEFAULT_OPTIONS()
-    return { ...defaults, ...parsedContent }
-  } catch (err) {
-    const e = new Error(`The following errored occured when reading config file "${configPath}": ${err.message}`)
-    e.stack = err.stack
-    throw e
-  }
-}
-
-export const getConfig = async (
-  cliOptions: CommandLineOptions
-): Promise<{ options: NLUServerOptions; source: ConfigSource }> => {
-  const envConfig = readEnvJSONConfig()
-  if (envConfig) {
-    return { options: envConfig, source: 'environment' }
-  }
-
-  if (cliOptions.config) {
-    const options = await readFileConfig(cliOptions.config)
-    return { options, source: 'file' }
-  }
-
+export const getConfig = async (cliOptions: CommandLineOptions): Promise<NLUServerOptions> => {
   const defaults = DEFAULT_OPTIONS()
   const options: NLUServerOptions = { ...defaults, ...cliOptions }
-  return { options, source: 'cli' }
+  return options
 }
 
 export const validateConfig = (options: NLUServerOptions) => {