feat(stan): get stan ready for launch (#4341)

* feat(stan): added documentation logging on startup + few bugs * complete documentation with proper colors in terminal * full training input example * missing chalk bold statement * validate all routes * update bitfan version
botpress · Jan 8, 2021 · 793b9f7 · 793b9f7
1 parent 4f4bbc3
commit 793b9f7
Show file tree

Hide file tree

Showing 13 changed files with 334 additions and 117 deletions.
diff --git a/.github/workflows/nlu-regression.yml b/.github/workflows/nlu-regression.yml
@@ -21,7 +21,7 @@ jobs:
         run: |
           yarn add npm-cli-login
           ./node_modules/npm-cli-login/bin/npm-cli-login.js -r https://npm.pkg.github.com -u botpressops -p ${{ secrets.PAT_BITFAN }} -e ops@botpress.com
-          yarn add @botpress/bitfan@0.3.2
+          yarn add @botpress/bitfan@0.3.3
       - name: Run Regression Test
         run: |
           yarn start nlu --silent --ducklingEnabled=false & 

diff --git a/src/bp/nlu-server/api-mapper.ts b/src/bp/nlu-server/api-mapper.ts
@@ -123,7 +123,7 @@ function mapEntity(entity: NLU.Entity): EntityPrediction {
 
   return {
     name,
-    type: type.split('.')[0] as EntityType,
+    type,
     start,
     end,
     confidence,
@@ -151,7 +151,7 @@ function mapOutputSlot(slot: NLU.Slot): SlotPrediction {
     confidence,
     start,
     end,
-    entity: entity && mapEntity(entity),
+    entity: entity ? mapEntity(entity) : null,
     name,
     source,
     value

diff --git a/src/bp/nlu-server/api.rest b/src/bp/nlu-server/api.rest
@@ -12,81 +12,82 @@ Content-Type: application/json
 
 {
   "language": "en",
-  "topics": [
+  "intents": [
     {
-      "name": "global",
-      "intents": [
+      "name": "fruit-is-moldy",
+      "contexts": ["grocery"],
+      "utterances": [
+        "fruit is moldy",
+        "this fruit is moldy",
+        "this [banana](fruit) is not good to eat",
+        "theses [oranges](fruit) have passed",
+        "theses [grapes](fruit) look bad",
+        "theses [apples](fruit) look soo moldy"
+      ],
+      "slots": [
         {
-          "name": "fruit-is-moldy",
-          "examples": [
-            "fruit is moldy",
-            "this fruit is moldy",
-            "this [banana](fruit) is not good to eat",
-            "theses [oranges](fruit) have passed",
-            "theses [grapes](fruit) look bad",
-            "theses [apples](fruit) look soo moldy"
-          ],
-          "variables": [
-            {
-              "name": "moldy_fruit",
-              "types": ["fruits"]
-            }
-          ]
-        },
-        {
-          "name": "hello",
-          "variables": [],
-          "examples": [
-            "good day!",
-            "good morning",
-            "holla",
-            "bonjour",
-            "hey there",
-            "hi bot",
-            "hey bot",
-            "hey robot",
-            "hey!",
-            "hi",
-            "hello"
-          ]
-        },
-        {
-          "name": "talk-to-manager",
-          "examples": [
-            "talk to manager",
-            "I want to talk to the manager",
-            "Who's your boss?",
-            "Can talk to the person in charge?",
-            "I'd like to speak to your manager",
-            "Can I talk to your boss? plz",
-            "I wanna speak to manager please",
-            "let me speak to your boss or someone"
-          ],
-          "variables": []
-        },
+          "name": "moldy_fruit",
+          "entities": ["fruits"]
+        }
+      ]
+    },
+    {
+      "name": "hello",
+      "contexts": ["global", "grocery"],
+      "slots": [],
+      "utterances": [
+        "good day!",
+        "good morning",
+        "holla",
+        "bonjour",
+        "hey there",
+        "hi bot",
+        "hey bot",
+        "hey robot",
+        "hey!",
+        "hi",
+        "hello"
+      ]
+    },
+    {
+      "name": "talk-to-manager",
+      "contexts": ["grocery"],
+      "utterances": [
+        "talk to manager",
+        "I want to talk to the manager",
+        "Who's your boss?",
+        "Can talk to the person in charge?",
+        "I'd like to speak to your manager",
+        "Can I talk to your boss? plz",
+        "I wanna speak to manager please",
+        "let me speak to your boss or someone"
+      ],
+      "slots": []
+    },
+    {
+      "name": "where-is",
+      "contexts": ["grocery"],
+      "utterances": [
+        "where is [milk](thing_to_search) ?",
+        "where are [apples](thing_to_search) ?",
+        "can you help me find [apples](thing_to_search) ?",
+        "I'm searching for [pie](thing_to_search) ?",
+        "where is the [milk](thing_to_search) ?",
+        "where are the [milk](thing_to_search) ?"
+      ],
+      "slots": [
         {
-          "name": "where-is",
-          "examples": [
-            "where is [milk](thing_to_search) ?",
-            "where are [apples](thing_to_search) ?",
-            "can you help me find [apples](thing_to_search) ?",
-            "I'm searching for [pie](thing_to_search) ?",
-            "where is the [milk](thing_to_search) ?",
-            "where are the [milk](thing_to_search) ?"
-          ],
-          "variables": [
-            {
-              "name": "thing_to_search",
-              "types": ["fruits", "any"]
-            }
-          ]
+          "name": "thing_to_search",
+          "entities": ["fruits", "any"]
         }
       ]
     }
   ],
-  "enums": [
+  "contexts": ["grocery", "global"],
+  "entities": [
     {
       "name": "fruits",
+      "type": "list",
       "fuzzy": 0.9,
       "values": [
         { "name": "banana", "synonyms": ["bananas"] },
@@ -124,6 +125,10 @@ POST {{api}}/predict/{{modelId}}
 Content-Type: application/json
 
 {
-  "texts" : ["These grapes look moldy", "Can I talk with the person in charge?"],
+  "utterances" : [
+    "These grapes look moldy",
+    "Can I talk with the person in charge?",
+    "My flight is at 9 pm"
+  ],
   "password": "123456"
 }
diff --git a/src/bp/nlu-server/api.ts b/src/bp/nlu-server/api.ts
@@ -17,7 +17,7 @@ import removeNoneIntent from './remove-none'
 import TrainService from './train-service'
 import TrainSessionService from './train-session-service'
 import { PredictOutput } from './typings_v1'
-import validateInput from './validation/validate'
+import { validateCancelRequestInput, validatePredictInput, validateTrainInput } from './validation/validate'
 
 export interface APIOptions {
   host: string
@@ -44,7 +44,7 @@ const createExpressApp = (options: APIOptions): Application => {
   app.use(bodyParser.json({ limit: options.bodySize }))
 
   app.use((req, res, next) => {
-    res.header('X-Powered-By', 'Botpress')
+    res.header('X-Powered-By', 'Botpress NLU')
     debugRequest(`incoming ${req.path}`, { ip: req.ip })
     next()
   })
@@ -74,7 +74,7 @@ const createExpressApp = (options: APIOptions): Application => {
 
 export default async function(options: APIOptions, engine: Engine) {
   const app = createExpressApp(options)
-  const logger = new Logger('API', options.silent)
+  const logger = new Logger('API')
 
   const modelRepo = new ModelRepository(options.modelDir)
   await modelRepo.init()
@@ -89,7 +89,7 @@ export default async function(options: APIOptions, engine: Engine) {
 
   router.post('/train', async (req, res) => {
     try {
-      const input = await validateInput(req.body)
+      const input = await validateTrainInput(req.body)
       const { intents, entities, seed, language, password } = mapTrainInput(input)
 
       const pickedSeed = seed ?? Math.round(Math.random() * 10000)
@@ -126,9 +126,10 @@ export default async function(options: APIOptions, engine: Engine) {
         const model = await modelRepo.getModel(modelId, password ?? '')
 
         if (!model) {
-          return res
-            .status(404)
-            .send({ success: false, error: `no model or training could be found for modelId: ${stringId}` })
+          return res.status(404).send({
+            success: false,
+            error: `no model or training could be found for modelId: ${stringId}`
+          })
         }
 
         session = {
@@ -148,8 +149,7 @@ export default async function(options: APIOptions, engine: Engine) {
   router.post('/train/:modelId/cancel', async (req, res) => {
     try {
       const { modelId: stringId } = req.params
-      let { password } = req.body
-      password = password ?? ''
+      const { password } = await validateCancelRequestInput(req.body)
 
       const modelId = modelIdService.fromString(stringId)
       const session = trainSessionService.getTrainingSession(modelId, password)
@@ -159,7 +159,7 @@ export default async function(options: APIOptions, engine: Engine) {
         return res.send({ success: true })
       }
 
-      res.status(404).send({ success: true, error: `no current training for model id: ${stringId}` })
+      res.status(404).send({ success: false, error: `no current training for model id: ${stringId}` })
     } catch (err) {
       res.status(500).send({ success: false, error: err.message })
     }
@@ -168,15 +168,16 @@ export default async function(options: APIOptions, engine: Engine) {
   router.post('/predict/:modelId', async (req, res) => {
     try {
       const { modelId: stringId } = req.params
-      const { texts, password } = req.body
+      const { utterances, password } = await validatePredictInput(req.body)
 
-      if (!_.isArray(texts) || (options.batchSize > 0 && texts.length > options.batchSize)) {
+      if (!_.isArray(utterances) || (options.batchSize > 0 && utterances.length > options.batchSize)) {
         throw new Error(
-          `Batch size of ${texts.length} is larger than the allowed maximum batch size (${options.batchSize}).`
+          `Batch size of ${utterances.length} is larger than the allowed maximum batch size (${options.batchSize}).`
         )
       }
 
       const modelId = modelIdService.fromString(stringId)
+      // once the model is loaded, there's no more password check
       if (!engine.hasModel(modelId)) {
         const model = await modelRepo.getModel(modelId, password)
         if (!model) {
@@ -186,7 +187,7 @@ export default async function(options: APIOptions, engine: Engine) {
         await engine.loadModel(model)
       }
 
-      const rawPredictions: BpPredictOutput[] = await Promise.map(texts as string[], async utterance => {
+      const rawPredictions: BpPredictOutput[] = await Promise.map(utterances as string[], async utterance => {
         const detectedLanguage = await engine.detectLanguage(utterance, { [modelId.languageCode]: modelId })
         const spellChecked = await engine.spellCheck(utterance, modelId)
         const { entities, predictions } = await engine.predict(utterance, modelId)
@@ -212,4 +213,5 @@ export default async function(options: APIOptions, engine: Engine) {
   })
 
   logger.info(`NLU Server is ready at http://${options.host}:${options.port}/`)
+  options.silent && logger.silence()
 }
diff --git a/src/bp/nlu-server/index.ts b/src/bp/nlu-server/index.ts
@@ -21,6 +21,9 @@ import API, { APIOptions } from './api'
 
 const debug = DEBUG('api')
 
+const GH_TYPINGS_FILE = 'https://github.com/botpress/botpress/blob/master/src/bp/nlu-server/typings_v1.d.ts'
+const GH_TRAIN_INPUT_EXAMPLE = 'https://github.com/botpress/botpress/blob/master/src/bp/nlu-server/train-example.json'
+
 type ArgV = APIOptions & {
   languageURL: string
   languageAuthToken?: string
@@ -92,7 +95,7 @@ export default async function(options: ArgV) {
   const { nluVersion } = engine.getSpecifications()
 
   logger.info(chalk`========================================
-{bold ${center('Botpress NLU Server', 40, 9)}}
+{bold ${center('Botpress Standalone NLU', 40, 9)}}
 {dim ${center(`Version ${nluVersion}`, 40, 9)}}
 {dim ${center(`OS ${process.distro}`, 40, 9)}}
 ${_.repeat(' ', 9)}========================================`)
@@ -126,5 +129,65 @@ ${_.repeat(' ', 9)}========================================`)
     logger.info(`batch size: allowing up to ${options.batchSize} predictions in one call to POST /predict`)
   }
 
+  if (!options.silent) {
+    const { host, port } = options
+
+    const baseUrl = `http://${host}:${port}/v1`
+
+    logger.info(chalk`
+
+{bold {underline Available Routes}}
+
+{green /**
+ * Gets the current version of botpress core NLU. Usefull to test if your installation is working.
+ * @returns {bold version}: botpress core NLU version number.
+*/}
+{bold GET ${baseUrl}/info}
+
+{green /**
+  * Starts a training.
+  * @body_parameter {bold language} Language to use for training.
+  * @body_parameter {bold intents} Intents definitions.
+  * @body_parameter {bold contexts} All available contexts.
+  * @body_parameter {bold entities} Entities definitions.
+  * @body_parameter {bold password} Password to protect your model. {yellow ** Optionnal **}
+  * @body_parameter {bold seed} Number to seed random number generators used during training (beta feature). {yellow ** Optionnal **}
+  * @returns {bold modelId} A model id for futur API calls
+ */}
+{bold POST ${baseUrl}/train}
+
+{green /**
+  * Gets a training progress status.
+  * @path_parameter {bold modelId} The model id for which you seek the training progress.
+  * @query_parameter {bold password} The password protecting your model.
+  * @returns {bold session} A training session data structure with information on desired model.
+ */}
+{bold GET ${baseUrl}/train/:modelId?password=XXXXXX}
+
+{green /**
+  * Cancels a training.
+  * @path_parameter {bold modelId} The model id for which you want to cancel the training.
+  * @body_parameter {bold password} The password protecting your model.
+ */}
+{bold POST ${baseUrl}/train/:modelId/cancel}
+
+{green /**
+  * Perform prediction for a text input.
+  * @path_parameter {bold modelId} The model id you want to use for prediction.
+  * @body_parameter {bold password} The password protecting your model.
+  * @body_parameter {bold utterances} Array of text for which you want a prediction.
+  * @returns {bold predictions} Array of predictions; Each prediction is a data structure reprensenting our understanding of the text.
+ */}
+{bold POST ${baseUrl}/predict/:modelId}
+
+{bold For more detailed information on typings, see
+${GH_TYPINGS_FILE}}.
+
+{bold For a complete example on training input, see
+${GH_TRAIN_INPUT_EXAMPLE}}.
+
+    `)
+  }
+
   await API(options, engine)
 }