feat(qna): qna maker integration

packages/functionals/botpress-qna/README.md

@@ -2,14 +2,14 @@
 
 Botpress Q&A is a Botpress module that adds the unified Q&A management interface to your bot admin panel.
 
-It relies on the NLU module for recognizing the questions. By default it also uses the `builtins` module to end the text response, but it's configurable (see below).
+It either relies on the NLU module or on [Microsoft QnA Maker](https://www.qnamaker.ai) for recognizing the questions. By default it also uses the `builtins` module to send the text response, but it's configurable (see below).
 
 # Installation
 
 ⚠️ **This module only works with the new [Botpress X](https://github.com/botpress/botpress).**
 
 - Install the peer dependencies and the module iteslf `yarn add @botpress/builtins @botpress/nlu @botpress/qna` (note: you can skip the `builtins` module if you use the custom text renderer.)
-- Configure [NLU](https://github.com/botpress/botpress/tree/master/packages/functionals/botpress-nlu#botpress-nlu-)
+- Configure [NLU](https://github.com/botpress/botpress/tree/master/packages/functionals/botpress-nlu#botpress-nlu-) or register at [Microsoft QnA Maker](https://www.qnamaker.ai) and provide the API key for it
 
 # Configuration
 
@@ -20,6 +20,8 @@ The following properties can be configured either in the `qna.json` file or usin
 | `qnaDir` | `QNA_DIR` | No | `./qna` | The directory where the Q&A data is stored.
 | `textRenderer` | `QNA_TEXT_RENDERER` | No | `#builtin_text` (requires `@botpress/builtins` to be installed) | The _renderer_ used to format the text responses.
 | `exportCsvEncoding` | `QNA_EXPORT_CSV_ENCODING` | No | `utf8` | Encoding for CSV that can be exported from Q&A module
+| `qnaMakerApiKey` | `QNA_MAKER_API_KEY` | No | | API-key for [Microsoft QnA Maker](https://www.qnamaker.ai). If provided QnA maker gets used to save items and search through them (instead of NLU-module)
+| `qnaMakerKnowledgebase` | `QNA_MAKER_KNOWLEDGEBASE` | No | `botpress` | Name of the QnA Maker knowledgebase to use
 
 # Usage
 
@@ -78,6 +80,16 @@ const questionsFlatExported = await bp.qna.export({ flat: true })
 // ]
 ```
 
+```js
+const answers = await bp.qna.answersOn('How can I reach you out?')
+// [ { questions: [ 'How can I reach you out?' ],
+//     answer: 'You could find us on Facebook!',
+//     id: 116,
+//     confidence: 100,
+//     enabled: true,
+//     action: 'text' } ]
+```
+
 ## Controlling if Q&A should intercept
 
 It may appear that it's not useful for Q&A to just intercept all the users' messages and try to match them against Q&A's intents. This can be customized by providing a hook to Q&A module that will prevent interception when returning `false` or a promise resolving to `false`.

packages/functionals/botpress-qna/package.json

@@ -55,13 +55,15 @@
     "webpack-node-externals": "^1.6.0"
   },
   "dependencies": {
+    "axios": "^0.18.0",
     "bluebird": "^3.5.1",
     "csv-parse": "^2.5.0",
     "iconv-lite": "^0.4.23",
     "json2csv": "^4.1.5",
     "lodash": "^4.17.4",
     "mkdirp": "^0.5.1",
     "moment": "^2.22.2",
+    "ms": "^2.1.1",
     "multer": "^1.3.0",
     "nanoid": "^1.2.0",
     "yn": "^2.0.0"

packages/functionals/botpress-qna/src/index.js

@@ -1,4 +1,5 @@
-import Storage from './storage'
+import NluStorage from './providers/nlu'
+import MicrosoftQnaMakerStorage from './providers/qnaMaker'
 import { processEvent } from './middleware'
 import * as parsers from './parsers.js'
 import _ from 'lodash'
@@ -10,7 +11,6 @@ import Promise from 'bluebird'
 import iconv from 'iconv-lite'
 import nanoid from 'nanoid'
 
-let storage
 let logger
 let shouldProcessMessage
 const csvUploadStatuses = {}
@@ -26,12 +26,15 @@ module.exports = {
   config: {
     qnaDir: { type: 'string', required: true, default: './qna', env: 'QNA_DIR' },
     textRenderer: { type: 'string', required: true, default: '#builtin_text', env: 'QNA_TEXT_RENDERER' },
-    exportCsvEncoding: { type: 'string', required: false, default: 'utf8', env: 'QNA_EXPORT_CSV_ENCODING' }
+    exportCsvEncoding: { type: 'string', required: false, default: 'utf8', env: 'QNA_EXPORT_CSV_ENCODING' },
+    qnaMakerApiKey: { type: 'string', required: false, env: 'QNA_MAKER_API_KEY' },
+    qnaMakerKnowledgebase: { type: 'string', required: false, default: 'botpress', env: 'QNA_MAKER_KNOWLEDGEBASE' }
   },
   async init(bp, configurator) {
     const config = await configurator.loadAll()
-    storage = new Storage({ bp, config })
-    await storage.initializeGhost()
+    const Storage = config.qnaMakerApiKey ? MicrosoftQnaMakerStorage : NluStorage
+    this.storage = new Storage({ bp, config })
+    await this.storage.initialize()
 
     logger = bp.logger
 
@@ -47,7 +50,7 @@ module.exports = {
             return next()
           }
         }
-        if (!await processEvent(event, { bp, storage, logger, config })) {
+        if (!await processEvent(event, { bp, storage: this.storage, logger, config })) {
           next()
         }
       },
@@ -57,6 +60,7 @@ module.exports = {
   },
   async ready(bp, configurator) {
     const config = await configurator.loadAll()
+    const storage = this.storage
     bp.qna = {
       /**
        * Parses and imports questions; consecutive questions with similar answer get merged
@@ -67,16 +71,17 @@ module.exports = {
        */
       async import(questions, { format = 'json', csvUploadStatusId } = {}) {
         recordCsvUploadStatus(csvUploadStatusId, 'Calculating diff with existing questions')
-        const existingQuestions = (await storage.getQuestions()).map(item =>
-          JSON.stringify(_.omit(item.data, 'enabled'))
-        )
+        const existingQuestions = (await storage.all()).map(item => JSON.stringify(_.omit(item.data, 'enabled')))
         const parsedQuestions = typeof questions === 'string' ? parsers[`${format}Parse`](questions) : questions
         const questionsToSave = parsedQuestions.filter(item => !existingQuestions.includes(JSON.stringify(item)))
 
-        let questionsSavedCount = 0
+        if (config.qnaMakerApiKey) {
+          return storage.insert(questionsToSave.map(question => ({ ...question, enabled: true })))
+        }
 
+        let questionsSavedCount = 0
         return Promise.each(questionsToSave, question =>
-          storage.saveQuestion({ ...question, enabled: true }, null, false).then(() => {
+          storage.insert({ ...question, enabled: true }).then(() => {
             questionsSavedCount += 1
             recordCsvUploadStatus(csvUploadStatusId, `Saved ${questionsSavedCount}/${questionsToSave.length} questions`)
           })
@@ -91,7 +96,7 @@ module.exports = {
        * @returns {Array.<{questions: Array, question: String, action: String, answer: String}>}
        */
       async export({ flat = false } = {}) {
-        const qnas = await storage.getQuestions()
+        const qnas = await storage.all()
 
         return qnas.flatMap(question => {
           const { data } = question
@@ -133,18 +138,26 @@ module.exports = {
        * @param {String} id - id of the question to look for
        * @returns {Object}
        */
-      getQuestion: storage.getQuestion.bind(storage)
+      getQuestion: storage.getQuestion.bind(storage),
+
+      /**
+       * @async
+       * Returns array of matchings questions-answers along with their confidence level
+       * @param {String} question - question to match against
+       * @returns {Array.<{questions: Array, answer: String, id: String, confidence: Number, metadata: Array}>}
+       */
+      answersOn: storage.answersOn.bind(storage)
     }
 
     const router = bp.getRouter('botpress-qna')
 
     router.get('/', async ({ query: { limit, offset } }, res) => {
       try {
-        const items = await storage.getQuestions({
+        const items = await this.storage.all({
           limit: limit ? parseInt(limit) : undefined,
           offset: offset ? parseInt(offset) : undefined
         })
-        const overallItemsCount = await storage.questionsCount()
+        const overallItemsCount = await this.storage.count()
         res.send({ items, overallItemsCount })
       } catch (e) {
         logger.error('QnA Error', e, e.stack)
@@ -154,7 +167,7 @@ module.exports = {
 
     router.post('/', async (req, res) => {
       try {
-        const id = await storage.saveQuestion(req.body)
+        const id = await this.storage.insert(req.body)
         res.send(id)
       } catch (e) {
         logger.error('QnA Error', e, e.stack)
@@ -164,7 +177,7 @@ module.exports = {
 
     router.put('/:question', async (req, res) => {
       try {
-        await storage.saveQuestion(req.body, req.params.question)
+        await this.storage.update(req.body, req.params.question)
         res.end()
       } catch (e) {
         logger.error('QnA Error', e, e.stack)
@@ -174,7 +187,7 @@ module.exports = {
 
     router.delete('/:question', async (req, res) => {
       try {
-        await storage.deleteQuestion(req.params.question)
+        await this.storage.delete(req.params.question)
         res.end()
       } catch (e) {
         logger.error('QnA Error', e, e.stack)
@@ -195,14 +208,12 @@ module.exports = {
       res.end(csvUploadStatusId)
       recordCsvUploadStatus(csvUploadStatusId, 'Deleting existing questions')
       if (yn(req.body.isReplace)) {
-        const questions = await storage.getQuestions()
-        await Promise.each(questions, ({ id }) => storage.deleteQuestion(id, false))
+        const questions = await this.storage.all()
+        await this.storage.delete(questions.map(({ id }) => id))
       }
 
       try {
         await bp.qna.import(req.file.buffer.toString(), { format: 'csv', csvUploadStatusId })
-        recordCsvUploadStatus(csvUploadStatusId, 'Syncing NLU-provider')
-        bp.nlu.provider.sync()
         recordCsvUploadStatus(csvUploadStatusId, 'Completed')
       } catch (e) {
         logger.error('QnA Error:', e)

packages/functionals/botpress-qna/src/middleware.js

@@ -1,40 +1,48 @@
-import { NLU_PREFIX } from './storage'
+import { NLU_PREFIX } from './providers/nlu'
 
 export const processEvent = async (event, { bp, storage, logger, config }) => {
-  // NB: we rely on NLU being loaded before we receive any event.
-  // I'm not sure yet if we can guarantee it
-  if (!event.nlu || !event.nlu.intent || !event.nlu.intent.startsWith(NLU_PREFIX)) {
-    return false
+  let answer
+  if (config.qnaMakerApiKey) {
+    answer = (await bp.qna.answersOn(event.text)).pop()
+    if (!answer) {
+      return false
+    }
+    logger.debug('QnA: matched QnA-maker question', answer.id)
+  } else {
+    // NB: we rely on NLU being loaded before we receive any event.
+    // I'm not sure yet if we can guarantee it
+    if (!(event.nlu || {}).intent || !event.nlu.intent.startsWith(NLU_PREFIX)) {
+      return false
+    }
+
+    logger.debug('QnA: matched NLU intent', event.nlu.intent.name)
+    const id = event.nlu.intent.name.substring(NLU_PREFIX.length)
+    answer = (await storage.getQuestion(id)).data
   }
 
-  logger.debug('QnA: matched NLU intent', event.nlu.intent.name)
-  const id = event.nlu.intent.name.substring(NLU_PREFIX.length)
-  const { data } = await storage.getQuestion(id)
-  // actually this shouldn't be the case as we delete intents
-  // for disabled questions
-  if (!data.enabled) {
+  if (!answer.enabled) {
     logger.debug('QnA: question disabled, skipping')
     return false
   }
 
-  if (data.action.includes('text')) {
-    logger.debug('QnA: replying to recognized question with plain text answer', id)
-    event.reply(config.textRenderer, { text: data.answer })
+  if (answer.action.includes('text')) {
+    logger.debug('QnA: replying to recognized question with plain text answer', answer.id)
+    event.reply(config.textRenderer, { text: answer.answer })
     // return `true` to prevent further middlewares from capturing the message
 
-    if (data.action === 'text') {
+    if (answer.action === 'text') {
       return true
     }
   }
 
-  if (data.action.includes('redirect')) {
-    logger.debug('QnA: replying to recognized question with redirect', id)
+  if (answer.action.includes('redirect')) {
+    logger.debug('QnA: replying to recognized question with redirect', answer.id)
     // TODO: This is used as the `stateId` by the bot template
     // Not sure if it's universal enough for every use-case but
     // I don't see a better alternative as of now
     const stateId = event.sessionId || event.user.id
-    logger.debug('QnA: jumping', stateId, data.redirectFlow, data.redirectNode)
-    await bp.dialogEngine.jumpTo(stateId, data.redirectFlow, data.redirectNode)
+    logger.debug('QnA: jumping', stateId, answer.redirectFlow, answer.redirectNode)
+    await bp.dialogEngine.jumpTo(stateId, answer.redirectFlow, answer.redirectNode)
     // We return false here because the we only jump to the right flow/node and let
     // the bot's natural middleware chain take care of processing the message the normal way
     return false

packages/functionals/botpress-qna/src/storage.js → packages/functionals/botpress-qna/src/providers/nlu.js

@@ -36,7 +36,7 @@ export default class Storage {
     this.qnaDir = config.qnaDir
   }
 
-  async initializeGhost() {
+  async initialize() {
     mkdirp.sync(path.resolve(this.projectDir, this.qnaDir))
     await this.ghost.addRootFolder(this.qnaDir, { filesGlob: '**/*.json' })
   }
@@ -47,7 +47,7 @@ export default class Storage {
     }
   }
 
-  async saveQuestion(data, id = null, syncNlu = true) {
+  async update(data, id) {
     id = id || getQuestionId(data)
     if (data.enabled) {
       await this.bp.nlu.storage.saveIntent(getIntentId(id), {
@@ -57,13 +57,28 @@ export default class Storage {
     } else {
       await this.bp.nlu.storage.deleteIntent(getIntentId(id))
     }
-    if (syncNlu) {
-      this.syncNlu()
-    }
+    this.syncNlu()
     await this.ghost.upsertFile(this.qnaDir, `${id}.json`, JSON.stringify({ id, data }, null, 2))
     return id
   }
 
+  async insert(qna) {
+    const ids = await Promise.all(
+      (_.isArray(qna) ? qna : [qna]).map(async data => {
+        const id = getQuestionId(data)
+        if (data.enabled) {
+          await this.bp.nlu.storage.saveIntent(getIntentId(id), {
+            entities: [],
+            utterances: normalizeQuestions(data.questions)
+          })
+        }
+        await this.ghost.upsertFile(this.qnaDir, `${id}.json`, JSON.stringify({ id, data }, null, 2))
+      })
+    )
+    this.syncNlu()
+    return ids
+  }
+
   async getQuestion(opts) {
     let filename
     if (typeof opts === 'string') {
@@ -76,27 +91,49 @@ export default class Storage {
     return JSON.parse(data)
   }
 
-  async questionsCount() {
+  async count() {
     const questions = await this.ghost.directoryListing(this.qnaDir, '.json')
     return questions.length
   }
 
-  async getQuestions({ limit, offset } = {}) {
+  async all({ limit, offset } = {}) {
     let questions = await this.ghost.directoryListing(this.qnaDir, '.json')
     if (typeof limit !== 'undefined' && typeof offset !== 'undefined') {
       questions = questions.slice(offset, offset + limit)
     }
     return Promise.map(questions, question => this.getQuestion({ filename: question }))
   }
 
-  async deleteQuestion(id, syncNlu = true) {
-    const data = await this.getQuestion(id)
-    if (data.data.enabled) {
-      await this.bp.nlu.storage.deleteIntent(getIntentId(id))
-      if (syncNlu) {
-        this.syncNlu()
-      }
+  async delete(qnaId) {
+    const ids = _.isArray(qnaId) ? qnaId : [qnaId]
+    if (ids.length === 0) {
+      return
     }
-    await this.ghost.deleteFile(this.qnaDir, `${id}.json`)
+    await Promise.all(
+      ids.map(async id => {
+        const data = await this.getQuestion(id)
+        if (data.data.enabled) {
+          await this.bp.nlu.storage.deleteIntent(getIntentId(id))
+        }
+        await this.ghost.deleteFile(this.qnaDir, `${id}.json`)
+      })
+    )
+    this.syncNlu()
+  }
+
+  async answersOn(text) {
+    const extract = await this.bp.nlu.provider.extract({ text })
+    const intents = _.chain([extract.intent, ...extract.intents])
+      .uniqBy('name')
+      .filter(({ name }) => name.startsWith('__qna__'))
+      .orderBy(['confidence'], ['desc'])
+      .value()
+
+    return Promise.all(
+      intents.map(async ({ name, confidence }) => {
+        const { data: { questions, answer } } = await this.getQuestion(name.replace('__qna__', ''))
+        return { questions, answer, confidence, id: name, metadata: [] }
+      })
+    )
   }
 }