feat(qna): added support for content elements

modules/qna/package.json

@@ -15,28 +15,26 @@
       "react": "React",
       "react-dom": "ReactDOM",
       "react-bootstrap": "ReactBootstrap",
-      "botpress/elements-list": "ElementsList"
+      "botpress/elements-list": "ElementsList",
+      "botpress/utils": "BotpressUtils"
     }
   },
   "devDependencies": {
-    "@babel/helpers": "^7.4.3",
+    "@blueprintjs/core": "^3.15.1",
+    "@types/react-bootstrap": "^0.32.19",
+    "@types/node": "^10.11.3",
+    "@types/react": "^16.8.17",
+    "@types/react-dom": "^16.8.4",
     "module-builder": "../../build/module-builder"
   },
   "dependencies": {
     "axios": "^0.18.1",
     "bluebird": "^3.5.3",
     "classnames": "^2.2.6",
-    "csv-parse": "^2.5.0",
-    "iconv-lite": "^0.4.23",
     "joi": "^13.6.0",
-    "json2csv": "^4.3.5",
     "lodash": "^4.17.11",
     "moment": "^2.24.0",
-    "ms": "^2.1.1",
-    "multer": "^1.4.1",
     "nanoid": "^2.0.1",
-    "react-icons": "^3.7.0",
-    "react-select": "^1.2.1",
-    "yn": "^2.0.0"
+    "react-select": "^1.2.1"
   }
 }

modules/qna/src/backend/api.ts

@@ -2,14 +2,12 @@ import * as sdk from 'botpress/sdk'
 import { validate } from 'joi'
 import _ from 'lodash'
 import moment from 'moment'
-import multer from 'multer'
 import nanoid from 'nanoid'
-import yn from 'yn'
 
-import { QnaEntry, QnaItem } from './qna'
+import { QnaEntry } from './qna'
 import Storage from './storage'
-import { importQuestions, prepareExport } from './transfer'
-import { QnaDefSchema, QnaItemArraySchema } from './validation'
+import { importQuestions, prepareExport, prepareImport } from './transfer'
+import { QnaDefSchema } from './validation'
 
 export default async (bp: typeof sdk, botScopedStorage: Map<string, Storage>) => {
   const jsonUploadStatuses = {}
@@ -93,20 +91,33 @@ export default async (bp: typeof sdk, botScopedStorage: Map<string, Storage>) =>
 
   router.get('/export', async (req, res) => {
     const storage = botScopedStorage.get(req.params.botId)
-    const data: string = await prepareExport(storage)
+    const data: string = await prepareExport(storage, bp)
+
     res.setHeader('Content-Type', 'application/json')
     res.setHeader('Content-disposition', `attachment; filename=qna_${moment().format('DD-MM-YYYY')}.json`)
     res.end(data)
   })
 
-  const upload = multer()
-  router.post('/import', upload.single('json'), async (req, res) => {
+  router.post('/import/summary', async (req, res) => {
     const storage = botScopedStorage.get(req.params.botId)
+    const cmsIds = await storage.getAllContentElementIds()
+    const importData = await prepareImport(JSON.parse(req.body.fileContent))
+
+    res.send({
+      qnaCount: await storage.count(),
+      cmsCount: (cmsIds && cmsIds.length) || 0,
+      fileQnaCount: (importData.questions && importData.questions.length) || 0,
+      fileCmsCount: (importData.content && importData.content.length) || 0
+    })
+  })
 
+  router.post('/import', async (req, res) => {
     const uploadStatusId = nanoid()
-    res.end(uploadStatusId)
+    res.send(uploadStatusId)
+
+    const storage = botScopedStorage.get(req.params.botId)
 
-    if (yn(req.body.isReplace)) {
+    if (req.body.importAction === 'clear_insert') {
       updateUploadStatus(uploadStatusId, 'Deleting existing questions')
       const questions = await storage.fetchQNAs()
 
@@ -115,10 +126,9 @@ export default async (bp: typeof sdk, botScopedStorage: Map<string, Storage>) =>
     }
 
     try {
-      const parsedJson: any = JSON.parse(req.file.buffer)
-      const questions = (await validate(parsedJson, QnaItemArraySchema)) as QnaItem[]
+      const importData = await prepareImport(JSON.parse(req.body.fileContent))
 
-      await importQuestions(questions, storage, updateUploadStatus, uploadStatusId)
+      await importQuestions(importData, storage, bp, updateUploadStatus, uploadStatusId)
       updateUploadStatus(uploadStatusId, 'Completed')
     } catch (e) {
       bp.logger.attachError(e).error('JSON Import Failure')

modules/qna/src/backend/setup.ts

@@ -30,7 +30,7 @@ export const initModule = async (bp: typeof sdk, botScopedStorage: Map<string, S
     description: 'Listen for predefined questions and send canned responses.'
   })
 
-  const getAlternativeAnswer = (qnaEntry: QnaEntry, lang: string) => {
+  const getAlternativeAnswer = (qnaEntry: QnaEntry, lang: string): string => {
     const randomIndex = Math.floor(Math.random() * qnaEntry.answers[lang].length)
     return qnaEntry.answers[lang][randomIndex]
   }
@@ -50,12 +50,21 @@ export const initModule = async (bp: typeof sdk, botScopedStorage: Map<string, S
     }
 
     if (qnaEntry.action.includes('text')) {
-      const args = {
+      let args: any = {
         user: _.get(event, 'state.user') || {},
         session: _.get(event, 'state.session') || {},
-        temp: _.get(event, 'state.temp') || {},
-        text: getAlternativeAnswer(qnaEntry, lang),
-        typing: true
+        temp: _.get(event, 'state.temp') || {}
+      }
+
+      const electedAnswer = getAlternativeAnswer(qnaEntry, lang)
+      if (electedAnswer.startsWith('#!')) {
+        renderer = `!${electedAnswer.replace('#!', '')}`
+      } else {
+        args = {
+          ...args,
+          text: electedAnswer,
+          typing: true
+        }
       }
 
       const element = await bp.cms.renderElement(renderer, args, {

modules/qna/src/backend/storage.ts

@@ -36,7 +36,7 @@ const normalizeQuestions = questions =>
 export default class Storage {
   private bp: typeof sdk
   private config
-  private botId: string
+  public botId: string
   private categories: string[]
 
   constructor(bp: typeof sdk, config, botId) {
@@ -268,6 +268,12 @@ export default class Storage {
     return { items, count }
   }
 
+  async getAllContentElementIds(list?: QnaItem[]): Promise<string[]> {
+    const qnas = list || (await this.fetchQNAs())
+    const allAnswers = _.flatMap(qnas, qna => _.flatMap(Object.keys(qna.data.answers), lang => qna.data.answers[lang]))
+    return _.uniq(_.filter(allAnswers, x => _.isString(x) && x.startsWith('#!')))
+  }
+
   async count() {
     const questions = await this.fetchQNAs()
     return questions.length

modules/qna/src/backend/transfer.ts

@@ -1,9 +1,52 @@
-import { QnaItem, QnaEntry } from './qna'
+import * as sdk from 'botpress/sdk'
+import { validate } from 'joi'
+import _ from 'lodash'
+
+import { QnaEntry, QnaItem } from './qna'
 import Storage from './storage'
+import { QnaItemArraySchema, QnaItemCmsArraySchema } from './validation'
+
+const debug = DEBUG('qna:import')
+
+type ContentData = Pick<sdk.ContentElement, 'id' | 'contentType' | 'formData'>
+
+interface ImportData {
+  questions?: QnaItem[]
+  content?: ContentData[]
+}
+
+export const prepareImport = async (parsedJson: any): Promise<ImportData> => {
+  try {
+    const result = (await validate(parsedJson, QnaItemCmsArraySchema)) as { cms: ContentData[]; qnas: QnaItem[] }
+    return { questions: result.qnas, content: result.cms }
+  } catch (err) {
+    debug(`New format doesn't match provided file: ${err.message}`)
+  }
+
+  try {
+    const result = (await validate(parsedJson, QnaItemArraySchema)) as QnaItem[]
+    return { questions: result, content: undefined }
+  } catch (err) {
+    debug(`Old format doesn't match provided file: ${err.message}`)
+  }
 
-export const importQuestions = async (questions: QnaItem[], storage, statusCallback, uploadStatusId) => {
+  return {}
+}
+
+export const importQuestions = async (data: ImportData, storage, bp, statusCallback, uploadStatusId) => {
   statusCallback(uploadStatusId, 'Calculating diff with existing questions')
 
+  const { questions, content } = data
+  if (!questions) {
+    return
+  }
+
+  if (content) {
+    for (const element of content) {
+      await bp.cms.createOrUpdateContentElement(storage.botId, element.contentType, element.formData, element.id)
+    }
+  }
+
   const existingQuestionItems = (await (storage as Storage).fetchQNAs()).map(item => item.id)
   const itemsToSave = questions.filter(item => !existingQuestionItems.includes(item.id))
   const entriesToSave = itemsToSave.map(q => q.data)
@@ -16,7 +59,14 @@ export const importQuestions = async (questions: QnaItem[], storage, statusCallb
   })
 }
 
-export const prepareExport = async (storage: Storage) => {
+export const prepareExport = async (storage: Storage, bp: typeof sdk) => {
   const qnas = await storage.fetchQNAs()
-  return JSON.stringify(qnas, undefined, 2)
+  const contentElementIds = await storage.getAllContentElementIds()
+
+  const cms = await Promise.mapSeries(contentElementIds, async id => {
+    const data = await bp.cms.getContentElement(storage.botId, id.replace('#!', ''))
+    return _.pick(data, ['id', 'contentType', 'formData', 'previews'])
+  })
+
+  return JSON.stringify({ qnas, cms }, undefined, 2)
 }

modules/qna/src/backend/validation.ts

@@ -24,3 +24,8 @@ const QnaItemSchema = Joi.object().keys({
 })
 
 export const QnaItemArraySchema = Joi.array().items(QnaItemSchema)
+
+export const QnaItemCmsArraySchema = Joi.object().keys({
+  qnas: Joi.array().items(QnaItemSchema),
+  cms: Joi.array().items(Joi.object())
+})