Merge pull request #1774 from botpress/ya-testing

feat: conversation scenarios

modules/testing/.gitignore

@@ -0,0 +1,7 @@
+/bin
+/node_modules
+/dist
+/assets/web/
+/assets/config.schema.json
+botpress.d.ts
+global.d.ts

modules/testing/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "testing",
+  "version": "1.0.0",
+  "description": "",
+  "private": true,
+  "main": "dist/backend/index.js",
+  "author": "Botpress, Inc.",
+  "license": "AGPL-3.0-only",
+  "scripts": {
+    "build": "./node_modules/.bin/module-builder build",
+    "watch": "./node_modules/.bin/module-builder watch",
+    "package": "./node_modules/.bin/module-builder package"
+  },
+  "devDependencies": {
+    "@types/node": "^10.11.3",
+    "module-builder": "../../build/module-builder"
+  },
+  "dependencies": {
+    "classnames": "^2.2.6",
+    "react-icons": "^3.6.1"
+  }
+}

modules/testing/src/backend/api.ts

@@ -0,0 +1,71 @@
+import * as sdk from 'botpress/sdk'
+import _ from 'lodash'
+
+import { TestByBot } from './typings'
+
+export default async (bp: typeof sdk, testByBot: TestByBot) => {
+  const router = bp.http.createRouterForBot('testing')
+
+  router.get('/scenarios', async (req, res) => {
+    const scenarios = await testByBot[req.params.botId].getScenarios()
+    const status = await testByBot[req.params.botId].getStatus()
+
+    res.send({ scenarios, status })
+  })
+
+  router.post('/runAll', async (req, res) => {
+    await testByBot[req.params.botId].executeAll()
+    res.sendStatus(200)
+  })
+
+  router.post('/run', async (req, res) => {
+    await testByBot[req.params.botId].executeSingle(req.body.scenario)
+    res.sendStatus(200)
+  })
+
+  router.get('/startRecording/:userId?', (req, res) => {
+    testByBot[req.params.botId].startRecording(req.params.userId || '')
+    res.sendStatus(200)
+  })
+
+  router.get('/stopRecording', async (req, res) => {
+    res.send(await testByBot[req.params.botId].endRecording())
+  })
+
+  router.post('/saveScenario', async (req, res) => {
+    const { name, steps } = req.body
+    if (!name || !steps || !name.length) {
+      return res.sendStatus(400)
+    }
+
+    await testByBot[req.params.botId].saveScenario(name, steps)
+    res.sendStatus(200)
+  })
+
+  router.post('/incomingEvent', (req, res) => {
+    const event = req.body as sdk.IO.IncomingEvent
+    return res.send(testByBot[req.params.botId].processIncomingEvent(event))
+  })
+
+  router.post('/processedEvent', (req, res) => {
+    const event = req.body as sdk.IO.IncomingEvent
+    res.send(testByBot[req.params.botId].processCompletedEvent(event))
+  })
+
+  router.post('/fetchPreviews', async (req, res) => {
+    const { elementIds } = req.body
+    if (!elementIds || !_.isArray(elementIds)) {
+      return res.sendStatus(400)
+    }
+
+    const elements = await bp.cms.getContentElements(req.params.botId, elementIds.map(x => x.replace('#!', '')))
+    const rendered = elements.map(element => {
+      return {
+        id: `#!${element.id}`,
+        preview: element.previews.en
+      }
+    })
+
+    res.send(rendered)
+  })
+}

modules/testing/src/backend/index.ts

@@ -0,0 +1,45 @@
+import * as sdk from 'botpress/sdk'
+import _ from 'lodash'
+
+import api from './api'
+
+import { Testing } from './testing'
+import { TestByBot } from './typings'
+
+const testByBot: TestByBot = {}
+
+const onServerStarted = async (bp: typeof sdk) => {}
+const onServerReady = async (bp: typeof sdk) => {
+  await api(bp, testByBot)
+}
+
+const onBotMount = async (bp: typeof sdk, botId: string) => {
+  testByBot[botId] = new Testing(bp, botId)
+}
+
+const onBotUnmount = async (bp: typeof sdk, botId: string) => {
+  delete testByBot[botId]
+}
+
+const onModuleUnmount = async (bp: typeof sdk) => {
+  bp.http.deleteRouterForBot('testing')
+}
+
+const entryPoint: sdk.ModuleEntryPoint = {
+  onServerStarted,
+  onServerReady,
+  onModuleUnmount,
+  onBotMount,
+  onBotUnmount,
+  definition: {
+    name: 'testing',
+    menuIcon: 'polymer',
+    menuText: 'Testing',
+    noInterface: false,
+    fullName: 'Testing',
+    homepage: 'https://botpress.io',
+    experimental: true
+  }
+}
+
+export default entryPoint

modules/testing/src/backend/recorder.ts

@@ -0,0 +1,76 @@
+import * as sdk from 'botpress/sdk'
+import _ from 'lodash'
+
+import { Scenario } from './typings'
+import { convertLastMessages } from './utils'
+
+export class Recorder {
+  private _lastEvent: sdk.IO.IncomingEvent
+  private _scenario?: Scenario
+  private _specificTarget?: string
+
+  constructor() {}
+
+  processIncoming(event: sdk.IO.IncomingEvent) {
+    if (!this._scenario || this._scenario.initialState) {
+      return
+    }
+
+    if (!this._specificTarget || this._specificTarget === event.target) {
+      this._scenario.initialState = event.state
+    }
+  }
+
+  processCompleted(event: sdk.IO.IncomingEvent) {
+    if (!this._scenario) {
+      return
+    }
+
+    if (this._specificTarget && this._specificTarget !== event.target) {
+      return
+    }
+
+    const interactions = convertLastMessages(event.state.session.lastMessages, event.id)
+    if (interactions) {
+      this._lastEvent = event
+      this._scenario.steps.push(interactions)
+    }
+  }
+
+  startRecording(chatUserId: string) {
+    this._lastEvent = undefined
+    this._scenario = {
+      initialState: undefined,
+      finalState: undefined,
+      steps: []
+    }
+
+    this._specificTarget = chatUserId.length > 0 && chatUserId
+  }
+
+  stopRecording(): Partial<Scenario> | void {
+    if (!this._scenario || !this._lastEvent) {
+      return
+    }
+
+    const finalScenario = {
+      ..._.pick(this._scenario, ['steps', 'initialState']),
+      finalState: this._lastEvent.state
+    }
+
+    this._scenario = undefined
+
+    return _.omit(finalScenario, [
+      'initialState.session.lastMessages',
+      'initialState.context.jumpPoints',
+      'initialState.context.queue',
+      'finalState.session.lastMessages',
+      'finalState.context.jumpPoints',
+      'finalState.context.queue'
+    ])
+  }
+
+  isRecording() {
+    return !!this._scenario
+  }
+}

modules/testing/src/backend/runner.ts

@@ -0,0 +1,162 @@
+import * as sdk from 'botpress/sdk'
+import _ from 'lodash'
+
+import { DialogStep, RunningScenario, Scenario, ScenarioMismatch, ScenarioStatus } from './typings'
+import { convertLastMessages } from './utils'
+
+const TIMEOUT = 3000
+
+export class SenarioRunner {
+  private _active: RunningScenario[]
+  private _status: ScenarioStatus
+  private _interval: any
+
+  constructor(private bp: typeof sdk) {
+    this._active = []
+  }
+
+  startReplay() {
+    this._status = {}
+    this._active = []
+    this._interval = setInterval(this._checkScenarioTimeout.bind(this), 5000)
+  }
+
+  processIncoming(event: sdk.IO.IncomingEvent): sdk.IO.EventState | void {
+    if (!this._active.length) {
+      return
+    }
+
+    const scenario = this._active.find(x => x.eventDestination.target === event.target)
+    if (scenario && !scenario.completedSteps.length) {
+      // The hook will replace the state with the one received here
+      return scenario.initialState
+    }
+  }
+
+  processCompleted(event: sdk.IO.IncomingEvent) {
+    if (!this._active.length) {
+      return
+    }
+
+    const scenario = this._active.find(x => x.eventDestination.target === event.target)
+    if (!scenario) {
+      return
+    }
+
+    const { name, completedSteps, steps } = scenario
+
+    const conversation = convertLastMessages(event.state.session.lastMessages, event.id)
+    if (!conversation) {
+      this._failScenario(name, { reason: 'Could not extract messages for the event ' + event.id })
+      return
+    }
+
+    const mismatch = this._findMismatch(steps[completedSteps.length], conversation)
+    if (mismatch) {
+      this._failScenario(name, mismatch)
+    } else {
+      completedSteps.push(conversation)
+      this._updateStatus(name, { completedSteps: completedSteps.length })
+    }
+
+    if (steps.length !== completedSteps.length) {
+      scenario.lastEventTs = +new Date()
+      this._sendMessage(steps[completedSteps.length].userMessage, scenario.eventDestination)
+    } else {
+      this._passScenario(name)
+    }
+  }
+
+  runScenario(scenario: Scenario, eventDestination: sdk.IO.EventDestination) {
+    const firstMessage = scenario.steps[0].userMessage
+    if (!firstMessage) {
+      return
+    }
+
+    this._active.push({ ...scenario, eventDestination, completedSteps: [] })
+    this._sendMessage(firstMessage, eventDestination)
+    this._updateStatus(scenario.name, { status: 'pending', completedSteps: 0 })
+  }
+
+  getStatus(scenarioName: string) {
+    return (this._status && this._status[scenarioName]) || {}
+  }
+
+  isRunning() {
+    return !!this._active.length
+  }
+
+  private _findMismatch(expected: DialogStep, received: DialogStep): ScenarioMismatch | void {
+    let mismatch = undefined
+
+    // This shouldn't happen
+    if (!expected || !received || expected.userMessage !== received.userMessage) {
+      return { reason: 'Expected or received step was invalid', expected, received }
+    }
+
+    // Inside each steps, the bot may reply multiple times
+    _.each(_.zip(expected.botReplies, received.botReplies), ([exp, rec], idx) => {
+      // This can happen if the bot doesn't respond
+      if (!exp || !rec) {
+        mismatch = { reason: 'Missing an expected or received reply', expected, received, index: idx }
+        return false
+      }
+
+      const sameSource = exp.replySource === rec.replySource
+      const sameResponse = exp.botResponse === rec.botResponse
+      const source = exp.replySource.split(' ').shift() // extracting the first part (module) for the reply
+
+      /**
+       * Different sources are definitely not what is expected
+       * If QNA has the exact same source, then we don't care about the response (variations)
+       * If the source is Dialog Manager, then the answer must be identical (either payload or content element id)
+       */
+      if (!sameSource || (source !== 'qna' && (source === 'dialogManager' && !sameResponse))) {
+        mismatch = { reason: 'The reply was invalid', expected, received, index: idx }
+        return false
+      }
+    })
+
+    return mismatch
+  }
+
+  private _checkScenarioTimeout() {
+    if (!this._active.length) {
+      this._interval && clearInterval(this._interval)
+      return
+    }
+
+    const now = +new Date()
+    const mismatch = { reason: 'The scenario timed out' }
+    this._active
+      .filter(s => s.lastEventTs !== undefined && now - s.lastEventTs > TIMEOUT)
+      .map(x => this._failScenario(x.name, mismatch))
+  }
+
+  private _passScenario(name: string) {
+    this._updateStatus(name, { status: 'pass' })
+    this._active = this._active.filter(x => x.name !== name)
+  }
+
+  private _failScenario(name: string, mismatch: ScenarioMismatch) {
+    this._updateStatus(name, { status: 'fail', mismatch })
+    this._active = this._active.filter(x => x.name !== name)
+  }
+
+  private _updateStatus(scenario, obj) {
+    this._status[scenario] = { ...(this._status[scenario] || {}), ...obj }
+  }
+
+  private _sendMessage = (message: string, eventDestination: sdk.IO.EventDestination) => {
+    setTimeout(() => {
+      const event = this.bp.IO.Event({
+        ...eventDestination,
+        direction: 'incoming',
+        payload: { type: 'text', text: message },
+        type: 'text'
+      })
+
+      this.bp.events.sendEvent(event)
+    }, 1000)
+  }
+}