feat(channels): add channel configuration page (#11646)

* chore(messaging): better communication

* print webhooks

* rename

* god tier code

* fix

* basic channel ui

* ui progress

* channels router start

* interact with backend start

* update channel config

* messaging service big refactor

* move code around

* little refacts

* db progress

* webhook section

* refact

* refact

* order by

* bump messaging client

* css

* fix telemetry

* bump messaging

* migration

* bp.messaging

* use bp.messaging in modules

* fix

* fix

* fix

* remove logs

* backward compatible

* change migration version

* fixes

* mig start

* better up migration

* move migration to studio

* migration

* fix typo

* bump messaging client

* fix

* bump studio

* bump messaging to 1.1.7

modules/channel-web/package.json

@@ -25,7 +25,6 @@
     "@types/react-intl": "^2.3.17"
   },
   "dependencies": {
-    "@botpress/messaging-client": "1.1.1",
     "@juggle/resize-observer": "^3.0.2",
     "apicache": "^1.6.2",
     "aws-sdk": "^2.905.0",

modules/channel-web/src/backend/api.ts

@@ -1,8 +1,8 @@
-import { Conversation, Message, MessagingClient } from '@botpress/messaging-client'
 import apicache from 'apicache'
 import aws from 'aws-sdk'
 import axios from 'axios'
 import * as sdk from 'botpress/sdk'
+import { Conversation, Message, MessagingClient } from 'botpress/sdk'
 import { asyncMiddleware as asyncMw, BPRequest } from 'common/http'
 import { Request, Response, NextFunction } from 'express'
 import FormData from 'form-data'
@@ -136,8 +136,8 @@ export default async (bp: typeof sdk, db: Database) => {
       return next(ERR_USER_ID_INVALID)
     }
 
-    req.messaging = await db.getMessagingClient(botId)
-    const userId = await db.mapVisitor(botId, req.visitorId, req.messaging)
+    req.messaging = bp.messaging.forBot(botId)
+    const userId = await db.mapVisitor(botId, req.visitorId)
 
     if (conversationId) {
       let conversation: Conversation
@@ -688,16 +688,15 @@ export default async (bp: typeof sdk, db: Database) => {
       const conversationId = req.params.id
       const { userId } = req.body
 
-      const messaging = await db.getMessagingClient(botId)
-      const conversation = await messaging.getConversation(conversationId)
+      const conversation = await bp.messaging.forBot(botId).getConversation(conversationId)
       if (!userId || conversation?.userId !== userId) {
         return res.status(400).send(ERR_BAD_CONV_ID)
       }
 
       const { visitorId } = await db.getMappingFromUser(userId)
       bp.realtime.sendPayload(bp.RealTimePayload.forVisitor(visitorId, 'webchat.clear', { conversationId }))
 
-      await messaging.deleteMessagesByConversation(conversationId)
+      await bp.messaging.forBot(botId).deleteMessagesByConversation(conversationId)
       res.sendStatus(204)
     })
   )

modules/channel-web/src/backend/db.ts

@@ -1,4 +1,3 @@
-import { MessagingClient } from '@botpress/messaging-client'
 import * as sdk from 'botpress/sdk'
 import _ from 'lodash'
 import LRUCache from 'lru-cache'
@@ -8,14 +7,12 @@ export default class WebchatDb {
   private knex: sdk.KnexExtended
   private cacheByVisitor: LRUCache<string, UserMapping>
   private cacheByUser: LRUCache<string, UserMapping>
-  private messagingClients: { [botId: string]: MessagingClient } = {}
 
   constructor(private bp: typeof sdk) {
     this.knex = bp.database
     this.cacheByVisitor = new LRUCache({ max: 10000, maxAge: ms('5min') })
     this.cacheByUser = new LRUCache({ max: 10000, maxAge: ms('5min') })
   }
-
   async initialize() {
     await this.knex.createTableIfNotExists('web_user_map', table => {
       table.string('botId')
@@ -31,12 +28,12 @@ export default class WebchatDb {
   // to add your changes there too.
   //===================================
 
-  async mapVisitor(botId: string, visitorId: string, messaging: MessagingClient) {
+  async mapVisitor(botId: string, visitorId: string) {
     const userMapping = await this.getMappingFromVisitor(botId, visitorId)
     let userId = userMapping?.userId
 
     const createUserAndMapping = async () => {
-      userId = (await messaging.createUser()).id
+      userId = (await this.bp.messaging.forBot(botId).createUser()).id
       await this.createUserMapping(botId, visitorId, userId)
     }
 
@@ -45,7 +42,7 @@ export default class WebchatDb {
     } else {
       // Prevents issues when switching between different Messaging servers
       // TODO: Remove this check once the 'web_user_map' table is removed
-      if (!(await this.checkUserExist(userMapping.userId, messaging))) {
+      if (!(await this.checkUserExist(botId, userMapping.userId))) {
         await this.deleteMappingFromVisitor(botId, visitorId)
         await createUserAndMapping()
       }
@@ -78,7 +75,8 @@ export default class WebchatDb {
   async deleteMappingFromVisitor(botId: string, visitorId: string): Promise<void> {
     try {
       this.cacheByVisitor.del(`${botId}_${visitorId}`)
-      await this.knex('web_user_map')
+      await this.bp
+        .database('web_user_map')
         .where({ botId, visitorId })
         .delete()
     } catch (err) {
@@ -129,33 +127,8 @@ export default class WebchatDb {
       .whereIn('events.messageId', messageIds)
   }
 
-  async getMessagingClient(botId: string) {
-    const client = this.messagingClients[botId]
-    if (client) {
-      return client
-    }
-
-    const { messaging } = await this.bp.bots.getBotById(botId)
-
-    const botClient = new MessagingClient({
-      url: process.core_env.MESSAGING_ENDPOINT
-        ? process.core_env.MESSAGING_ENDPOINT
-        : `http://localhost:${process.MESSAGING_PORT}`,
-      clientId: messaging.id,
-      clientToken: messaging.token,
-      axios: { headers: { password: process.INTERNAL_PASSWORD }, proxy: false }
-    })
-    this.messagingClients[botId] = botClient
-
-    return botClient
-  }
-
-  removeMessagingClient(botId: string) {
-    this.messagingClients[botId] = undefined
-  }
-
-  private async checkUserExist(userId: string, messaging: MessagingClient): Promise<boolean> {
-    const user = await messaging.getUser(userId)
+  private async checkUserExist(botId: string, userId: string): Promise<boolean> {
+    const user = await this.bp.messaging.forBot(botId).getUser(userId)
 
     return user?.id === userId
   }

modules/channel-web/src/backend/index.ts

@@ -22,15 +22,10 @@ const onModuleUnmount = async (bp: typeof sdk) => {
   bp.http.deleteRouterForBot('channel-web')
 }
 
-const onBotUnmount = async (bp: typeof sdk, botId: string) => {
-  db.removeMessagingClient(botId)
-}
-
 const entryPoint: sdk.ModuleEntryPoint = {
   onServerStarted,
   onServerReady,
   onModuleUnmount,
-  onBotUnmount,
   definition: {
     name: 'channel-web',
     fullName: 'Web Chat',

modules/channel-web/src/backend/socket.ts

@@ -21,7 +21,7 @@ export default async (bp: typeof sdk, db: Database) => {
       return next()
     }
 
-    const messaging = await db.getMessagingClient(event.botId)
+    const messaging = bp.messaging.forBot(event.botId)
     const messageType = event.type === 'default' ? 'text' : event.type
     const userId = event.target
     const mapping = await db.getMappingFromUser(userId)

modules/channel-web/src/messaging-migration/up.ts

@@ -180,7 +180,7 @@ export abstract class MessagingUpMigrator {
     if (exists && !this.isDryRun) {
       try {
         await this.bp.config.mergeBotConfig(botId, {
-          messaging: { id: client.id, token, channels: {} }
+          messaging: { id: client.id, token, channels: {} } as any
         })
       } catch {
         // fails when no bot.config.json is present.

modules/channel-web/yarn.lock

@@ -45,20 +45,6 @@
     "@babel/helper-validator-identifier" "^7.14.9"
     to-fast-properties "^2.0.0"
 
-"@botpress/messaging-base@1.1.0":
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/@botpress/messaging-base/-/messaging-base-1.1.0.tgz#0a266f8748f45afdcb93e158e72c03dc5d7c3035"
-  integrity sha512-mIQ90leVlYw+DuYixPa1pGWt/Sc/EU/r8k6IjS/BUl0XalvlgmeWIJsZ8xTh9B2H6f6xidlJO63tsgiPcimf7Q==
-
-"@botpress/messaging-client@1.1.1":
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/@botpress/messaging-client/-/messaging-client-1.1.1.tgz#ccc9441d421b1641d825074f590ecbde9680519d"
-  integrity sha512-Rflsn282f/VyRBgtbi5mB/UcJEOesom9GNLlqx7LGF2Lf71mBwMEnxvLEuUA6BdkG4SsUZPBQvjY3RYaXUgIpg==
-  dependencies:
-    "@botpress/messaging-base" "1.1.0"
-    axios "^0.21.4"
-    joi "^17.6.0"
-
 "@emotion/babel-utils@^0.6.4":
   version "0.6.10"
   resolved "https://registry.npmjs.org/@emotion/babel-utils/-/babel-utils-0.6.10.tgz#83dbf3dfa933fae9fc566e54fbb45f14674c6ccc"
@@ -106,40 +92,11 @@
   resolved "https://registry.npmjs.org/@emotion/utils/-/utils-0.8.2.tgz#576ff7fb1230185b619a75d258cbc98f0867a8dc"
   integrity sha512-rLu3wcBWH4P5q1CGoSSH/i9hrXs7SlbRLkoq9IGuoPYNGQvDJ3pt/wmOM+XgYjIDRMVIdkUWt0RsfzF50JfnCw==
 
-"@hapi/hoek@^9.0.0":
-  version "9.2.1"
-  resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.2.1.tgz#9551142a1980503752536b5050fd99f4a7f13b17"
-  integrity sha512-gfta+H8aziZsm8pZa0vj04KO6biEiisppNgA1kbJvFrrWu9Vm7eaUEy76DIxsuTaWvti5fkJVhllWc6ZTE+Mdw==
-
-"@hapi/topo@^5.0.0":
-  version "5.1.0"
-  resolved "https://registry.yarnpkg.com/@hapi/topo/-/topo-5.1.0.tgz#dc448e332c6c6e37a4dc02fd84ba8d44b9afb012"
-  integrity sha512-foQZKJig7Ob0BMAYBfcJk8d77QtOe7Wo4ox7ff1lQYoNNAb6jwcY1ncdoy2e9wQZzvNy7ODZCYJkK8kzmcAnAg==
-  dependencies:
-    "@hapi/hoek" "^9.0.0"
-
 "@juggle/resize-observer@^3.0.2":
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/@juggle/resize-observer/-/resize-observer-3.0.2.tgz#f4f326806826c0da5221411374e4c54902c327bb"
   integrity sha512-9fEfZIxZ1qTahMSX50pigfSJ/1N6X2oABhmgd4Dt2SsPkZi0lTlvoHu5V2yrGZ6m+oALfS4tJrpx9nbVIxwOYQ==
 
-"@sideway/address@^4.1.3":
-  version "4.1.3"
-  resolved "https://registry.yarnpkg.com/@sideway/address/-/address-4.1.3.tgz#d93cce5d45c5daec92ad76db492cc2ee3c64ab27"
-  integrity sha512-8ncEUtmnTsMmL7z1YPB47kPUq7LpKWJNFPsRzHiIajGC5uXlWGn+AmkYPcHNl8S4tcEGx+cnORnNYaw2wvL+LQ==
-  dependencies:
-    "@hapi/hoek" "^9.0.0"
-
-"@sideway/formula@^3.0.0":
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/@sideway/formula/-/formula-3.0.0.tgz#fe158aee32e6bd5de85044be615bc08478a0a13c"
-  integrity sha512-vHe7wZ4NOXVfkoRb8T5otiENVlT7a3IAiw7H5M2+GO+9CDgcVUUsX1zalAztCmwyOr2RUTGJdgB+ZvSVqmdHmg==
-
-"@sideway/pinpoint@^2.0.0":
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/@sideway/pinpoint/-/pinpoint-2.0.0.tgz#cff8ffadc372ad29fd3f78277aeb29e632cc70df"
-  integrity sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==
-
 "@types/apicache@^1.2.2":
   version "1.2.2"
   resolved "https://registry.yarnpkg.com/@types/apicache/-/apicache-1.2.2.tgz#b820659b1d95e66ec0e71dcd0317e9d30f0c154b"
@@ -327,13 +284,6 @@ axios@^0.21.2:
   dependencies:
     follow-redirects "^1.14.0"
 
-axios@^0.21.4:
-  version "0.21.4"
-  resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575"
-  integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==
-  dependencies:
-    follow-redirects "^1.14.0"
-
 babel-plugin-emotion@^9.2.11:
   version "9.2.11"
   resolved "https://registry.npmjs.org/babel-plugin-emotion/-/babel-plugin-emotion-9.2.11.tgz#319c005a9ee1d15bb447f59fe504c35fd5807728"
@@ -776,17 +726,6 @@ jmespath@0.15.0:
   resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217"
   integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=
 
-joi@^17.6.0:
-  version "17.6.0"
-  resolved "https://registry.yarnpkg.com/joi/-/joi-17.6.0.tgz#0bb54f2f006c09a96e75ce687957bd04290054b2"
-  integrity sha512-OX5dG6DTbcr/kbMFj0KGYxuew69HPcAE3K/sZpEV2nP6e/j/C0HV+HNiBPCASxdx5T7DMoa0s8UeHWMnb6n2zw==
-  dependencies:
-    "@hapi/hoek" "^9.0.0"
-    "@hapi/topo" "^5.0.0"
-    "@sideway/address" "^4.1.3"
-    "@sideway/formula" "^3.0.0"
-    "@sideway/pinpoint" "^2.0.0"
-
 "js-tokens@^3.0.0 || ^4.0.0", js-tokens@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499"

modules/hitlnext/package.json

@@ -15,7 +15,6 @@
   "license": "AGPL-3.0-only",
   "dependencies": {
     "@blueprintjs/core": "^3.36.0",
-    "@botpress/messaging-client": "1.1.1",
     "@webscopeio/react-textarea-autocomplete": "^4.7.2",
     "axios": "^0.21.1",
     "bluebird": "^3.5",

modules/hitlnext/src/backend/api.ts

@@ -226,9 +226,8 @@ export default async (bp: typeof sdk, state: StateType, repository: Repository)
 
       let handoff = await repository.findHandoff(req.params.botId, req.params.id)
 
-      const messaging = await repository.getMessagingClient(botId)
-      const userId = await repository.mapVisitor(botId, agentId, messaging)
-      const conversation = await messaging.createConversation(userId)
+      const userId = await repository.mapVisitor(botId, agentId)
+      const conversation = await bp.messaging.forBot(botId).createConversation(userId)
 
       const agentThreadId = conversation.id
       const payload: Pick<IHandoff, 'agentId' | 'agentThreadId' | 'assignedAt' | 'status'> = {

modules/hitlnext/src/backend/index.ts

@@ -36,15 +36,10 @@ const onModuleUnmount = async (bp: typeof sdk) => {
   await unregisterMiddleware(bp)
 }
 
-const onBotUnmount = async (bp: typeof sdk, botId: string) => {
-  repository.removeMessagingClient(botId)
-}
-
 const entryPoint: sdk.ModuleEntryPoint = {
   onServerStarted,
   onServerReady,
   onModuleUnmount,
-  onBotUnmount,
   translations: { en, fr },
   definition: {
     name: MODULE_NAME,

modules/hitlnext/src/backend/middleware.ts

@@ -43,8 +43,7 @@ const registerMiddleware = async (bp: typeof sdk, state: StateType) => {
   const handleIncomingFromUser = async (handoff: IHandoff, event: sdk.IO.IncomingEvent) => {
     // There only is an agentId & agentThreadId after assignation
     if (handoff.status === 'assigned') {
-      const messaging = await repository.getMessagingClient(handoff.botId)
-      const userId = await repository.mapVisitor(handoff.botId, handoff.agentId, messaging)
+      const userId = await repository.mapVisitor(handoff.botId, handoff.agentId)
       return pipeEvent(event, {
         botId: handoff.botId,
         target: userId,

modules/hitlnext/src/backend/repository.ts

@@ -1,4 +1,3 @@
-import { MessagingClient } from '@botpress/messaging-client'
 import axios from 'axios'
 import * as sdk from 'botpress/sdk'
 import { BPRequest } from 'common/http'
@@ -58,7 +57,6 @@ const userColumnsPrefixed = userColumns.map(s => userPrefix.concat(':', s))
 export default class Repository {
   private agentCache: Dic<Omit<IAgent, 'online'>> = {}
   private cacheByVisitor: LRUCache<string, UserMapping>
-  private messagingClients: { [botId: string]: MessagingClient } = {}
 
   /**
    *
@@ -537,12 +535,12 @@ export default class Repository {
   // Copy pasted from channel-web db.ts
   //===================================
 
-  async mapVisitor(botId: string, visitorId: string, messaging: MessagingClient) {
+  async mapVisitor(botId: string, visitorId: string) {
     const userMapping = await this.getMappingFromVisitor(botId, visitorId)
     let userId = userMapping?.userId
 
     const createUserAndMapping = async () => {
-      userId = (await messaging.createUser()).id
+      userId = (await this.bp.messaging.forBot(botId).createUser()).id
       await this.createUserMapping(botId, visitorId, userId)
     }
 
@@ -551,7 +549,7 @@ export default class Repository {
     } else {
       // Prevents issues when switching between different Messaging servers
       // TODO: Remove this check once the 'web_user_map' table is removed
-      if (!(await this.checkUserExist(userMapping.userId, messaging))) {
+      if (!(await this.checkUserExist(botId, userMapping.userId))) {
         await this.deleteMappingFromVisitor(botId, visitorId)
         await createUserAndMapping()
       }
@@ -608,33 +606,8 @@ export default class Repository {
     }
   }
 
-  async getMessagingClient(botId: string) {
-    const client = this.messagingClients[botId]
-    if (client) {
-      return client
-    }
-
-    const { messaging } = await this.bp.bots.getBotById(botId)
-
-    const botClient = new MessagingClient({
-      url: process.core_env.MESSAGING_ENDPOINT
-        ? process.core_env.MESSAGING_ENDPOINT
-        : `http://localhost:${process.MESSAGING_PORT}`,
-      clientId: messaging.id,
-      clientToken: messaging.token,
-      axios: { headers: { password: process.INTERNAL_PASSWORD }, proxy: false }
-    })
-    this.messagingClients[botId] = botClient
-
-    return botClient
-  }
-
-  removeMessagingClient(botId: string) {
-    this.messagingClients[botId] = undefined
-  }
-
-  private async checkUserExist(userId: string, messaging: MessagingClient): Promise<boolean> {
-    const user = await messaging.getUser(userId)
+  private async checkUserExist(botId: string, userId: string): Promise<boolean> {
+    const user = await this.bp.messaging.forBot(botId).getUser(userId)
 
     return user?.id === userId
   }