feat(channels): channel versioning (#328)

* feat(channels): channel versioning

* migration

* webhook versioning

* fix validation

* fix tests

* better validation

* fix

* fix

* fix tests

* fix

* fix

* fix

* refact

* fix

* fix

packages/channels/src/base/meta.ts

@@ -3,7 +3,8 @@ import Joi from 'joi'
 export interface ChannelMeta {
   id: string
   name: string
+  version: string
   initiable: boolean
   lazy: boolean
-  schema: Joi.ObjectSchema
+  schema: { [field: string]: Joi.Schema }
 }

packages/channels/src/messenger/channel.ts

@@ -14,6 +14,7 @@ export class MessengerChannel extends ChannelTemplate<
     return {
       id: 'c4bb1487-b3bd-49b3-a3dd-36db908d165d',
       name: 'messenger',
+      version: '0.1.0',
       schema: MessengerConfigSchema,
       initiable: true,
       lazy: true

packages/channels/src/messenger/config.ts

@@ -39,12 +39,12 @@ export interface NestedButton {
   call_to_actions: CallToAction[]
 }
 
-export const MessengerConfigSchema = Joi.object({
+export const MessengerConfigSchema = {
   accessToken: Joi.string().required(),
   appSecret: Joi.string().required(),
   verifyToken: Joi.string().required(),
   disabledActions: Joi.array().items(Joi.string().valid('typing_on', 'typing_off', 'mark_seen')).optional(),
   greeting: Joi.string().optional(),
   getStarted: Joi.string().optional(),
   persistentMenu: Joi.array().optional()
-}).options({ stripUnknown: true })
+}

packages/channels/src/slack/channel.ts

@@ -9,6 +9,7 @@ export class SlackChannel extends ChannelTemplate<SlackConfig, SlackService, Sla
     return {
       id: 'd6111009-712d-485e-a62d-1540f966f4f3',
       name: 'slack',
+      version: '0.1.0',
       schema: SlackConfigSchema,
       initiable: false,
       lazy: true

packages/channels/src/slack/config.ts

@@ -7,8 +7,8 @@ export interface SlackConfig extends ChannelConfig {
   useRTM: boolean
 }
 
-export const SlackConfigSchema = Joi.object({
+export const SlackConfigSchema = {
   botToken: Joi.string().required(),
   signingSecret: Joi.string().required(),
   useRTM: Joi.boolean().optional()
-}).options({ stripUnknown: true })
+}

packages/channels/src/smooch/channel.ts

@@ -9,6 +9,7 @@ export class SmoochChannel extends ChannelTemplate<SmoochConfig, SmoochService,
     return {
       id: '3c5c160f-d673-4ef8-8b6f-75448af048ce',
       name: 'smooch',
+      version: '0.1.0',
       schema: SmoochConfigSchema,
       initiable: true,
       lazy: true

packages/channels/src/smooch/config.ts

@@ -7,8 +7,8 @@ export interface SmoochConfig extends ChannelConfig {
   forwardRawPayloads?: string[]
 }
 
-export const SmoochConfigSchema = Joi.object({
+export const SmoochConfigSchema = {
   keyId: Joi.string().required(),
   secret: Joi.string().required(),
   forwardRawPayloads: Joi.array().items(Joi.string()).optional()
-}).options({ stripUnknown: true })
+}

packages/channels/src/teams/channel.ts

@@ -9,6 +9,7 @@ export class TeamsChannel extends ChannelTemplate<TeamsConfig, TeamsService, Tea
     return {
       id: '0491806d-ceb4-4397-8ebf-b8e6deb038da',
       name: 'teams',
+      version: '0.1.0',
       schema: TeamsConfigSchema,
       initiable: false,
       lazy: true

packages/channels/src/teams/config.ts

@@ -10,9 +10,9 @@ export interface TeamsConfig extends ChannelConfig {
   }
 }
 
-export const TeamsConfigSchema = Joi.object({
+export const TeamsConfigSchema = {
   appId: Joi.string().required(),
   appPassword: Joi.string().required(),
   tenantId: Joi.string().optional(),
   proactiveMessages: Joi.object().optional()
-}).options({ stripUnknown: true })
+}

packages/channels/src/telegram/channel.ts

@@ -9,6 +9,7 @@ export class TelegramChannel extends ChannelTemplate<TelegramConfig, TelegramSer
     return {
       id: '0198f4f5-6100-4549-92e5-da6cc31b4ad1',
       name: 'telegram',
+      version: '0.1.0',
       schema: TelegramConfigSchema,
       initiable: true,
       lazy: true

packages/channels/src/telegram/config.ts

@@ -4,6 +4,6 @@ export interface TelegramConfig {
   botToken: string
 }
 
-export const TelegramConfigSchema = Joi.object({
+export const TelegramConfigSchema = {
   botToken: Joi.string().required()
-}).options({ stripUnknown: true })
+}

packages/channels/src/twilio/channel.ts

@@ -9,6 +9,7 @@ export class TwilioChannel extends ChannelTemplate<TwilioConfig, TwilioService,
     return {
       id: '330ca935-6441-4159-8969-d0a0d3f188a1',
       name: 'twilio',
+      version: '0.1.0',
       schema: TwilioConfigSchema,
       initiable: false,
       lazy: true

packages/channels/src/twilio/config.ts

@@ -6,7 +6,7 @@ export interface TwilioConfig extends ChannelConfig {
   authToken: string
 }
 
-export const TwilioConfigSchema = Joi.object({
+export const TwilioConfigSchema = {
   accountSID: Joi.string().required(),
   authToken: Joi.string().required()
-}).options({ stripUnknown: true })
+}

packages/channels/src/vonage/channel.ts

@@ -9,6 +9,7 @@ export class VonageChannel extends ChannelTemplate<VonageConfig, VonageService,
     return {
       id: 'bf045a3c-5627-416d-974d-5cfeb277a23f',
       name: 'vonage',
+      version: '0.1.0',
       schema: VonageConfigSchema,
       initiable: false,
       lazy: true

packages/channels/src/vonage/config.ts

@@ -10,11 +10,11 @@ export interface VonageConfig extends ChannelConfig {
   useTestingApi?: boolean
 }
 
-export const VonageConfigSchema = Joi.object({
+export const VonageConfigSchema = {
   apiKey: Joi.string().required(),
   apiSecret: Joi.string().required(),
   signatureSecret: Joi.string().required(),
   applicationId: Joi.string().required(),
   privateKey: Joi.string().required(),
   useTestingApi: Joi.boolean().optional()
-}).options({ stripUnknown: true })
+}

packages/server/src/channels/api.ts

@@ -8,16 +8,26 @@ export class ChannelApi {
 
   async setup() {
     const logger = this.app.logger.root.sub('channels')
-    const webhookRouter = Router()
+    const routers: { [version: string]: { router: Router; path: string } } = {
+      '0.1.0': {
+        router: Router(),
+        path: ''
+      },
+      '1.0.0': {
+        router: Router(),
+        path: '/v1'
+      }
+    }
 
     for (const channel of this.app.channels.list()) {
-      await channel.setup(webhookRouter, logger.sub(channel.meta.name))
+      const router = routers[channel.meta.version]
+      await channel.setup(router.router, logger.sub(channel.meta.name))
 
       channel.logger = this.app.logger.root.sub(channel.meta.name)
       channel.kvs = this.app.kvs
 
       channel.makeUrl(async (scope: string) => {
-        return `${process.env.EXTERNAL_URL}/webhooks/${scope}/${channel.meta.name}`
+        return `${process.env.EXTERNAL_URL}/webhooks${router.path}/${scope}/${channel.meta.name}`
       })
 
       channel.on('message', async ({ scope, endpoint, content }) => {
@@ -45,7 +55,8 @@ export class ChannelApi {
       })
     }
 
-    this.router.use('/webhooks', webhookRouter)
+    this.router.use('/webhooks/v1', routers['1.0.0'].router)
+    this.router.use('/webhooks', routers['0.1.0'].router)
   }
 
   async map(channel: Channel, scope: string, endpoint: Endpoint, content: any): Promise<Mapping | undefined> {

packages/server/src/channels/service.ts

@@ -10,13 +10,15 @@ import {
   VonageChannel
 } from '@botpress/messaging-channels'
 import { Service, DatabaseService } from '@botpress/messaging-engine'
+import semver from 'semver'
 import { ChannelTable } from './table'
 
 export class ChannelService extends Service {
   private table: ChannelTable
 
   private channels: Channel[]
-  private channelsByName: { [name: string]: Channel }
+  private channelsByNameAndVersion: { [name: string]: Channel }
+  private channelsByName: { [name: string]: Channel[] }
   private channelsById: { [id: string]: Channel }
 
   constructor(private db: DatabaseService) {
@@ -34,12 +36,22 @@ export class ChannelService extends Service {
       new VonageChannel()
     ]
 
+    this.channelsByNameAndVersion = {}
     this.channelsByName = {}
     this.channelsById = {}
 
     for (const channel of this.channels) {
-      this.channelsByName[channel.meta.name] = channel
+      this.channelsByNameAndVersion[`${channel.meta.name}@${channel.meta.version}`] = channel
       this.channelsById[channel.meta.id] = channel
+
+      if (!this.channelsByName[channel.meta.name]) {
+        this.channelsByName[channel.meta.name] = []
+      }
+      this.channelsByName[channel.meta.name].push(channel)
+    }
+
+    for (const [name, channels] of Object.entries(this.channelsByName)) {
+      this.channelsByName[name] = channels.sort((a, b) => (semver.gt(a.meta.version, b.meta.version) ? -1 : 1))
     }
   }
 
@@ -49,14 +61,14 @@ export class ChannelService extends Service {
 
   async postSetup() {
     for (const channel of this.channels) {
-      if (!(await this.getInDb(channel.meta.name))) {
+      if (!(await this.getInDb(channel.meta.name, channel.meta.version))) {
         await this.createInDb(channel)
       }
     }
   }
 
-  getByName(name: string) {
-    return this.channelsByName[name]
+  getByNameAndVersion(name: string, version: string) {
+    return this.channelsByNameAndVersion[`${name}@${version}`]
   }
 
   getById(id: uuid) {
@@ -67,8 +79,12 @@ export class ChannelService extends Service {
     return this.channels
   }
 
-  private async getInDb(name: string) {
-    const rows = await this.query().where({ name })
+  listByName(name: string) {
+    return this.channelsByName[name]
+  }
+
+  private async getInDb(name: string, version: string) {
+    const rows = await this.query().where({ name, version })
     if (rows?.length) {
       return rows[0]
     } else {
@@ -80,6 +96,7 @@ export class ChannelService extends Service {
     await this.query().insert({
       id: channel.meta.id,
       name: channel.meta.name,
+      version: channel.meta.version,
       lazy: channel.meta.lazy,
       initiable: channel.meta.initiable
     })

packages/server/src/channels/table.ts

@@ -8,8 +8,10 @@ export class ChannelTable extends Table {
 
   create(table: Knex.CreateTableBuilder) {
     table.uuid('id').primary()
-    table.string('name').unique().notNullable()
+    table.string('name').notNullable()
+    table.string('version').notNullable()
     table.boolean('lazy').notNullable()
     table.boolean('initiable').notNullable()
+    table.unique(['name', 'version'])
   }
 }

packages/server/src/conduits/service.ts

@@ -7,6 +7,7 @@ import {
   ServerCache2D,
   Service
 } from '@botpress/messaging-engine'
+import Joi from 'joi'
 import _ from 'lodash'
 import { v4 as uuidv4 } from 'uuid'
 import { ChannelService } from '../channels/service'
@@ -49,7 +50,7 @@ export class ConduitService extends Service {
 
   async create(providerId: uuid, channelId: uuid, config: any): Promise<Conduit> {
     const channel = this.channelService.getById(channelId)
-    const validConfig = await channel.meta.schema.validateAsync(config)
+    const validConfig = await Joi.object(channel.meta.schema).validateAsync(config)
 
     const conduit = {
       id: uuidv4(),
@@ -77,7 +78,7 @@ export class ConduitService extends Service {
   async updateConfig(id: uuid, config: any) {
     const conduit = await this.get(id)
     const channel = this.channelService.getById(conduit.channelId)
-    const validConfig = await channel.meta.schema.validateAsync(config)
+    const validConfig = await Joi.object(channel.meta.schema).validateAsync(config)
 
     this.cacheById.del(id, true)
     this.cacheByProviderAndChannel.del(conduit.providerId, conduit.channelId, true)

packages/server/src/migrations/1.0.2-channel-versions.ts

@@ -0,0 +1,60 @@
+import { getTableId, Migration } from '@botpress/messaging-engine'
+
+export class ChannelVersionsMigration extends Migration {
+  meta = {
+    name: ChannelVersionsMigration.name,
+    description: 'Adds a version column to the msg_channels table',
+    version: '1.0.2'
+  }
+
+  async valid() {
+    return this.trx.schema.hasTable(getTableId('msg_channels'))
+  }
+
+  async applied() {
+    return this.trx.schema.hasColumn(getTableId('msg_channels'), 'version')
+  }
+
+  async up() {
+    await this.trx.schema.alterTable(getTableId('msg_channels'), (table) => {
+      table.dropUnique(['name'])
+      table.string('version').nullable()
+      table.unique(['name', 'version'])
+    })
+
+    const channels: { id: string; name: string }[] = await this.trx(getTableId('msg_channels'))
+
+    for (const channel of channels) {
+      if (channel.name.includes('@')) {
+        const [name, version] = channel.name.split('@')
+        await this.trx(getTableId('msg_channels')).update({ name, version }).where({ id: channel.id })
+      } else {
+        await this.trx(getTableId('msg_channels')).update({ version: '0.1.0' }).where({ id: channel.id })
+      }
+    }
+
+    await this.trx.schema.alterTable(getTableId('msg_channels'), (table) => {
+      table.string('version').notNullable().alter()
+    })
+  }
+
+  async down() {
+    const newChannels: { id: string; name: string; version: string }[] = await this.trx(
+      getTableId('msg_channels')
+    ).whereNot({
+      version: '0.1.0'
+    })
+
+    for (const newChannel of newChannels) {
+      await this.trx(getTableId('msg_channels'))
+        .update({ name: `${newChannel.name}@${newChannel.version}` })
+        .where({ id: newChannel.id })
+    }
+
+    await this.trx.schema.alterTable(getTableId('msg_channels'), (table) => {
+      table.dropUnique(['name', 'version'])
+      table.dropColumn('version')
+      table.unique(['name'])
+    })
+  }
+}

packages/server/src/migrations/index.ts

@@ -3,10 +3,12 @@ import { InitMigration } from './0.0.1-init'
 import { StatusMigration } from './0.1.19-status'
 import { FixClientSchemaMigration } from './0.1.20-fix-client-schema'
 import { ClientTokensMigration } from './0.1.21-client-tokens'
+import { ChannelVersionsMigration } from './1.0.2-channel-versions'
 
 export const Migrations: { new (): Migration }[] = [
   InitMigration,
   StatusMigration,
   FixClientSchemaMigration,
-  ClientTokensMigration
+  ClientTokensMigration,
+  ChannelVersionsMigration
 ]