Merge pull request #1664 from botpress/ML

feat(cms): added multilanguage support

docs/guide/docs/build/content.md

@@ -103,3 +103,11 @@ function renderElement(data, channel) {
   return []
 }
 ```
+
+## Translation
+
+Your bots can support multiple languages. If a specific translation is not available for the current language, the bot will use the default language. The first time a user chats with your bot, we extract the browser's language and save it as a user attribute (available on the event as `user.language`).
+
+Once that property is set, it will never be overwritten. Therefore, it is possible for you to ask the user what is his preferred language, or to use the NLU engine to detect it.
+
+When rendering content elements, we will try to render the user's configured language, otherwise it will use the bot's default one.

docs/guide/docs/developers/migrate.md

@@ -3,6 +3,42 @@ id: migrate
 title: Migration
 ---
 
+## Migration from 11.7 to 11.8
+
+### Channel-Web Refactor
+
+There was a big refactor of the module to make it easier to customize (custom components and CSS). Every styling classes were extracted in a single file, `default.css`. If you previously customized the webchat, some changes will be required.
+
+Please refer to [default.css](https://github.com/botpress/botpress/tree/master/modules/channel-web/assets/default.css) for the different classes
+
+### Multiple Languages
+
+This feature should have no impact on existing users. If you have created custom content types, however, slight modifications will be required.
+
+- The method `computePreviewText`, should return `undefined` if the required property is not set, instead of trying to display something
+
+```js
+// For example
+computePreviewText: formData => 'Text: ' + formData.text
+
+// Would become
+computePreviewText: formData => formData.text && 'Text: ' + formData.text
+```
+
+- When editing a language where a translation is missing, the type of the field must be `i18n_field` to display the original text of the default language.
+
+```js
+// Example in the text content type:
+ uiSchema: {
+  text: {
+    // This custom field includes a placeholder with the default language text
+    'ui:field': 'i18n_field',
+
+    // This isn't mandatory, but it will display a textarea instead of a simple input field.
+    $subtype: 'textarea'
+  },
+```
+
 ## Migration from 11.5 to 11.6
 
 ### Custom Modules (target: developers)

modules/basic-skills/src/views/full/choice.jsx

@@ -84,7 +84,7 @@ export class Choice extends React.Component {
 
   onContentChanged = (element, force = false) => {
     if (element && (force || element.id !== this.state.contentId)) {
-      this.choices = _.get(element, 'formData.choices') || [] //CHANGED
+      this.choices = _.get(element, 'formData.choices$' + this.props.contentLang) || []
       const initialKeywords = element.id === this.state.contentId ? this.state.keywords : {}
       const keywords = this.choices.reduce((acc, v) => {
         if (!acc[v.value]) {

modules/builtin/src/bot-templates/empty-bot/bot.config.json

@@ -16,5 +16,7 @@
   },
   "logs": {
     "expiration": "1 week"
-  }
+  },
+  "defaultLanguage": "en",
+  "languages": ["en"]
 }

modules/builtin/src/bot-templates/small-talk/bot.config.json

@@ -16,5 +16,7 @@
   },
   "logs": {
     "expiration": "1 week"
-  }
+  },
+  "defaultLanguage": "en",
+  "languages": ["en"]
 }

modules/builtin/src/bot-templates/welcome-bot/bot.config.json

@@ -17,10 +17,6 @@
   "logs": {
     "expiration": "1 week"
   },
-  "id": "welcome-bot",
-  "name": "Welcome Bot",
-  "category": null,
-  "disabled": false,
-  "private": false,
-  "details": {}
+  "defaultLanguage": "en",
+  "languages": ["en"]
 }

modules/builtin/src/content-types/card.js

@@ -42,6 +42,6 @@ module.exports = {
     }
   },
 
-  computePreviewText: formData => `Card: ${formData.title}`,
+  computePreviewText: formData => formData.title && `Card: ${formData.title}`,
   renderElement: (data, channel) => Carousel.renderElement({ items: [data], ...data }, channel)
 }

modules/builtin/src/content-types/carousel.js

@@ -116,6 +116,6 @@ module.exports = {
       ...base.typingIndicators
     }
   },
-  computePreviewText: formData => `Carousel: (${formData.items.length}) ${formData.items[0].title}`,
+  computePreviewText: formData => formData.items && `Carousel: (${formData.items.length}) ${formData.items[0].title}`,
   renderElement: renderElement
 }

modules/builtin/src/content-types/image.js

@@ -80,17 +80,24 @@ module.exports = {
     }
   },
 
-  uiSchema: {},
+  uiSchema: {
+    title: {
+      'ui:field': 'i18n_field'
+    }
+  },
 
   computePreviewText: formData => {
-    let fileName = path.basename(formData.image)
+    if (!formData.image) {
+      return
+    }
 
+    let fileName = path.basename(formData.image)
     if (fileName.includes('-')) {
       fileName = tail(fileName.split('-')).join('-')
     }
 
     const title = formData.title ? ' | ' + formData.title : ''
-    return `Image (${fileName})${title}`
+    return `Image (${fileName}) ${title}`
   },
 
   renderElement: renderElement

modules/builtin/src/content-types/single_choice.js

@@ -92,14 +92,15 @@ module.exports = {
   },
 
   uiSchema: {
-    variations: {
-      'ui:options': {
-        orderable: false
-      }
+    text: {
+      'ui:field': 'i18n_field'
+    },
+    choices: {
+      'ui:field': 'i18n_array'
     }
   },
-
-  computePreviewText: formData => `Choices (${formData.choices.length}) ${formData.text}`,
+  computePreviewText: formData =>
+    formData.choices && formData.text && `Choices (${formData.choices.length}) ${formData.text}`,
   renderElement: renderElement,
   hidden: true
 }

modules/builtin/src/content-types/text.js

@@ -70,15 +70,16 @@ module.exports = {
 
   uiSchema: {
     text: {
-      'ui:widget': 'textarea'
+      'ui:field': 'i18n_field',
+      $subtype: 'textarea'
     },
     variations: {
       'ui:options': {
         orderable: false
       }
     }
   },
+  computePreviewText: formData => formData.text && 'Text: ' + formData.text,
 
-  computePreviewText: formData => 'Text: ' + formData.text,
   renderElement: renderElement
 }

modules/channel-web/src/backend/api.ts

@@ -115,7 +115,7 @@ export default async (bp: typeof sdk, db: Database) => {
         return res.status(400).send(ERR_USER_ID_REQ)
       }
 
-      await bp.users.getOrCreateUser('web', userId) // Create the user if it doesn't exist
+      const user = await bp.users.getOrCreateUser('web', userId)
       const payload = req.body || {}
 
       let { conversationId = undefined } = req.query || {}
@@ -132,11 +132,13 @@ export default async (bp: typeof sdk, db: Database) => {
       }
 
       if (payload.type === 'visit') {
-        const { timezone } = payload
+        const { timezone, language } = payload
         const isValidTimezone = _.isNumber(timezone) && timezone >= -12 && timezone <= 14 && timezone % 0.5 === 0
+        const isValidLanguage = language.length < 4 && !_.get(user, 'result.attributes.language')
 
         const newAttributes = {
-          ...(isValidTimezone && { timezone })
+          ...(isValidTimezone && { timezone }),
+          ...(isValidLanguage && { language })
         }
 
         if (Object.getOwnPropertyNames(newAttributes).length) {

modules/channel-web/src/views/lite/main.jsx

@@ -280,10 +280,12 @@ export default class Web extends React.Component {
       .then(this.fetchConversations)
       .then(this.fetchCurrentConversation)
       .then(() => {
+        const locale = navigator.language || navigator.userLanguage
         this.handleSendData({
           type: 'visit',
           text: 'User visit',
-          timezone: new Date().getTimezoneOffset() / 60
+          timezone: new Date().getTimezoneOffset() / 60,
+          language: locale && locale.substring(0, locale.indexOf('-'))
         }).catch(this.checkForExpiredExternalToken)
       })
   }

src/bp/common/supported-languages.ts

@@ -0,0 +1 @@
+export default ['en', 'fr', 'ar', 'ja', 'pt']

src/bp/common/validation.ts

@@ -3,6 +3,7 @@ import Joi from 'joi'
 export const BOTID_REGEX = /^[A-Z0-9]+[A-Z0-9_-]{2,}[A-Z0-9]+$/i
 
 export const isValidBotId = (botId: string): boolean => BOTID_REGEX.test(botId)
+import supportedLangs from './supported-languages'
 
 export const BotCreationSchema = Joi.object().keys({
   id: Joi.string()
@@ -40,6 +41,16 @@ export const BotEditSchema = Joi.object().keys({
     .required(),
   disabled: Joi.bool(),
   private: Joi.bool(),
+  defaultLanguage: Joi.string()
+    .valid(supportedLangs)
+    .min(2)
+    .max(3),
+  languages: Joi.array().items(
+    Joi.string()
+      .min(2)
+      .max(3)
+      .valid(supportedLangs)
+  ),
   details: {
     website: Joi.string()
       .uri()

src/bp/core/api.ts

@@ -180,9 +180,7 @@ const cms = (cmsService: CMSService, mediaService: MediaService): typeof sdk.cms
     getContentElements(botId: string, ids: string[]): Promise<any[]> {
       return cmsService.getContentElements(botId, ids)
     },
-    listContentElements(botId: string, contentTypeId?: string, searchParams?: sdk.SearchParams): Promise<any> {
-      return cmsService.listContentElements(botId, contentTypeId, searchParams)
-    },
+    listContentElements: cmsService.listContentElements.bind(cmsService),
     deleteContentElements: cmsService.deleteContentElements.bind(cmsService),
     getAllContentTypes(botId?: string): Promise<any[]> {
       return cmsService.getAllContentTypes(botId)

src/bp/core/botpress.ts

@@ -4,7 +4,7 @@ import { WrapErrorsWith } from 'errors'
 import fse from 'fs-extra'
 import { inject, injectable, tagged } from 'inversify'
 import { AppLifecycle, AppLifecycleEvents } from 'lifecycle'
-import _ from 'lodash'
+import _, { Partial } from 'lodash'
 import moment from 'moment'
 import nanoid from 'nanoid'
 import path from 'path'
@@ -152,6 +152,14 @@ export class Botpress {
         'Your pipeline has more than a single stage. To enable the pipeline feature, please upgrade to Botpress Pro.'
       )
     }
+    const bots = await this.botService.getBots()
+    bots.forEach(bot => {
+      if (!process.IS_PRO_ENABLED && bot.languages && bot.languages.length > 1) {
+        throw new Error(
+          'A bot has more than a single language. To enable the multilangual feature, please upgrade to Botpress Pro.'
+        )
+      }
+    })
     if (process.IS_PRO_ENABLED && !process.CLUSTER_ENABLED) {
       this.logger.warn(
         'Botpress can be run on a cluster. If you want to do so, make sure Redis is running and properly configured in your environment variables'
@@ -211,9 +219,10 @@ export class Botpress {
     if (pipeline.length > 4) {
       this.logger.warn('It seems like you have more than 4 stages in your pipeline, consider to join stages together.')
     }
-    // @deprecated > 11: bot will always include default pipeline stage
-    const changes = await this._ensureBotsDefineStage(bots, pipeline[0])
-    if (changes) {
+
+    // @deprecated > 11: bot will always include default pipeline stage & must have a default language
+    const botConfigChanged = await this._ensureBotConfigCorrect(bots, pipeline[0])
+    if (botConfigChanged) {
       bots = await this.botService.getBots()
     }
 
@@ -236,23 +245,30 @@ export class Botpress {
   }
 
   // @deprecated > 11: bot will always include default pipeline stage
-  private async _ensureBotsDefineStage(bots: Map<string, BotConfig>, stage: sdk.Stage): Promise<Boolean> {
+  private async _ensureBotConfigCorrect(bots: Map<string, BotConfig>, stage: sdk.Stage): Promise<Boolean> {
     let hasChanges = false
     await Promise.mapSeries(bots.values(), async bot => {
+      const updatedConfig: any = {}
+
+      if (!bot.defaultLanguage) {
+        this.logger.warn(`Bot "${bot.id}" doesn't have a default language, which is now required, go to your admin console to fix this issue.`)
+        updatedConfig.disabled = true
+      }
+
       if (!bot.pipeline_status) {
-        hasChanges = true
-        const pipeline_migration_configs = {
-          pipeline_status: <sdk.BotPipelineStatus>{
-            current_stage: {
-              id: stage.id,
-              promoted_by: 'system',
-              promoted_on: new Date()
-            }
-          },
-          locked: false
+        updatedConfig.locked = false
+        updatedConfig.pipeline_status = {
+          current_stage: {
+            id: stage.id,
+            promoted_by: 'system',
+            promoted_on: new Date()
+          }
         }
+      }
 
-        await this.configProvider.mergeBotConfig(bot.id, pipeline_migration_configs)
+      if (Object.getOwnPropertyNames(updatedConfig).length) {
+        hasChanges = true
+        await this.configProvider.mergeBotConfig(bot.id, updatedConfig)
       }
     })
 

src/bp/core/config/bot.config.ts

@@ -17,6 +17,8 @@ export type BotConfig = {
   }
   dialog?: DialogConfig
   logs?: LogsConfig
+  defaultLanguage: string
+  languages: string[]
   locked: boolean
   pipeline_status: BotPipelineStatus
 }

src/bp/core/config/config-loader.ts

@@ -62,7 +62,7 @@ export class ConfigProvider {
     await this.ghostService.forBot(botId).upsertFile('/', 'bot.config.json', JSON.stringify(config, undefined, 2))
   }
 
-  async mergeBotConfig(botId, partialConfig: PartialDeep<BotConfig>): Promise<BotConfig> {
+  async mergeBotConfig(botId: string, partialConfig: PartialDeep<BotConfig>): Promise<BotConfig> {
     const originalConfig = await this.getBotConfig(botId)
     const config = _.merge(originalConfig, partialConfig)
     await this.setBotConfig(botId, config)

src/bp/core/routers/admin/bots.ts

@@ -67,7 +67,7 @@ export class BotsRouter extends CustomRouter {
       '/',
       this.needPermissions('write', this.resource),
       this.asyncMiddleware(async (req, res) => {
-        const bot = <BotConfig>_.pick(req.body, ['id', 'name', 'category'])
+        const bot = <BotConfig>_.pick(req.body, ['id', 'name', 'category', 'defaultLanguage'])
 
         this.workspaceService.assertUserExists(req.tokenUser!.email)