Merge pull request #25 from botpress/rl_engine

feat(core): Add dialog engine

lerna.json

@@ -1,6 +1,6 @@
 {
   "lerna": "2.11.0",
-  "packages": ["modules/*", "packages/*"],
+  "packages": ["packages/*"],
   "version": "0.0.1",
   "npmClient": "yarn"
 }

package.json

@@ -2,6 +2,7 @@
   "license": "AGPL-3.0-only",
   "devDependencies": {
     "@types/jest": "^23.3.0",
+    "@types/mustache": "^0.8.31",
     "@types/node": "^10.5.2",
     "@types/tmp": "^0.0.33",
     "babel-core": "^6.26.0",

packages/core/package.json

@@ -40,6 +40,8 @@
     "lru-cache": "^4.1.3",
     "moment": "^2.22.2",
     "ms": "^2.1.1",
+    "mustache": "^2.3.1",
+    "mware-ts": "^1.0.1",
     "nanoid": "^1.1.0",
     "pg": "^7.4.3",
     "plur": "^3.0.1",

packages/core/src/botpress.ts

@@ -13,6 +13,9 @@ import { Logger } from './misc/interfaces'
 import { TYPES } from './misc/types'
 import { ModuleLoader } from './module-loader'
 import HTTPServer from './server'
+import { CMSService } from './services/cms/cms-service'
+import { DialogEngine } from './services/dialog/engine'
+import { DialogProcessor } from './services/dialog/processor'
 import { HookService } from './services/hook/hook-service'
 
 @injectable()
@@ -25,7 +28,9 @@ export class Botpress {
 
   constructor(
     @inject(TYPES.ConfigProvider) private configProvider: ConfigProvider,
+    @inject(TYPES.CMSService) private cmsService: CMSService,
     @inject(TYPES.Database) private database: Database,
+    @inject(TYPES.DialogEngine) private dialogEngine: DialogEngine,
     @inject(TYPES.Logger)
     @tagged('name', 'Server')
     private logger: Logger,
@@ -59,6 +64,7 @@ export class Botpress {
   }
 
   private async initializeServices() {
+    await this.cmsService.initialize()
     await this.botLoader.loadAllBots()
   }
 
@@ -77,6 +83,7 @@ export class Botpress {
 
   private async loadModules(): Promise<void> {
     const modules = await this.moduleLoader.getAvailableModules()
+    this.logger.info(`Loaded ${modules.length} modules`)
   }
 
   private async startServer() {

packages/core/src/database/index.ts

@@ -45,10 +45,11 @@ export default class Database {
     await Promise.mapSeries(AllTables, async Tbl => {
       const table = new Tbl(this.knex!)
       await table.bootstrap()
+      this.logger.debug(`Created table '${table.name}'`)
       this.tables.push(table)
     })
 
-    this.logger.debug(`Created ${AllTables.length} tables`)
+    this.logger.info('Created database')
   }
 
   runMigrations() {

packages/core/src/database/tables/bot-specific/dialog_sessions.ts

@@ -0,0 +1,18 @@
+import { Table } from '../../interfaces'
+
+export default class DialogSessionTable extends Table {
+  name: string = 'dialog_sessions'
+
+  async bootstrap() {
+    await this.knex
+      .createTableIfNotExists(this.name, table => {
+        table.increments('id')
+        table.text('state')
+        table.text('context')
+        table.timestamp('active_on')
+        table.text('created_on')
+        table.text('modified_on')
+      })
+      .then()
+  }
+}

packages/core/src/database/tables/index.ts

@@ -1,10 +1,11 @@
 import { ExtendedKnex, Table } from '../interfaces'
 
+import DialogSessionTable from './bot-specific/dialog_sessions'
 import BotsTable from './server-wide/bots'
 import MetadataTable from './server-wide/metadata'
 import MigrationsTable from './server-wide/migrations'
 import ModulesTable from './server-wide/modules'
 
-const tables: (typeof Table)[] = [BotsTable, MigrationsTable, MetadataTable, ModulesTable]
+const tables: (typeof Table)[] = [BotsTable, DialogSessionTable, MigrationsTable, MetadataTable, ModulesTable]
 
 export default <(new (knex: ExtendedKnex) => Table)[]>tables

packages/core/src/database/tables/server-wide/bots.ts

@@ -4,19 +4,19 @@ export default class BotsTable extends Table {
   name: string = 'srv_bots'
 
   async bootstrap() {
-    await this.knex.schema.dropTableIfExists(this.name).then(() => {
-      this.knex.schema
-        .createTable(this.name, table => {
-          table.increments('id')
-          table.string('name')
-          table.string('version')
-          table.string('description')
-          table.string('author')
-          table.string('license')
-          table.timestamps(true, true)
-        })
-        .then(() => {
-          // TODO: Use knex seed api instead
+    await this.knex
+      .createTableIfNotExists(this.name, table => {
+        table.increments('id')
+        table.string('name')
+        table.string('version')
+        table.string('description')
+        table.string('author')
+        table.string('license')
+        table.timestamps(true, true)
+      })
+      .then(created => {
+        // TODO: Use knex seed api instead
+        if (created) {
           return this.knex
             .insert({
               id: 123,
@@ -27,7 +27,7 @@ export default class BotsTable extends Table {
               license: 'AGPL-3.0'
             })
             .into(this.name)
-        })
-    })
+        }
+      })
   }
 }

packages/core/src/misc/types.ts

@@ -21,7 +21,10 @@ const TYPES = {
   IsPackaged: Symbol.for('IsPackaged'),
   Queue: Symbol.for('Queue'),
   HookService: Symbol.for('HookService'),
-  EventEngine: Symbol.for('EventEngine')
+  EventEngine: Symbol.for('EventEngine'),
+  DialogEngine: Symbol.for('DialogEngine'),
+  SessionRepository: Symbol.for('StateManager'),
+  SessionService: Symbol.for('SessionService')
 }
 
 export { TYPES }

packages/core/src/repositories/repositories.inversify.ts

@@ -4,7 +4,9 @@ import { TYPES } from '../misc/types'
 
 import { BotRepository } from './bot-repository'
 import { GhostBotRepository } from './ghost-bot-repository'
+import { KnexSessionRepository, SessionRepository } from './session-repository'
 
 export const RepositoriesContainerModule = new ContainerModule((bind: interfaces.Bind) => {
   bind<BotRepository>(TYPES.BotRepository).to(GhostBotRepository)
+  bind<SessionRepository>(TYPES.SessionRepository).to(KnexSessionRepository)
 })

packages/core/src/repositories/session-repository.ts

@@ -0,0 +1,109 @@
+import { inject, injectable } from 'inversify'
+
+import { ExtendedKnex } from '../database/interfaces'
+import { TYPES } from '../misc/types'
+
+export type DialogSession = {
+  id: string
+  state: string
+  context: any
+  created_on: string
+  modified_on: string
+  active_on: string
+}
+
+export interface SessionRepository {
+  get(id: string): Promise<DialogSession>
+  upsert(session: DialogSession)
+  delete(id: string)
+  update(session: DialogSession)
+}
+
+@injectable()
+export class KnexSessionRepository implements SessionRepository {
+  private readonly tableName = 'dialog_sessions'
+
+  constructor(@inject(TYPES.Database) private knex: ExtendedKnex) {}
+
+  async get(id: string): Promise<DialogSession> {
+    const session = <DialogSession>await this.knex(this.tableName)
+      .where({ id })
+      .limit(1)
+      .get(0)
+      .then()
+    return session ? this.jsonParse(session) : this.createSession(id)
+  }
+
+  async upsert(session: DialogSession) {
+    const params = {
+      tableName: this.tableName,
+      id: session.id,
+      state: JSON.stringify(session.state),
+      context: JSON.stringify(session.context),
+      now: this.knex.date.now()
+    }
+
+    let sql: string
+    if (this.knex.isLite) {
+      sql = `
+      INSERT OR REPLACE INTO :tableName (id, state, context, active_on, created_on, modified_on)
+      VALUES (:id, :state, :context, :now, :now, :now)
+      `
+    } else {
+      sql = `
+      INSERT INTO :tableName (id, state, context, active_on, created_on, modified_on)
+      VALUES (:id, :state, :context, :now, :now, :now)
+      ON CONFLICT (id) DO UPDATE
+      SET state = :state, context = :context, active_on = :now, modified_on = :now
+      `
+    }
+
+    return this.knex.raw(sql, params)
+  }
+
+  async update(session: DialogSession) {
+    session.modified_on = this.knex.date.now().toString()
+
+    return await this.knex(this.tableName)
+      .update(session)
+      .where('id', session.id)
+      .then()
+  }
+
+  async delete(id: string) {
+    await this.knex(this.tableName)
+      .where({ id })
+      .del()
+      .then()
+  }
+
+  private async createSession(id: string): Promise<DialogSession> {
+    const params = {
+      tableName: this.tableName,
+      id,
+      state: '',
+      context: '',
+      now: this.knex.date.now()
+    }
+
+    const sql = `
+    INSERT INTO :tableName: (id, state, active_on, created_on)
+    VALUES (:id, :state, :now, :now)
+    ON CONFLICT (id) DO UPDATE
+      SET created_on = :now, active_on = :now, modified_on = :now, state = :state, context = :context`
+
+    await this.knex.raw(sql, params)
+    const session = <DialogSession>await this.knex(this.tableName)
+      .select('*')
+      .where({ id })
+      .then()
+
+    return this.jsonParse(session)
+  }
+
+  private jsonParse(session: DialogSession) {
+    session.context = JSON.parse(session.context)
+    session.state = JSON.parse(session.state)
+    return session
+  }
+}

packages/core/src/services/cms/cms-service.ts

@@ -1,4 +1,4 @@
-import { inject, injectable, postConstruct, tagged } from 'inversify'
+import { inject, injectable, tagged } from 'inversify'
 import _ from 'lodash'
 import nanoid from 'nanoid'
 import path from 'path'
@@ -36,7 +36,6 @@ export class CMSService implements IDisposeOnExit {
   }
 
   // TODO Test this class
-  @postConstruct()
   async initialize() {
     this.ghost.global().addRootFolder(this.typesDir, { filesGlob: '**.js' })
     this.ghost.forAllBots().addRootFolder(this.elementsDir, { filesGlob: '**.json' })
@@ -74,11 +73,11 @@ export class CMSService implements IDisposeOnExit {
       contentElements = _.concat(contentElements, fileContentElements)
     }
 
-    return Promise.mapSeries(contentElements, element =>
-      this.memDb(this.contentTable)
+    return Promise.map(contentElements, async element => {
+      await this.memDb(this.contentTable)
         .insert(this.transformItemApiToDb(botId, element))
-        .then()
-    )
+        .then(() => this.logger.debug(`Loaded content '${element.id}' for '${botId}'`))
+    })
   }
 
   private async loadContentTypesFromFiles(): Promise<void> {
@@ -92,23 +91,21 @@ export class CMSService implements IDisposeOnExit {
     this.sandbox = new SafeCodeSandbox(codeFiles)
     let filesLoaded = 0
 
-    try {
-      for (const file of this.sandbox.ls()) {
-        try {
-          const filename = path.basename(file)
-          if (filename.startsWith('_')) {
-            // File to exclude
-            continue
-          }
-          await this.loadContentTypeFromFile(file)
-          filesLoaded++
-        } catch (e) {
-          this.logger.error(e, `Could not load Content Type "${file}"`)
+    for (const file of this.sandbox.ls()) {
+      try {
+        const filename = path.basename(file)
+        if (filename.startsWith('_')) {
+          // File to exclude
+          continue
         }
+        await this.loadContentTypeFromFile(file)
+        filesLoaded++
+      } catch (e) {
+        this.logger.error(e, `Could not load Content Type "${file}"`)
       }
-    } finally {
-      this.logger.debug(`Loaded ${filesLoaded} content types`)
     }
+
+    this.logger.info(`Loaded ${filesLoaded} content types`)
   }
 
   private async loadContentTypeFromFile(fileName: string): Promise<void> {