feat(migration): implement migration service (#142)

* meta service * meta schema * migrations start * fix * http version use meta * semver * list migs in correct order * progress * fix * k * fix * fixes * dry run * logger.window * MIGRATE_TARGET * refact * down migration * fix * ShutDownSignal * refact * refact * refact * fix * yargs * yargs+ * migrate command * refact * fix * fix * fix * pr comments * remove user token migration * fix * tests
botpress · Aug 24, 2021 · a6fef7c · a6fef7c
1 parent e2a1216
commit a6fef7c
Show file tree

Hide file tree

Showing 16 changed files with 515 additions and 54 deletions.
diff --git a/packages/server/package.json b/packages/server/package.json
@@ -19,7 +19,9 @@
     "@types/ms": "^0.7.31",
     "@types/node": "^12.20.20",
     "@types/redlock": "^4.0.2",
-    "@types/uuid": "^8.3.1"
+    "@types/semver": "^7.3.8",
+    "@types/uuid": "^8.3.1",
+    "@types/yargs": "^17.0.2"
   },
   "dependencies": {
     "@botpress/messaging-base": "0.0.1",
@@ -50,12 +52,14 @@
     "pg": "^8.6.0",
     "portfinder": "^1.0.28",
     "redlock": "^4.2.0",
+    "semver": "^7.3.5",
     "smooch-core": "^8.11.4",
     "socket.io": "^4.1.3",
     "sqlite3": "^5.0.2",
     "telegraf": "^3.27.1",
     "twilio": "^3.61.0",
     "uuid": "^8.3.2",
+    "yargs": "^17.1.1",
     "yn": "^4.0.0"
   }
 }
diff --git a/packages/server/src/api.ts b/packages/server/src/api.ts
@@ -10,8 +10,6 @@ import { SocketManager } from './socket/manager'
 import { SyncApi } from './sync/api'
 import { UserApi } from './users/api'
 
-const pkg = require('../package.json')
-
 export class Api {
   public readonly sockets: SocketManager
 
@@ -90,6 +88,6 @@ export class Api {
   }
 
   private version(_req: Request, res: Response) {
-    res.send(pkg.version)
+    res.send(this.app.meta.app().version)
   }
 }
diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts
@@ -15,6 +15,8 @@ import { KvsService } from './kvs/service'
 import { LoggerService } from './logger/service'
 import { MappingService } from './mapping/service'
 import { MessageService } from './messages/service'
+import { MetaService } from './meta/service'
+import { MigrationService } from './migration/service'
 import { PostService } from './post/service'
 import { ProviderService } from './providers/service'
 import { SocketService } from './socket/service'
@@ -27,6 +29,8 @@ export class App {
   logger: LoggerService
   config: ConfigService
   database: DatabaseService
+  meta: MetaService
+  migration: MigrationService
   crypto: CryptoService
   distributed: DistributedService
   caching: CachingService
@@ -53,6 +57,8 @@ export class App {
     this.logger = new LoggerService()
     this.config = new ConfigService()
     this.database = new DatabaseService(this.config)
+    this.meta = new MetaService(this.database)
+    this.migration = new MigrationService(this.database, this.meta)
     this.crypto = new CryptoService(this.config)
     this.distributed = new DistributedService(this.config)
     this.caching = new CachingService(this.distributed)
@@ -123,6 +129,8 @@ export class App {
     await this.logger.setup()
     await this.config.setup()
     await this.database.setup()
+    await this.meta.setup()
+    await this.migration.setup()
     await this.crypto.setup()
     await this.distributed.setup()
     await this.caching.setup()

diff --git a/packages/server/src/base/errors.ts b/packages/server/src/base/errors.ts
@@ -0,0 +1,5 @@
+export class ShutDownSignal extends Error {
+  constructor() {
+    super()
+  }
+}
diff --git a/packages/server/src/global.ts b/packages/server/src/global.ts
@@ -20,6 +20,11 @@ export interface MessagingEnv {
   BATCHING_ENABLED?: string
   NO_LOGO?: string
   SINGLE_LINE_LOGGING?: string
+  AUTO_MIGRATE?: string
+  MIGRATE_TARGET?: string
+  MIGRATE_CMD?: string
+  MIGRATE_DRYRUN?: string
+  TESTMIG_DB_VERSION?: string
 }
 
 declare global {

diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts
@@ -1,16 +1,67 @@
 import './rewire'
 import express from 'express'
+import yargs from 'yargs'
 import { Api } from './api'
 import { App } from './app'
 import { Launcher } from './launcher'
 
 // Set NODE_ENV to production when starting messaging using the binary version
 process.env.NODE_ENV = (<any>process).pkg ? 'production' : process.env.NODE_ENV
 
-const router = express()
+const launch = async () => {
+  const router = express()
 
-const app = new App()
-const api = new Api(app, router)
+  const app = new App()
+  const api = new Api(app, router)
 
-const launcher = new Launcher(router, app, api)
-void launcher.launch()
+  const launcher = new Launcher(router, app, api)
+  await launcher.launch()
+}
+
+void yargs
+  .scriptName('')
+  .command(
+    ['$0', 'serve'],
+    'Start server',
+    (yargs) => {
+      return yargs.options({ autoMigrate: { type: 'boolean', default: false } })
+    },
+    async (argv) => {
+      if (argv.autoMigrate) {
+        process.env.AUTO_MIGRATE = 'true'
+      }
+
+      await launch()
+    }
+  )
+  .command('migrate', 'Migrate database', (yargs) => {
+    const start = async (cmd: string, target: string, dry: boolean) => {
+      process.env.AUTO_MIGRATE = 'true'
+      if (cmd) {
+        process.env.MIGRATE_CMD = cmd
+      }
+      if (target) {
+        process.env.MIGRATE_TARGET = target
+      }
+      if (dry) {
+        process.env.MIGRATE_DRYRUN = 'true'
+      }
+      await launch()
+    }
+
+    return yargs
+      .command('up', 'Migrate to the latest version (unless --target is specified)', {}, async (argv) => {
+        await start('up', argv.target as string, argv.dry as boolean)
+      })
+      .command('down', 'Downgrade to a previous version (--target must be specified)', {}, async (argv) => {
+        await start('down', argv.target as string, argv.dry as boolean)
+      })
+      .option('target', {
+        alias: 't',
+        describe: 'Target a specific version'
+      })
+      .option('dryrun', {
+        alias: 'dry',
+        describe: 'Displays the list of migrations that will be executed, without applying the changes'
+      })
+  }).argv
diff --git a/packages/server/src/launcher.ts b/packages/server/src/launcher.ts
@@ -7,6 +7,7 @@ import portfinder from 'portfinder'
 import yn from 'yn'
 import { Api } from './api'
 import { App } from './app'
+import { ShutDownSignal } from './base/errors'
 import { Logger } from './logger/types'
 
 const pkg = require('../package.json')
@@ -48,35 +49,42 @@ export class Launcher {
   }
 
   async launch() {
-    await this.app.config.setupEnv()
-    this.printLogo()
-    await this.app.setup()
-    this.printChannels()
-
-    await this.api.setup()
-
-    let port = process.env.PORT || this.app.config.current.server?.port
-    if (!port) {
-      portfinder.basePort = 3100
-      port = (await portfinder.getPortPromise()).toString()
-    }
+    try {
+      await this.app.config.setupEnv()
+      this.printLogo()
+      await this.app.setup()
+      this.printChannels()
+
+      await this.api.setup()
+
+      let port = process.env.PORT || this.app.config.current.server?.port
+      if (!port) {
+        portfinder.basePort = 3100
+        port = (await portfinder.getPortPromise()).toString()
+      }
 
-    const server = this.express.listen(port)
-    await this.api.sockets.setup(server)
-    this.httpTerminator = createHttpTerminator({ server, gracefulTerminationTimeout: this.shutdownTimeout })
+      const server = this.express.listen(port)
+      await this.api.sockets.setup(server)
+      this.httpTerminator = createHttpTerminator({ server, gracefulTerminationTimeout: this.shutdownTimeout })
 
-    if (!yn(process.env.SPINNED)) {
-      this.logger.info(`Server is listening at: http://localhost:${port}`)
+      if (!yn(process.env.SPINNED)) {
+        this.logger.info(`Server is listening at: http://localhost:${port}`)
+
+        const externalUrl = process.env.EXTERNAL_URL || this.app.config.current.server?.externalUrl
+        if (externalUrl?.length) {
+          this.logger.info(`Server is exposed at: ${externalUrl}`)
+        }
+      } else {
+        this.logger.info(clc.blackBright(`Messaging is listening at: http://localhost:${port}`))
+      }
 
-      const externalUrl = process.env.EXTERNAL_URL || this.app.config.current.server?.externalUrl
-      if (externalUrl?.length) {
-        this.logger.info(`Server is exposed at: ${externalUrl}`)
+      await this.app.monitor()
+    } catch (e) {
+      if (!(e instanceof ShutDownSignal)) {
+        this.logger.error(e, 'Error occurred starting server')
       }
-    } else {
-      this.logger.info(clc.blackBright(`Messaging is listening at: http://localhost:${port}`))
+      await this.shutDown()
     }
-
-    await this.app.monitor()
   }
 
   async shutDown(code?: number) {
@@ -103,20 +111,7 @@ export class Launcher {
       return
     }
 
-    const centerText = (text: string, width: number, indent: number = 0) => {
-      const padding = Math.floor((width - text.length) / 2)
-      return _.repeat(' ', padding + indent) + text + _.repeat(' ', padding)
-    }
-
-    const width = yn(process.env.SPINNED) ? 45 : 33
-    this.logger.info(
-      '========================================\n' +
-        clc.bold(centerText('Botpress Messaging', 40, width)) +
-        '\n' +
-        clc.blackBright(centerText(`Version ${pkg.version}`, 40, width)) +
-        '\n' +
-        centerText('========================================', 40, width)
-    )
+    this.logger.window([clc.bold('Botpress Messaging'), clc.blackBright(`Version ${pkg.version}`)])
   }
 
   private printChannels() {

diff --git a/packages/server/src/logger/types.ts b/packages/server/src/logger/types.ts
@@ -1,4 +1,5 @@
 import clc from 'cli-color'
+import _ from 'lodash'
 import moment from 'moment'
 import yn from 'yn'
 
@@ -21,32 +22,63 @@ export class Logger {
     [LoggerLevel.Info]: clc.green
   }
 
-  constructor(private scope: string) {}
+  constructor(private scope: string, private logPrefix?: string) {}
 
   sub(scope: string) {
     return new Logger(`${this.scope}:${scope}`)
   }
 
+  prefix(prefix: string) {
+    return new Logger(this.scope, prefix)
+  }
+
   info(message: string, data?: Param) {
-    this.print([message, data], LoggerLevel.Info)
+    this.printPrefix([message, data], LoggerLevel.Info)
   }
 
   debug(message: string, data?: Param) {
-    this.print([message, data], LoggerLevel.Debug)
+    this.printPrefix([message, data], LoggerLevel.Debug)
   }
 
   warn(message: string, data?: Param) {
-    this.print([message, data], LoggerLevel.Warn)
+    this.printPrefix([message, data], LoggerLevel.Warn)
+  }
+
+  window(lines: string[], level = LoggerLevel.Info) {
+    const line = '========================================'
+    this.print(
+      [
+        line +
+          '\n' +
+          lines
+            .map((x) => this.center(this.logPrefix ? `${clc.blackBright(this.logPrefix)}${x}` : x, line.length))
+            .join('\n') +
+          '\n' +
+          this.center(line, line.length)
+      ],
+      level
+    )
   }
 
-  error(error: undefined, message: string, data?: Param): void
-  error(error: Error, message?: string, data?: Param): void
   error(error: Error | undefined, message?: string, data?: Param) {
     if (message?.length && message[message.length - 1] !== '.') {
       message += '.'
     }
 
-    this.print([message, data, error?.stack], LoggerLevel.Error)
+    this.printPrefix([message, data, error?.stack], LoggerLevel.Error)
+  }
+
+  private center(text: string, width: number) {
+    const indent = (yn(process.env.SPINNED) ? 37 : 25) + this.scope.length
+    const padding = Math.floor((width - clc.strip(text).length) / 2)
+    return _.repeat(' ', padding + indent) + text + _.repeat(' ', padding)
+  }
+
+  private printPrefix(params: Param[], level: LoggerLevel) {
+    if (this.logPrefix) {
+      params[0] = `${clc.blackBright(this.logPrefix)}${params[0]}`
+    }
+    this.print(params, level)
   }
 
   private print(params: Param[], level: LoggerLevel) {