Merge pull request #2250 from botpress/ya-ghost-fix2

fix(ghost): more work toward stable ghost

src/bp/bpfs.ts

@@ -0,0 +1,237 @@
+import axios, { AxiosInstance, AxiosRequestConfig } from 'axios'
+import chalk from 'chalk'
+import { bytesToString } from 'core/misc/utils'
+import followRedirects from 'follow-redirects'
+import fse from 'fs-extra'
+import glob from 'glob'
+import _ from 'lodash'
+import path from 'path'
+import rimraf from 'rimraf'
+
+import { extractArchive } from './core/misc/archive'
+import { createArchiveFromFolder } from './core/misc/archive'
+import { asBytes } from './core/misc/utils'
+import { BpfsScopedChange, FileChange } from './core/services'
+
+// This is a dependency of axios, and sets the default body limit to 10mb. Need it to be higher
+followRedirects.maxBodyLength = asBytes('500mb')
+
+// If the push will cause one of these actions, then a force will be required
+const blockingActions = ['del', 'edit']
+
+/**
+ * These files must be ignored when pushing them to the remote host.
+ * assets and models are auto-generated and .js.map files are not required
+ */
+const pushedArchiveIgnoredFiles = ['assets/**/*', 'bots/*/models/**', '**/*.js.map']
+
+interface ProcessedChanges {
+  localFiles: string[]
+  blockingChanges: FileChange[]
+  changeList: FileChange[]
+}
+
+class BPFS {
+  private serverUrl: string
+  private authToken: string
+  private targetDir: string
+  private sourceDir: string
+
+  constructor(args, action: string) {
+    this.serverUrl = args.url.replace(/\/+$/, '')
+    this.authToken = args.authToken
+    this.targetDir = args.targetDir && path.resolve(args.targetDir)
+    this.sourceDir = args.sourceDir && path.resolve(args.sourceDir)
+
+    if (!this.serverUrl || !this.authToken || !this.authToken.length) {
+      this._endWithError(`Missing parameter "url" or "authToken"`)
+    }
+
+    if (action === 'pull' && !this.targetDir) {
+      this._endWithError(`Target directory is not valid: "${this.targetDir}"`)
+    }
+
+    if (action === 'push' && !this.sourceDir) {
+      this._endWithError(`Source directory must be set: "${this.sourceDir}"`)
+    }
+  }
+
+  async pullChanges() {
+    const cleanBefore = process.argv.includes('--clean')
+    const axiosClient = this._getPullAxiosClient()
+    const dryRun = process.argv.includes('--dry')
+
+    try {
+      // We clear only those two folders, so assets are preserved
+      if (cleanBefore) {
+        console.log(chalk.blue(`Cleaning data folder before pulling data...`))
+        await this._clearDir(path.join(this.targetDir, 'global'))
+        await this._clearDir(path.join(this.targetDir, 'bots'))
+      } else if (fse.existsSync(this.targetDir)) {
+        const fileCount = await this._filesCount(this.targetDir)
+        console.log(chalk.blue(`Remote files will be pulled in an existing folder containing ${fileCount} files`))
+      }
+
+      console.log(chalk.blue(`Pulling all remote changes from ${this.serverUrl} to ${this.targetDir} ...`))
+
+      const { data: archive } = await axiosClient.get('export')
+      if (dryRun) {
+        console.log(chalk.yellow(`Dry run completed. Successfully downloaded archive.`))
+        return process.exit()
+      }
+
+      console.log(
+        chalk.blue(`Extracting archive to local file system... (archive size: ${bytesToString(archive.length)}`)
+      )
+
+      await extractArchive(archive, this.targetDir)
+
+      const fileCount = await this._filesCount(this.targetDir)
+      console.log(chalk.green(`Successfully extracted ${fileCount} files from the remote server!`))
+    } catch (err) {
+      const error = err.response ? `${err.response.statusText} (${err.response.status})` : err.message
+      this._endWithError(`Could not pull changes: ${error}`)
+    }
+  }
+
+  async pushChanges() {
+    const useForce = process.argv.includes('--force')
+    const dryRun = process.argv.includes('--dry')
+
+    if (!fse.existsSync(this.sourceDir)) {
+      this._endWithError(`Specified folder "${this.sourceDir}" doesn't exist.`)
+    }
+
+    try {
+      console.log(chalk.blue(`Preparing an archive of your local files from ${this.sourceDir}...`))
+
+      const archive = await createArchiveFromFolder(this.sourceDir, pushedArchiveIgnoredFiles)
+      const axiosClient = this._getPushAxiosClient(archive.length)
+
+      console.log(
+        chalk.blue(`Sending archive to server for comparison... (archive size: ${bytesToString(archive.length)})`)
+      )
+      const { data } = await axiosClient.post('changes', archive)
+      const { changeList, blockingChanges, localFiles } = this._processChanges(data)
+
+      if (_.isEmpty(blockingChanges) || useForce) {
+        this._printChangeList(changeList)
+
+        if (dryRun) {
+          console.log(chalk.yellow(`Dry run completed. Nothing was pushed to server`))
+          return process.exit()
+        }
+
+        console.log(chalk.blue(`Pushing local changes to ${this.serverUrl}... ${useForce ? '(using --force)' : ''}`))
+
+        await axiosClient.post('update', archive)
+        console.log(chalk.green(`Successfully pushed ${localFiles.length} local files to remote server!`))
+      } else {
+        this._printOutOfSync()
+        this._printChangeList(changeList)
+        console.log(chalk.red(`Nothing was pushed on the remote server.`))
+      }
+    } catch (err) {
+      const error = err.response ? `${err.response.statusText} (${err.response.status})` : err.message
+      this._endWithError(`Could not push changes: ${error}`)
+    }
+  }
+
+  private async _filesCount(directory: string): Promise<number> {
+    const files: string[] = await Promise.fromCallback(cb =>
+      glob('**/*', { cwd: directory, nodir: true, dot: true }, cb)
+    )
+    return files.length
+  }
+
+  private _processChanges(data: BpfsScopedChange[]): ProcessedChanges {
+    const changeList = _.flatten(data.map(x => x.changes))
+    return {
+      localFiles: _.flatten(data.map(x => x.localFiles)),
+      blockingChanges: changeList.filter(x => blockingActions.includes(x.action)),
+      changeList
+    }
+  }
+
+  private _getPushAxiosClient(archiveSize: number): AxiosInstance {
+    return axios.create({
+      baseURL: `${this.serverUrl}/api/v1/admin/versioning`,
+      headers: {
+        Authorization: `Bearer ${this.authToken}`,
+        'Content-Type': 'application/tar+gzip',
+        'Content-Disposition': `attachment; filename=archive_${Date.now()}.tgz`,
+        'Content-Length': archiveSize
+      }
+    })
+  }
+
+  private _getPullAxiosClient(): AxiosInstance {
+    return axios.create({
+      baseURL: `${this.serverUrl}/api/v1/admin/versioning`,
+      headers: {
+        Authorization: `Bearer ${this.authToken}`
+      },
+      responseType: 'arraybuffer'
+    })
+  }
+
+  private async _clearDir(destination: string): Promise<void> {
+    if (fse.existsSync(destination)) {
+      return Promise.fromCallback(cb => rimraf(destination, cb))
+    }
+  }
+
+  private _printLine({ action, path, add, del }): string {
+    if (action === 'add') {
+      return chalk.green(` + ${path}`)
+    } else if (action === 'del') {
+      return chalk.red(` - ${path}`)
+    } else if (action === 'edit') {
+      return ` o ${path} (${chalk.green('+' + add)} / -${chalk.redBright(del)})`
+    }
+    return ''
+  }
+
+  private _printOutOfSync() {
+    console.log(`
+${chalk.yellow('Conflict warning')}
+Remote has changes that are not synced to your environment.
+Backup your changes and use "pull" to get those changes on your file system.`)
+
+    console.log(`
+Use ${chalk.yellow('--force')} to overwrite remote changes with yours.
+`)
+  }
+
+  private _printChangeList(changes) {
+    if (!changes.length) {
+      return
+    }
+
+    const lines = _.orderBy(changes, 'action')
+      .map(this._printLine)
+      .join('\n')
+
+    return console.log(`
+Differences between ${chalk.green('local')} and ${chalk.red('remote')} changes
+
+${lines}
+`)
+  }
+
+  private _endWithError(message: string) {
+    console.log(chalk.red(`${chalk.bold('Error:')} ${message}`))
+    process.exit()
+  }
+}
+
+export default async (argv, action) => {
+  const bpfs = new BPFS(argv, action)
+  console.log(`\n`)
+  if (action === 'pull') {
+    await bpfs.pullChanges()
+  } else if (action === 'push') {
+    await bpfs.pushChanges()
+  }
+  console.log(`\n`)
+}

src/bp/core/misc/utils.ts

@@ -1,4 +1,6 @@
 import { Logger } from 'botpress/sdk'
+import globrex from 'globrex'
+import _ from 'lodash'
 
 export type MockObject<T> = { T: T } & { readonly [key in keyof T]: jest.Mock }
 
@@ -84,3 +86,18 @@ export const asBytes = (size: string) => {
 
   return Number(matches[1])
 }
+
+export const bytesToString = (bytes: number): string => {
+  const units = ['bytes', 'kb', 'mb', 'gb', 'tb']
+  const power = Math.log2(bytes)
+  const unitNumber = Math.min(Math.floor(power / 10), 4)
+  const significand = bytes / Math.pow(2, unitNumber * 10)
+
+  return `${significand.toFixed(0)} ${units[unitNumber]}`
+}
+
+export function filterByGlobs<T>(array: T[], iteratee: (value: T) => string, globs: string[]): T[] {
+  const rules: { regex: RegExp }[] = globs.map(g => globrex(g, { globstar: true }))
+
+  return array.filter(x => _.every(rules, rule => !rule.regex.test(iteratee(x))))
+}

src/bp/core/routers/admin/bots.ts

@@ -180,7 +180,7 @@ export class BotsRouter extends CustomRouter {
       this.needPermissions('write', this.resource),
       this.asyncMiddleware(async (req, res) => {
         if (!req.is('application/tar+gzip')) {
-          res.status(400).send('Bot should be imported from archive')
+          return res.status(400).send('Bot should be imported from archive')
         }
         const buffers: any[] = []
         req.on('data', chunk => {

src/bp/core/routers/admin/index.ts

@@ -7,7 +7,6 @@ import { GhostService } from 'core/services'
 import { AlertingService } from 'core/services/alerting-service'
 import AuthService, { TOKEN_AUDIENCE } from 'core/services/auth/auth-service'
 import { BotService } from 'core/services/bot-service'
-import { CMSService } from 'core/services/cms'
 import { MonitoringService } from 'core/services/monitoring'
 import { WorkspaceService } from 'core/services/workspace-service'
 import { RequestHandler, Router } from 'express'
@@ -45,15 +44,14 @@ export class AdminRouter extends CustomRouter {
     configProvider: ConfigProvider,
     monitoringService: MonitoringService,
     alertingService: AlertingService,
-    moduleLoader: ModuleLoader,
-    cmsService: CMSService
+    moduleLoader: ModuleLoader
   ) {
     super('Admin', logger, Router({ mergeParams: true }))
     this.checkTokenHeader = checkTokenHeader(this.authService, TOKEN_AUDIENCE)
     this.botsRouter = new BotsRouter(logger, this.workspaceService, this.botService, configProvider)
     this.usersRouter = new UsersRouter(logger, this.authService, this.workspaceService)
     this.licenseRouter = new LicenseRouter(logger, this.licenseService, configProvider)
-    this.versioningRouter = new VersioningRouter(logger, this.ghostService, this.botService, cmsService)
+    this.versioningRouter = new VersioningRouter(logger, this.ghostService, this.botService)
     this.rolesRouter = new RolesRouter(logger, this.workspaceService)
     this.serverRouter = new ServerRouter(logger, monitoringService, alertingService, configProvider, ghostService)
     this.languagesRouter = new LanguagesRouter(logger, moduleLoader, this.workspaceService)

src/bp/core/routers/admin/versioning.ts

@@ -2,7 +2,6 @@ import { Logger } from 'botpress/sdk'
 import { extractArchive } from 'core/misc/archive'
 import { GhostService } from 'core/services'
 import { BotService } from 'core/services/bot-service'
-import { CMSService } from 'core/services/cms'
 import { Router } from 'express'
 import _ from 'lodash'
 import mkdirp from 'mkdirp'
@@ -12,37 +11,23 @@ import tmp from 'tmp'
 import { CustomRouter } from '../customRouter'
 
 export class VersioningRouter extends CustomRouter {
-  constructor(
-    logger: Logger,
-    private ghost: GhostService,
-    private botService: BotService,
-    private cmsService: CMSService
-  ) {
+  constructor(private logger: Logger, private ghost: GhostService, private botService: BotService) {
     super('Versioning', logger, Router({ mergeParams: true }))
     this.setupRoutes()
   }
 
   setupRoutes() {
-    this.router.get(
-      '/pending',
-      this.asyncMiddleware(async (req, res) => {
-        const botIds = await this.botService.getBotsIds()
-        res.send(await this.ghost.getPending(botIds))
-      })
-    )
-
     this.router.get(
       '/export',
       this.asyncMiddleware(async (req, res) => {
-        const botIds = await this.botService.getBotsIds()
-        const tarball = await this.ghost.exportArchive(botIds)
+        const archive = await this.ghost.exportArchive()
 
         res.writeHead(200, {
           'Content-Type': 'application/tar+gzip',
           'Content-Disposition': `attachment; filename=archive_${Date.now()}.tgz`,
-          'Content-Length': tarball.length
+          'Content-Length': archive.length
         })
-        res.end(tarball)
+        res.end(archive)
       })
     )
 
@@ -54,9 +39,9 @@ export class VersioningRouter extends CustomRouter {
         try {
           await this.extractArchiveFromRequest(req, tmpDir.name)
 
-          res.send({ changes: await this.ghost.listFileChanges(tmpDir.name) })
+          res.send(await this.ghost.listFileChanges(tmpDir.name))
         } catch (error) {
-          res.status(500).send('Error while pushing changes')
+          res.status(500).send('Error while listing changes')
         } finally {
           tmpDir.removeCallback()
         }
@@ -74,6 +59,9 @@ export class VersioningRouter extends CustomRouter {
           await this.extractArchiveFromRequest(req, tmpDir.name)
           const newBotIds = await this.ghost.forceUpdate(tmpDir.name)
 
+          this.logger.info(`Unmounting bots: ${beforeBotIds.join(', ')}`)
+          this.logger.info(`Mounting bots: ${newBotIds.join(', ')}`)
+
           // Unmount all previous bots and re-mount only the remaining (and new) bots
           await Promise.map(beforeBotIds, id => this.botService.unmountBot(id))
           await Promise.map(newBotIds, id => this.botService.mountBot(id))