feat(logs): add logs janitor

packages/core/botpress/src/botpress.js

@@ -51,6 +51,7 @@ import createUsers from './users'
 import createContentManager from './content/service'
 import defaultGetItemProviders from './content/getItemProviders'
 import createHelpers from './helpers'
+import createJanitor from './janitor'
 import stats from './stats'
 
 import EventBus from './bus'
@@ -194,7 +195,9 @@ class botpress {
       botpressPath: this.botpressPath
     })
 
-    logger.enableDbStorageIfNeeded(db)
+    const janitor = createJanitor({ db, logger })
+
+    logger.enableDbStorageIfNeeded({ db, janitor })
     logger.info(`Starting botpress version ${version}`)
 
     const kvs = db._kvs
@@ -331,6 +334,7 @@ class botpress {
       licensing,
       modules,
       db,
+      janitor,
       kvs,
       configManager,
       cloud,

packages/core/botpress/src/janitor/index.js

@@ -0,0 +1,71 @@
+import Promise from 'bluebird'
+import nanoid from 'nanoid'
+import moment from 'moment'
+import { findIndex } from 'lodash'
+
+import helpers from '../database/helpers'
+
+const DEFAULTS = {
+  timestampColumn: 'created_on'
+}
+
+const createJanitor = ({ db, logger, intervalMin = 1 }) => {
+  const tasks = []
+  let currentPromise = null
+
+  // TODO: impplement `debuounce` param which, when set,
+  // prevents the specific task form running too often
+  // The goal is to have the interval reasonably low (1/5/10s)
+  // for some tasks like dialog sessions
+  // but don't run other tasks like logs more often than every 1/5/10min
+  const runTask = async ({ table, ttl, timestampColumn }) => {
+    logger.info(`[DB Janitor] Running for table "${table}"`)
+    const knex = await db.get()
+    const outdatedCondition = helpers(knex).date.isBefore(timestampColumn, moment().subtract(ttl, 'seconds'))
+    return knex(table)
+      .where(outdatedCondition)
+      .del()
+      .then()
+  }
+
+  const runTasks = () => {
+    logger.debug('[DB Janitor] Running tasks')
+    if (currentPromise) {
+      // don't run the tasks if the previous batch didn't finish yet
+      logger.debug('[DB Janitor] Skipping the current run, previous operation still running')
+      return
+    }
+    currentPromise = Promise.each(tasks, runTask)
+      .catch(err => {
+        logger.error('[DB Janitor] Error:', err.message)
+      })
+      .finally(() => {
+        currentPromise = null
+      })
+  }
+
+  const intervalId = setInterval(runTasks, intervalMin * 60 * 1000)
+  logger.info('[DB Janitor] Started')
+
+  const add = opts => {
+    logger.info(`[DB Janitor] Added table "${opts.table}"`)
+    const id = nanoid()
+    tasks.push(Object.assign({ id }, DEFAULTS, opts))
+    return id
+  }
+
+  const remove = id => {
+    const i = findIndex(tasks, { id })
+    const [opts] = tasks.splice(i, 1)
+    logger.info(`[DB Janitor] Removed table "${opts.table}"`)
+  }
+
+  const stop = () => {
+    clearInterval(intervalId)
+    logger.info('[DB Janitor] Stopped')
+  }
+
+  return { add, remove, stop }
+}
+
+export default createJanitor

packages/core/botpress/src/logger/db-transport.js

@@ -8,15 +8,19 @@ export default class DbTransport extends winston.Transport {
   constructor(opts) {
     super(opts)
     this.name = 'DBLogger'
-    this.opts = opts
+    this.db = opts.db
+    opts.janitor.add({
+      table: 'logs',
+      ttl: opts.ttl
+    })
   }
 
   log(level, message, meta, callback) {
     if (!isEmpty(meta)) {
       message += ' ' + JSON.stringify(meta)
     }
 
-    this.opts.db
+    this.db
       .get()
       .then(knex =>
         knex('logs').insert({

packages/core/botpress/src/logger/index.js

@@ -34,13 +34,11 @@ module.exports = logConfig => {
     ]
   })
 
-  logger.enableDbStorageIfNeeded = db => {
+  logger.enableDbStorageIfNeeded = ({ db, janitor }) => {
     if (logConfig.enabled) {
       _db = db
-      logger.add(DbTransport, {
-        ttl: logConfig.keepDays * 24 * 3600,
-        db
-      })
+      const ttl = (logConfig.keepDays || 30) * 24 * 3600
+      logger.add(DbTransport, { ttl, db, janitor })
     }
   }
 

packages/core/botpress/src/web/views/Logs/index.jsx

@@ -50,7 +50,7 @@ class LoggerView extends Component {
     const message = line.message.replace(/\[\d\d?m/gi, '')
 
     return (
-      <li key={`log_event_${index}`} className={styles.line}>
+      <li key={`${index}.${message}`} className={styles.line}>
         <span className={styles.time}>{time}</span>
         <span className={styles['level-' + line.level]}>{line.level + ': '}</span>
         <span className={styles.message}>{message}</span>