Merge branch 'master' into imc_profile_api

api/v1/controllers/botData.js

@@ -12,6 +12,7 @@
 'use strict';
 
 const helper = require('../helpers/nouns/botData');
+const doPost = require('../helpers/verbs/doPost');
 const doFind = require('../helpers/verbs/doFind');
 const doGet = require('../helpers/verbs/doGet');
 const doPatch = require('../helpers/verbs/doPatch');
@@ -58,6 +59,19 @@ module.exports = {
     doFind(req, res, next, helper);
   },
 
+  /**
+   * POST /botData
+   *
+   * Creates a new botData and sends it back in the response.
+   *
+   * @param {IncomingMessage} req - The request object
+   * @param {ServerResponse} res - The response object
+   * @param {Function} next - The next middleware function in the stack
+   */
+  postBotData(req, res, next) {
+    doPost(req, res, next, helper);
+  },
+
   /**
    * GET /botData/{key}
    *

api/v1/controllers/generators.js

@@ -21,7 +21,6 @@ const doFind = require('../helpers/verbs/doFind');
 const doGet = require('../helpers/verbs/doGet');
 const doPatch = require('../helpers/verbs/doPatch');
 const doPost = require('../helpers/verbs/doPost');
-const doDelete = require('../helpers/verbs/doDelete');
 const doPut = require('../helpers/verbs/doPut');
 const u = require('../helpers/verbs/utils');
 const httpStatus = require('../constants').httpStatus;

api/v1/helpers/nouns/botData.js

@@ -16,6 +16,7 @@ const m = 'BotData';
 
 module.exports = {
   apiLinks: {
+    POST: `Create a new ${m}`,
     GET: `Retrieve ${m}`,
     PATCH: `Update selected attributes of ${m}`,
     DELETE: `Delete ${m}`,

api/v1/helpers/verbs/doPut.js

@@ -70,9 +70,10 @@ function doPut(req, res, next, props) {
             // take nullified fields out of changed fields
             o.changed(key, false);
           } else {
-
-            // value may have changed. set changed to true to
-            // trigger checks in the model
+            /**
+             * value may have changed. set changed to true to
+             * trigger checks in the model
+             */
             o.changed(key, true);
             o.set(key, toPut[key]);
           }

api/v1/swagger.yaml

@@ -7651,6 +7651,7 @@ paths:
           $ref: "#/responses/403"
         default:
           $ref: "#/responses/genericError"
+
 # =============================================================================
   /botData:
     x-swagger-router-controller: botData
@@ -7680,6 +7681,63 @@ paths:
           $ref: "#/responses/400"
         401:
           $ref: "#/responses/401"
+        403:
+          $ref: "#/responses/403"
+        404:
+          $ref: "#/responses/404"
+        default:
+          $ref: "#/responses/genericError"
+
+    post:
+      security:
+        - jwt: []
+      summary: Create a new botData
+      tags: [ botData ]
+      description: Create new bot data for a rooms specific bots
+      operationId: postBotData
+      parameters:
+        -
+          name: queryBody
+          in: body
+          required: true
+          schema:
+            type: object
+            description: Create botData properties
+            properties:
+              name:
+                type: string
+                maxLength: 254
+                description: Name of the bot data
+                pattern: ^[0-9A-Za-z_\\-]{0,60}$
+              value:
+                type: string
+                maxLength: 254
+                description: Value of the bot data
+              botId:
+                type: string
+                maxLength: 254
+                description: UUID for the corresponding bot
+              roomId:
+                type: integer
+                maxLength: 254
+                description: Number for the corresponding room
+            required:
+              - name
+              - value
+              - roomId
+              - botId
+      responses:
+        201:
+          description: >-
+            Created botData
+          schema:
+            $ref: "#/definitions/BotDataResponse"
+        400:
+          $ref: "#/responses/400"
+        401:
+          $ref: "#/responses/401"
+        403:
+          $ref: "#/responses/403"
         404:
           $ref: "#/responses/404"
         default:
@@ -7802,6 +7860,8 @@ paths:
           $ref: "#/responses/400"
         401:
           $ref: "#/responses/401"
+        403:
+          $ref: "#/responses/403"
         404:
           $ref: "#/responses/404"
         default:

clock/index.js

@@ -56,4 +56,7 @@ if (featureToggles.isFeatureEnabled('enableQueueStatsActivityLogs')) {
 }
 
 // Clean up completed jobs
-setInterval(jobCleanup.execute, conf.JOB_REMOVAL_INTERVAL_SECONDS * 1000);
+setInterval(jobCleanup.enqueue, conf.JOB_REMOVAL_INTERVAL);
+
+// Reset the job id counter
+setInterval(jobCleanup.resetCounter, conf.JOB_COUNTER_RESET_INTERVAL);

clock/scheduledJobs/jobCleanup.js

@@ -14,42 +14,110 @@
 const featureToggles = require('feature-toggles');
 const Promise = require('bluebird');
 const kue = require('kue');
-const jobSetup = require('../../jobQueue/setup');
-const jobQueue = jobSetup.jobQueue;
+const jobType = require('../../jobQueue/setup').jobType;
+const jobWrapper = require('../../jobQueue/jobWrapper');
+const conf = require('../../config');
 const rangeByStateAsync = Promise.promisify(kue.Job.rangeByState);
 kue.Job.prototype.removeAsync = Promise.promisify(kue.Job.prototype.remove);
 
 /**
  * Execute the call to clean up jobs
  *
+ * @param {Number} batchSize - the number of jobs to delete in each batch
+ * @param {Number} delay - the delay, in ms, before completed jobs should be deleted
  * @returns {Promise}
  */
-function execute() {
+function execute(batchSize, delay) {
+  const now = Date.now();
+  let removedJobCount = 0;
+  let skippedJobCount = 0;
+
   if (featureToggles.isFeatureEnabled('instrumentKue')) {
     console.log('[KJI] Ready to remove completed jobs');
   }
 
-  return rangeByStateAsync('complete', 0, -1, 'asc') // get all completed jobs
-  .then((jobs) =>
-    Promise.all(jobs.map((job) =>
-      job.removeAsync()
-      .then(() => {
-        if (featureToggles.isFeatureEnabled('instrumentKue')) {
-          console.log(`[KJI] Removed completed job ${job.id}`);
-        }
-      })
-      .catch((err) => {
-        console.log(`Error removing job ${job.id}`, err);
-        return Promise.reject(err);
-      })
-    ))
-  )
-  .catch((err) => {
-    console.log('Error getting completed jobs from queue', err);
-    return Promise.reject(err);
+  return deleteNextNJobs(batchSize)
+  .then(() => {
+    if (featureToggles.isFeatureEnabled('instrumentKue')) {
+      console.log(`[KJI] Removed ${removedJobCount} jobs`);
+    }
   });
+
+  function deleteNextNJobs(n) {
+    if (n === 0) return Promise.resolve();
+    const from = skippedJobCount;
+    const to = skippedJobCount + n - 1;
+
+    // get n jobs
+    return rangeByStateAsync('complete', from, to, 'asc')
+    .catch((err) => {
+      console.log('Error getting completed jobs from queue', err);
+      return Promise.reject(err);
+    })
+
+    // delete n jobs
+    .then((jobs) =>
+      Promise.all(jobs.map((job) => {
+        if (now - job.updated_at < delay) {
+          return 'skipped';
+        } else {
+          return job.removeAsync()
+          .catch((err) => {
+            console.log(`Error removing job ${job.id}`, err);
+            return Promise.reject(err);
+          });
+        }
+      }))
+    )
+
+    // count skipped jobs
+    .then((results) => {
+      const skippedCount = results.reduce((count, result) =>
+        result === 'skipped' ? count + 1 : count,
+      0);
+      skippedJobCount += skippedCount;
+      removedJobCount += results.length - skippedCount;
+      return Promise.resolve(results);
+    })
+
+    // get and delete next n jobs
+    .then((results) => {
+      if (results.length < n) {
+        return Promise.resolve();
+      } else {
+        return deleteNextNJobs(n);
+      }
+    });
+  }
+
 } // execute
 
+/**
+ * Send the job to the worker or execute directly
+ */
+function enqueue() {
+  if (featureToggles.isFeatureEnabled('enableWorkerProcess')) {
+    const job = jobWrapper.createJob(
+      jobType.JOB_CLEANUP, { reqStartTime: Date.now() }
+    );
+    return Promise.resolve(job);
+  } else {
+    // If not using worker process, execute directly;
+    return execute(conf.JOB_REMOVAL_BATCH_SIZE, conf.JOB_REMOVAL_DELAY);
+  }
+} // enqueue
+
+/**
+ * Reset the job counter so job ids will be assigned starting from zero again
+ */
+function resetCounter() {
+  const client = kue.Job.client;
+  const key = client.getKey('ids');
+  client.del(key);
+} // resetCounter
+
 module.exports = {
+  enqueue,
+  resetCounter,
   execute,
 };

config.js

@@ -71,11 +71,24 @@ const readReplicas = configUtil.getReadReplicas(pe, replicaConfigLabel);
 const DEFAULT_JOB_QUEUE_TTL_SECONDS_ASYNC = 3600;
 const DEFAULT_JOB_QUEUE_TTL_SECONDS_SYNC = 25;
 
-// default set to 30 minutes
-const DEFAULT_JOB_REMOVAL_INTERVAL_SECONDS = 1800;
+const DEFAULT_JOB_REMOVAL_INTERVAL_MINUTES = 30;
+const JOB_REMOVAL_INTERVAL = 60 * 1000 * (pe.KUE_JOBS_REMOVAL_INTERVAL_MINUTES ||
+  DEFAULT_JOB_REMOVAL_INTERVAL_MINUTES);
 
-const JOB_REMOVAL_INTERVAL_SECONDS = pe.KUE_JOBS_REMOVAL_INTERVAL ||
-  DEFAULT_JOB_REMOVAL_INTERVAL_SECONDS;
+const DEFAULT_JOB_REMOVAL_DELAY_MINUTES = 30;
+const JOB_REMOVAL_DELAY = 60 * 1000 * (pe.KUE_JOBS_REMOVAL_DELAY_MINUTES ||
+  DEFAULT_JOB_REMOVAL_DELAY_MINUTES);
+
+const DEFAULT_JOB_REMOVAL_BATCH_SIZE = 1000;
+const JOB_REMOVAL_BATCH_SIZE = pe.KUE_JOBS_REMOVAL_BATCH_SIZE ||
+  DEFAULT_JOB_REMOVAL_BATCH_SIZE;
+
+// This must be set to several times the value of JOB_REMOVAL_INTERVAL or
+// completed jobs may be overwritten sooner that JOB_REMOVAL_DELAY
+const DEFAULT_JOB_COUNTER_RESET_INTERVAL_MINUTES = 24 * 60;
+const JOB_COUNTER_RESET_INTERVAL = 60 * 1000 *
+  (pe.KUE_JOB_COUNTER_RESET_INTERVAL_MINUTES ||
+  DEFAULT_JOB_COUNTER_RESET_INTERVAL_MINUTES);
 
 /*
  * If you're using worker dynos, you can set env vars PRIORITIZE_JOBS_FROM
@@ -243,7 +256,10 @@ module.exports = {
   CACHE_EXPIRY_IN_SECS,
   JOB_QUEUE_TTL_SECONDS_ASYNC,
   JOB_QUEUE_TTL_SECONDS_SYNC,
-  JOB_REMOVAL_INTERVAL_SECONDS,
+  JOB_REMOVAL_INTERVAL,
+  JOB_REMOVAL_DELAY,
+  JOB_REMOVAL_BATCH_SIZE,
+  JOB_COUNTER_RESET_INTERVAL,
   deprioritizeJobsFrom,
   endpointToLimit,
   httpMethodToLimit,

db/model/generator.js

@@ -34,7 +34,7 @@ const generatorTemplateSchema = {
   },
 };
 
-module.exports = function user(seq, dataTypes) {
+module.exports = function generator(seq, dataTypes) {
   const Generator = seq.define('Generator', {
     description: {
       type: dataTypes.TEXT,
@@ -103,13 +103,14 @@ module.exports = function user(seq, dataTypes) {
     },
   }, {
     classMethods: {
-      getGeneratortemplateAssociations() {
+      getGeneratorAssociations() {
         return assoc;
       },
 
       postImport(models) {
-        assoc.createdBy = Generator.belongsTo(models.User, {
+        assoc.user = Generator.belongsTo(models.User, {
           foreignKey: 'createdBy',
+          as: 'user',
         });
 
         assoc.collectors = Generator.belongsToMany(models.Collector, {
@@ -123,6 +124,18 @@ module.exports = function user(seq, dataTypes) {
           through: 'GeneratorWriters',
           foreignKey: 'generatorId',
         });
+
+        Generator.addScope('defaultScope', {
+          include: [
+            {
+              association: assoc.user,
+              attributes: ['name', 'email'],
+            },
+          ],
+          order: ['name'],
+        }, {
+          override: true,
+        });
       },
     },
 

db/model/generatortemplate.js

@@ -181,15 +181,28 @@ module.exports = function user(seq, dataTypes) {
       },
 
       postImport(models) {
-        assoc.createdBy = GeneratorTemplate.belongsTo(models.User, {
+        assoc.user = GeneratorTemplate.belongsTo(models.User, {
           foreignKey: 'createdBy',
+          as: 'user',
         });
 
         assoc.writers = GeneratorTemplate.belongsToMany(models.User, {
           as: 'writers',
           through: 'GeneratorTemplateWriters',
           foreignKey: 'generatorTemplateId',
         });
+
+        GeneratorTemplate.addScope('defaultScope', {
+          include: [
+            {
+              association: assoc.user,
+              attributes: ['name', 'email'],
+            },
+          ],
+          order: ['name'],
+        }, {
+          override: true,
+        });
       },
     },