Transform testing part 1 (#82)

* transform testing part 1 - error Handlers * transform testing part 1 - eval args * transform testing part 1 - misc * transform testing part 1 - code style * transform testing part 1 - review changes * transform testing part 1 - change url to preparedUrl
salesforce · Oct 18, 2017 · ce41a49 · ce41a49
1 parent e662ea6
commit ce41a49
Show file tree

Hide file tree

Showing 15 changed files with 871 additions and 76 deletions.
diff --git a/src/heartbeat/utils.js b/src/heartbeat/utils.js
@@ -89,7 +89,7 @@ function assignContext(ctx, def, refocusInstance, res) {
  * @param {Object} generator - Generator object from the heartbeat
  */
 function setupRepeater(generator) {
-  if (generator.generatorTemplate.connection.bulk === true) {
+  if (commonUtils.isBulk(generator)) {
     repeater.createGeneratorRepeater(generator);
   } else {
     // bulk is false

diff --git a/src/remoteCollection/collect.js b/src/remoteCollection/collect.js
@@ -86,7 +86,8 @@ function prepareHeaders(headers, ctx) {
  */
 function sendRemoteRequest(generator, connection, simpleOauth=null) {
   return new Promise((resolve) => {
-    const remoteUrl = prepareUrl(generator);
+    // Add the url to the generator so the handler has access to it later.
+    generator.preparedUrl = prepareUrl(generator);
 
     // If token is present then add token to request header.
     if (generator.token) {
@@ -103,7 +104,7 @@ function sendRemoteRequest(generator, connection, simpleOauth=null) {
 
     // Remote request for fetching data.
     request
-    .get(remoteUrl)
+    .get(generator.preparedUrl)
     .set(headers)
     .end((err, res) => {
       if (err) {

diff --git a/src/remoteCollection/errorSamples.js b/src/remoteCollection/errorSamples.js
@@ -0,0 +1,47 @@
+/**
+ * Copyright (c) 2017, salesforce.com, inc.
+ * All rights reserved.
+ * Licensed under the BSD 3-Clause license.
+ * For full license text, see LICENSE.txt file in the repo root or
+ * https://opensource.org/licenses/BSD-3-Clause
+ */
+
+/**
+ * src/remoteCollection/errorSamples.js
+ */
+const debug = require('debug')
+('refocus-collector:remoteCollection:errorSamples');
+
+module.exports = (collectResponse, messageBody) => {
+  debug('errorSamples', collectResponse.name, messageBody);
+  const samples = [];
+  collectResponse.aspects.forEach((a) => {
+    collectResponse.subjects.forEach((s) => {
+      const relatedLinks = [];
+
+      if (collectResponse.apiLinks) {
+        relatedLinks.push({
+          name: 'Sample Generator',
+          url: collectResponse.apiLinks.GET,
+        });
+      }
+
+      const repository = collectResponse.generatorTemplate.repository;
+      if (repository && repository.url) {
+        relatedLinks.push({
+          name: 'Sample Generator Template',
+          url: repository.url,
+        });
+      }
+
+      samples.push({
+        name: `${s.absolutePath}|${a.name}`,
+        messageCode: 'ERROR',
+        messageBody,
+        value: 'ERROR',
+        relatedLinks,
+      });
+    });
+  });
+  return samples;
+};
diff --git a/src/remoteCollection/handleCollectResponse.js b/src/remoteCollection/handleCollectResponse.js
@@ -12,8 +12,10 @@
 const debug = require('debug')('refocus-collector:handleCollectResponse');
 const evalUtils = require('../utils/evalUtils');
 const errors = require('../errors');
+const errorSamples = require('./errorSamples');
 const logger = require('winston');
 const enqueue = require('../sampleQueue/sampleQueueOps').enqueue;
+const httpStatus = require('../constants').httpStatus;
 
 /**
  * Validates the response from the collect function. Confirms that it is an
@@ -32,15 +34,33 @@ function validateCollectResponse(cr) {
       'null, and must not be an Array.');
   }
 
-  if (!cr.res) {
+  if (!cr.name) {
     throw new errors.ValidationError('The argument passed to the ' +
-      '"handleCollectResponse" function must have a "res" attribute.');
+      '"handleCollectResponse" function must have a "name" attribute.');
   }
 
-  if (!cr.name) {
+  if (!cr.preparedUrl) {
     throw new errors.ValidationError('The argument passed to the ' +
-      '"handleCollectResponse" function must have a "name" attribute.');
+      '"handleCollectResponse" function must have a "preparedUrl" attribute.');
   }
+
+  // No response.
+  if (!cr.res) {
+    throw new errors.ValidationError(`No response from ${cr.preparedUrl}`);
+  }
+
+  // Invalid response: missing status code.
+  if (!cr.res.hasOwnProperty('statusCode')) {
+    throw new errors.ValidationError(`Invalid response from ${cr.preparedUrl}: `
+    + 'missing HTTP status code');
+  }
+
+  // Expecting response status code to be 3 digits.
+  if (!/\d\d\d/.test(cr.res.statusCode)) {
+    throw new errors.ValidationError(`Invalid response from ${cr.preparedUrl}: `
+    + `invalid HTTP status code "${cr.res.statusCode}"`);
+  }
+
 } // validateCollectResponse
 
 /**
@@ -60,23 +80,76 @@ function validateCollectResponse(cr) {
 function handleCollectResponse(collectResponse) {
   debug('Entered handleCollectResponse');
   return collectResponse.then((collectRes) => {
-    try {
-      validateCollectResponse(collectRes);
-      const tr = collectRes.generatorTemplate.transform;
-      const t = Array.isArray(tr) ? tr.join('\n') : tr;
-      const transformedSamples = evalUtils.safeTransform(t, collectRes);
+    validateCollectResponse(collectRes);
+
+    /*
+     * If the transform is a string, then we use that function for all
+     * status codes.
+     */
+    const tr = collectRes.generatorTemplate.transform;
+    const args = evalUtils.prepareTransformArgs(collectRes);
+    if (typeof tr === 'string') { // match all status codes
+      const samplesToEnqueue = evalUtils.safeTransform(tr, args);
       logger.info(`{
         generator: ${collectRes.name},
-        numSamples: ${transformedSamples.length},
+        url: ${collectRes.preparedUrl},
+        numSamples: ${samplesToEnqueue.length},
       }`);
-      enqueue(transformedSamples);
-    } catch (err) {
-      debug(err);
-      logger.error('handleCollectResponse threw an error: ', err.name,
-        err.message);
-      return Promise.reject(err);
+      return enqueue(samplesToEnqueue);
+    } else {
+      /*
+       * The transform is *not* a string, so handle the response based on the
+       * status code.
+       */
+      const status = collectRes.res.statusCode;
+      let func;
+
+      // the response was OK, so use the default transform
+      if (status === httpStatus.OK) {
+        func = collectRes.generatorTemplate.transform.transform;
+      }
+
+      /*
+       * Check for a status code regex match which maps to a transform for
+       * error samples. Use the first one to match. If 200 is matched, it
+       * will override the default transform.
+       */
+      if (tr.errorHandlers) {
+        Object.keys(tr.errorHandlers).forEach((statusMatcher) => {
+          const re = new RegExp(statusMatcher);
+          if (re.test(status)) {
+            func = tr.errorHandlers[statusMatcher];
+          }
+        });
+      }
+
+      if (func) {
+        const samplesToEnqueue = evalUtils.safeTransform(func, args);
+        logger.info(`{
+          generator: ${collectRes.name},
+          url: ${collectRes.preparedUrl},
+          numSamples: ${samplesToEnqueue.length},
+        }`);
+        return enqueue(samplesToEnqueue);
+      } else {
+        /*
+         * If there is no transform designated for this HTTP status code, just
+         * generate default error samples.
+         */
+        const errorMessage = `${collectRes.preparedUrl} returned HTTP status ` +
+          `${collectRes.res.statusCode}: ${collectRes.res.statusMessage}`;
+        const samplesToEnqueue = errorSamples(collectRes, errorMessage);
+        logger.info(`{
+          generator: ${collectRes.name},
+          url: ${collectRes.preparedUrl},
+          error: ${errorMessage},
+          numSamples: ${samplesToEnqueue.length},
+        }`);
+        return enqueue(samplesToEnqueue);
+      }
     }
-  }).catch((err) => {
+  })
+  .catch((err) => {
     debug(err);
     logger.error('handleCollectResponse threw an error: ', err.name,
       err.message);

diff --git a/src/repeater/repeater.js b/src/repeater/repeater.js
@@ -17,6 +17,7 @@ const handleCollectResponse =
   require('../remoteCollection/handleCollectResponse').handleCollectResponse;
 const collect = require('../remoteCollection/collect').collect;
 const repeaterSchema = require('../utils/schema').repeater;
+const u = require('../utils/commonUtils');
 
 /**
  * Tracks all the repeaters defined in the collectors.
@@ -186,7 +187,7 @@ function createGeneratorRepeater(generator) {
     interval: generator.interval,
     func: () => collect(generator),
     onProgress: handleCollectResponse,
-    bulk: generator.generatorTemplate.connection.bulk,
+    bulk: u.isBulk(generator),
     subjects: generator.subjects,
   });
 } // createGeneratorRepeater

diff --git a/src/utils/commonUtils.js b/src/utils/commonUtils.js
@@ -171,4 +171,16 @@ module.exports = {
     return crypted;
   }, // encrypt
 
+  /**
+   * Determine if the given Generator is set up for bulk collection
+   * @param  {Object} generator - A Generator object
+   * @returns {Boolean}
+   */
+  isBulk(generator) {
+    const gt = generator.generatorTemplate;
+    const connection = gt && gt.connection;
+    const bulk = connection && connection.bulk;
+    return Boolean(bulk);
+  }, // encrypt
+
 };
diff --git a/src/utils/evalUtils.js b/src/utils/evalUtils.js
@@ -17,6 +17,7 @@ const logger = require('winston');
 const errors = require('../errors');
 const evalValidation = require('./evalValidation');
 const sampleSchema = require('./schema').sample;
+const u = require('./commonUtils');
 const ERROR_MESSAGE = {
   TRANSFORM: {
     NOT_ARRAY: 'The transform function must return an array.',
@@ -78,7 +79,7 @@ function validateTransformArgs(args) {
   return evalValidation.isObject('ctx', args.ctx) &&
     evalValidation.isObject('res', args.res) &&
     evalValidation.aspects(args.aspects) &&
-    evalValidation.subjects(args.subjects);
+    evalValidation.validateSubjectArgs(args);
 } // validateTransformArgs
 
 /**
@@ -121,26 +122,21 @@ function validateSamples(sampleArr, generator) {
     throw new errors.TransformError(ERROR_MESSAGE.TRANSFORM.NOT_ARRAY);
   }
 
-  const subjectArr = []; // array of subject absolute paths
-  const aspectArr = []; // array of aspect names
+  let subjectArr; // array of subject absolute paths
+  let aspectArr; // array of aspect names
 
-  // create subject array
   if (generator.subjects) {
-    generator.subjects.forEach((s) => {
-      subjectArr.push(s.absolutePath.toLowerCase());
-    });
+    subjectArr = generator.subjects.map(s => s.absolutePath.toLowerCase());
   } else if (generator.subject) {
-    subjectArr.push(generator.subject.absolutePath.toLowerCase());
+    subjectArr = [generator.subject.absolutePath.toLowerCase()];
   } else {
     throw new errors.ValidationError(
-      'Generator should have subject/subjects attribute.'
+      'Generator passed to validateSamples should have "subjects" or "subject"'
     );
   }
 
   // create aspect array, generator will have aspects attribute always
-  generator.aspects.forEach((a) => {
-    aspectArr.push(a.name.toLowerCase());
-  });
+  aspectArr = generator.aspects.map(a => a.name.toLowerCase());
 
   // no of samples should not exceed the (no. of subjects * the no. of aspects)
   if (sampleArr.length > subjectArr.length * aspectArr.length) {
@@ -220,6 +216,29 @@ function safeTransform(functionBody, args) {
   return retval;
 }
 
+/**
+ * Prepare arguments to be passed to the transform function
+ * @param  {Object} generator - Generator object
+ * @throws {TransformError} - if transform function does not return an array
+ *  of zero or more samples
+ * @throws {ValidationError} - if any of the above mentioned check fails
+ */
+function prepareTransformArgs(generator) {
+  const args = {};
+
+  args.ctx = generator.ctx;
+  args.res = generator.res;
+  args.aspects = generator.aspects;
+
+  if (u.isBulk(generator)) {
+    args.subjects = generator.subjects;
+  } else {
+    args.subject = generator.subjects[0];
+  }
+
+  return args;
+}
+
 /**
  * Safely executes the toUrl function with the arguments provided.
  *
@@ -257,6 +276,7 @@ module.exports = {
   safeEval, // exporting for testability
   safeToUrl,
   safeTransform,
+  prepareTransformArgs,
   validateTransformArgs, // exporting for testability
   validateToUrlArgs, // exporting for testability
   validateSamples, // exporting for testability

diff --git a/src/utils/evalValidation.js b/src/utils/evalValidation.js
@@ -11,6 +11,7 @@
  */
 const debug = require('debug')('refocus-collector:evalUtils');
 const errors = require('../errors');
+const u = require('./commonUtils');
 
 function isObject(name, val) {
   debug('Entered evalValidation.isObject:', name, val);
@@ -28,6 +29,18 @@ function isObject(name, val) {
 module.exports = {
   isObject,
 
+  validateSubjectArgs(args) {
+    if (!args.subjects && !args.subject) {
+      throw new errors.ArgsError('Must include either "subjects" or "subject".');
+    } else if (args.subjects && !args.subject) {
+      return this.subjects(args.subjects);
+    } else if (!args.subjects && args.subject) {
+      return this.subject(args.subject);
+    } else if (args.subjects && args.subject) {
+      throw new errors.ArgsError('Must not include both "subjects" and "subject".');
+    }
+  },
+
   aspects: (aspects) => {
     debug('Entered evalValidation.aspects:', aspects);
     if (!aspects) {
@@ -75,4 +88,18 @@ module.exports = {
 
     return true;
   }, // subjects
+
+  subject: (subject) => {
+    debug('Entered evalValidation.subject:', subject);
+    if (!subject) {
+      throw new errors.ArgsError('Must include a "subject" attribute.');
+    }
+
+    isObject(`subject`, subject);
+    if (typeof subject.absolutePath !== 'string') {
+      throw new errors.ArgsError('"subject" attribute must be a valid subject.');
+    }
+
+    return true;
+  }, // subject
 };
diff --git a/src/utils/schema.js b/src/utils/schema.js
@@ -25,7 +25,7 @@ const repeater = Joi.object().keys({
 });
 
 const sample = Joi.object().keys({
-  name: Joi.string().regex(/^[0-9A-Za-z_\\-]{1,4096}\|[0-9A-Za-z_\\-]{1,60}$/)
+  name: Joi.string().regex(/^[0-9A-Za-z_.\\-]{1,4096}\|[0-9A-Za-z_\\-]{1,60}$/)
     .required(),
   messageBody: Joi.string().max(4096),
   messageCode: Joi.string().max(5),