Retrieve behavior by name and retrieve all behaviors (#18)

* added ability to lookup behaviors by name fixes #16
added ability to retrieve all behaviors via the API server

improved autoscroll behavior

introduced behavior running configuration
added behaviorState util functions to the stand library
added behavior logging fixes #13

do not set _terminationPromise in behaviorLookup to null

updated cli info

* fixed a memory leak in internal/runner.js caused by not removing the stop listeners after the autorun function finishes

* switched twitter timeline behavior to new traversal system which should, unless updated, allow new twitter to work
cleaned up the twitter hashtag behavior
updated the soundcloud behavior to work again after site internal updates
improved runner's auto run implementation in order to handle rapid rebuilds

* left dev stuff in behaviors

.prettierrc

@@ -1,3 +1,4 @@
 {
-  "singleQuote": true
+  "singleQuote": true,
+  "trailingComma": "es5"
 }

.travis.yml

@@ -1,3 +1,3 @@
 language: node_js
-node_js: 11.13.0
+node_js: 12.2.0
 dist: xenial

Dockerfile

@@ -1,4 +1,4 @@
-ARG NODE=node:12.0.0
+ARG NODE=node:12.2.0
 FROM $NODE
 
 EXPOSE 3030

api/behaviorLookup.js

@@ -5,6 +5,7 @@ const fp = require('fastify-plugin');
 const uuid = require('uuid/v4');
 const EventEmitter = require('eventemitter3');
 const msgTypes = require('./msgTypes');
+const { autobind } = require('./utils');
 
 /**
  * @typedef {Object} LookupWorkerConfig
@@ -25,7 +26,7 @@ function noop() {}
 const WorkerEvents = {
   error: Symbol('worker-error'),
   exit: Symbol('worker-exit'),
-  message: Symbol('worker-message')
+  message: Symbol('worker-message'),
 };
 
 /**
@@ -101,9 +102,7 @@ class LookupWorker extends EventEmitter {
      * @private
      */
     this._terminationPromise = null;
-    this._onWorkerError = this._onWorkerError.bind(this);
-    this._onWorkerExit = this._onWorkerExit.bind(this);
-    this._onWorkerMsg = this._onWorkerMsg.bind(this);
+    autobind(this);
   }
 
   /**
@@ -114,11 +113,11 @@ class LookupWorker extends EventEmitter {
     this._worker = new Worker(this.workerPath, {
       workerData: {
         workerId: this.id,
-        behaviorInfo: this.behaviorInfo
-      }
+        behaviorInfo: this.behaviorInfo,
+      },
     });
     this._worker.postMessage({ serverCom: this._channel.port1 }, [
-      this._channel.port1
+      this._channel.port1,
     ]);
     this._channel.port2.on('message', this._onWorkerMsg);
     this._worker.on('error', this._onWorkerError);
@@ -160,6 +159,9 @@ class LookupWorker extends EventEmitter {
    * @private
    */
   _onWorkerExit(code) {
+    if (this._terminationPromise) {
+      this._terminationPromise.resolve();
+    }
     this.emit(WorkerEvents.exit, { id: this.id, code });
   }
 
@@ -216,12 +218,7 @@ class BehaviorLookUp {
      */
     this._msgIdsToPromises = new Map();
 
-    this.lookupBehavior = this.lookupBehavior.bind(this);
-    this.lookupBehaviorInfo = this.lookupBehaviorInfo.bind(this);
-    this.reloadBehaviors = this.reloadBehaviors.bind(this);
-    this._onLookupWorkerExit = this._onLookupWorkerExit.bind(this);
-    this._onLookupWorkerError = this._onLookupWorkerError.bind(this);
-    this._onLookupWorkerMsg = this._onLookupWorkerMsg.bind(this);
+    autobind(this);
   }
 
   /**
@@ -234,7 +231,7 @@ class BehaviorLookUp {
       const worker = new LookupWorker({
         behaviorInfo,
         id: workerId,
-        workerPath
+        workerPath,
       });
       worker.init();
       worker.on(WorkerEvents.error, this._onLookupWorkerError);
@@ -262,22 +259,32 @@ class BehaviorLookUp {
    * Initiates a behavior lookup action returning, a promise containing the
    * results of this action that resolves once the results of this action have
    * been received
-   * @param {string} url - The URL to retrieve the behavior for
+   * @param {Object} query - The query for the behavior
    * @return {Promise<string>}
    */
-  lookupBehavior(url) {
-    return this._sendMsg(msgTypes.lookupBehavior, url);
+  lookupBehavior(query) {
+    return this._sendMsg(msgTypes.lookupBehavior, query);
   }
 
   /**
    * Initiates a behavior info lookup action, returning a promise containing the
    * results of this action that resolves once the results of this action have
    * been received
-   * @param {string} url - The URL to retrieve the behavior info for
+   * @param {Object} query - The query for the behavior
    * @return {Promise<Object>}
    */
-  lookupBehaviorInfo(url) {
-    return this._sendMsg(msgTypes.lookupBehaviorInfo, url);
+  lookupBehaviorInfo(query) {
+    return this._sendMsg(msgTypes.lookupBehaviorInfo, query);
+  }
+
+  /**
+   * Initiates a behavior info lookup action, returning a promise containing the
+   * results of this action that resolves once the results of this action have
+   * been received
+   * @return {Promise<Object>}
+   */
+  behaviorList() {
+    return this._sendMsg(msgTypes.behaviorList);
   }
 
   /**
@@ -290,7 +297,7 @@ class BehaviorLookUp {
     const reloadResponses = new Array(this._numWorkers);
     for (let workerId = 0; workerId < this._numWorkers; workerId++) {
       reloadResponses[workerId] = this._sendDirectMessage(workerId, {
-        type: msgTypes.reloadBehaviors
+        type: msgTypes.reloadBehaviors,
       });
     }
     return Promise.all(reloadResponses).then(arrayFirstElement);
@@ -312,24 +319,28 @@ class BehaviorLookUp {
         .get(workerId)
         .terminate();
     }
-    Promise.all(terminationPromises).then(() => {
-      this._workersById.clear();
-      doneCB();
-    });
+    Promise.all(terminationPromises)
+      .then(() => {
+        this._workersById.clear();
+        doneCB();
+      })
+      .catch(error => {
+        doneCB();
+      });
   }
 
   /**
    * Sends a message to a worker. The worker the message is sent to is
    * determined by {@link nextWorkerId}
    * @param {string} type - The type of the action to be performed
-   * @param {string} [url] - Optional URL to be sent to worker
+   * @param {Object} [query] - Optional query to be sent to worker
    * @return {Promise<Object|String>}
    * @private
    */
-  _sendMsg(type, url) {
+  _sendMsg(type, query) {
     const workerId = this.nextWorkerId();
     const msgId = uuid();
-    const msg = { id: msgId, type, url };
+    const msg = { id: msgId, type, query };
     const prr = promiseResolveReject();
     this._msgIdsToPromises.set(msgId, prr);
     this._workersById.get(workerId).sendMsg(msg);
@@ -362,6 +373,9 @@ class BehaviorLookUp {
     if (lookupProm) {
       if (!msg.results.wasError) {
         switch (msg.type) {
+          case msgTypes.behaviorListResults:
+            lookupProm.resolve(msg.results.list);
+            break;
           case msgTypes.behaviorLookupResults:
             lookupProm.resolve(msg.results.behavior);
             break;
@@ -394,24 +408,5 @@ class BehaviorLookUp {
     console.log(`Lookup worker ${id} exited with code ${code}`);
   }
 }
-/**
- *
- * @param {fastify.FastifyInstance} server
- * @param opts
- * @param pluginNext
- */
-function behaviorLookup(server, opts, pluginNext) {
-  const behaviorLookerUpper = new BehaviorLookUp(server.conf);
-  behaviorLookerUpper.init();
-  server
-    .decorate('behaviorLookUp', behaviorLookerUpper)
-    .decorate('lookupBehavior', behaviorLookerUpper.lookupBehavior)
-    .decorate('lookupBehaviorInfo', behaviorLookerUpper.lookupBehaviorInfo)
-    .decorate('reloadBehaviors', behaviorLookerUpper.reloadBehaviors)
-    .addHook('onClose', (server, done) => {
-      behaviorLookerUpper.shutdown(done);
-    });
-  pluginNext();
-}
 
-module.exports = fp(behaviorLookup);
+module.exports = BehaviorLookUp;

api/conf.js

@@ -155,6 +155,7 @@ function ensureNumWorkers(value) {
 
 program
   .version(pkg.version)
+  .usage('[options]')
   .option(
     '-p, --port [port]',
     'The port the api server is to bind to',
@@ -193,7 +194,7 @@ const enableLogging = Utils.envFlagToBool(process.env.BEHAVIOR_API_LOGGING);
 const qsOpts = {
   charset: 'iso-8859-1',
   interpretNumericEntities: true,
-  charsetSentinel: true
+  charsetSentinel: true,
 };
 
 /**
@@ -207,14 +208,14 @@ const config = {
   behaviorInfo: {
     behaviorDir: program.behaviorDir,
     mdataPath: program.behaviorMetadata,
-    build: program.buildBehaviors
+    build: program.buildBehaviors,
   },
   fastifyOpts: {
     trustProxy: true,
     maxParamLength: 5e12,
     logger: enableLogging || process.env.LOG != null,
-    querystringParser: url => qs.parse(url, qsOpts)
-  }
+    querystringParser: url => qs.parse(url, qsOpts),
+  },
 };
 
 module.exports = config;

api/lookupWorker.js

@@ -7,7 +7,7 @@ const msgTypes = require('./msgTypes');
 console.log(
   `Lookup Worker starting with configuration\n${util.inspect(workerData, {
     depth: null,
-    compact: false
+    compact: false,
   })}\n`
 );
 
@@ -38,17 +38,21 @@ const workerId = workerData.workerId;
 
 /**
  * Returns T/F indicating if the supplied URL matches a behaviors match object
- * @param {string} url - The URL to match to a behavior
- * @param {Object} match - A behaviors match object
+ * @param {Object} query - The URL to match to a behavior
+ * @param {Object} behavior - A behavior
  * @return {boolean}
  */
-function behaviorMatches(url, match) {
+function behaviorMatches(query, behavior) {
+  if (query.name) {
+    return query.name === behavior.name;
+  }
+  const match = behavior.match;
+  const url = query.url;
   if (match.regex) {
     if (match.regex.base) {
       if (match.regex.base.test(url)) {
         const sub = match.regex.sub;
-        let subIdx = sub.length;
-        while (subIdx--) {
+        for (let subIdx = 0; subIdx < sub.length; subIdx++) {
           if (sub[subIdx].test(url)) return true;
         }
       }
@@ -98,6 +102,8 @@ function onMsg(msg) {
       return lookupBehavior(msg);
     case msgTypes.lookupBehaviorInfo:
       return lookupBehaviorInfo(msg);
+    case msgTypes.behaviorList:
+      return behaviorList(msg);
     case msgTypes.shutdown:
       if (serverCom) {
         serverCom.close();
@@ -109,16 +115,17 @@ function onMsg(msg) {
 /**
  * Attempts to find a matching behavior returning the matching behavior
  * if a match was made otherwise the default behavior
- * @param {Object} msg - The message sent to the LookupWorker from the
+ * @param {Object} query - The message sent to the LookupWorker from the
  * parent process
  * @return {Object}
  */
-function findBehavior(msg) {
-  const url = msg.url;
+function findBehavior(query) {
+  if (query.name && behaviorMetadata.defaultBehavior.name === query.name) {
+    return behaviorMetadata.defaultBehavior;
+  }
   const behaviors = behaviorMetadata.behaviors;
-  let behaviorIdx = behaviors.length;
-  while (behaviorIdx--) {
-    if (behaviorMatches(url, behaviors[behaviorIdx].match)) {
+  for (let behaviorIdx = 0; behaviorIdx < behaviors.length; behaviorIdx++) {
+    if (behaviorMatches(query, behaviors[behaviorIdx])) {
       return behaviors[behaviorIdx];
     }
   }
@@ -135,10 +142,10 @@ async function lookupBehavior(msg) {
   const results = {
     behavior: null,
     wasError: false,
-    errorMsg: null
+    errorMsg: null,
   };
   try {
-    const foundBehavior = findBehavior(msg);
+    const foundBehavior = findBehavior(msg.query);
     results.behavior = path.join(behaviorDir, foundBehavior.fileName);
   } catch (error) {
     results.wasError = true;
@@ -157,15 +164,10 @@ async function lookupBehaviorInfo(msg) {
   const results = {
     behavior: null,
     wasError: false,
-    errorMsg: null
+    errorMsg: null,
   };
   try {
-    const foundBehavior = findBehavior(msg);
-    results.behavior = {
-      name: foundBehavior.name,
-      description: foundBehavior.description,
-      defaultBehavior: foundBehavior.defaultBehavior || false
-    };
+    results.behavior = findBehavior(msg.query);
   } catch (error) {
     results.wasError = true;
     results.errorMsg = error.message;
@@ -184,14 +186,22 @@ function reloadBehaviors(msg) {
   const results = {
     reloadResults: {
       defaultBehavior: behaviorMetadata.defaultBehavior.name,
-      numBehaviors: behaviorMetadata.behaviors.length
+      numBehaviors: behaviorMetadata.behaviors.length,
     },
     wasError: false,
-    errorMsg: null
+    errorMsg: null,
   };
   reply(msg.id, msgTypes.reloadBehaviorsResults, results);
 }
 
+function behaviorList(msg) {
+  reply(msg.id, msgTypes.behaviorListResults, {
+    list: behaviorMetadata,
+    wasError: false,
+    errorMsg: null,
+  });
+}
+
 parentPort.once('message', msg => {
   if (!(msg.serverCom instanceof MessagePort)) {
     throw new Error('The serverCom was not an instance of MessagePort!');

api/msgTypes.js

@@ -10,5 +10,7 @@ module.exports = {
   lookupBehavior: 'lookup-behavior',
   lookupBehaviorInfo: 'lookup-behavior-info',
   behaviorLookupResults: 'behavior-lookup-results',
-  shutdown: 'shutdown'
+  behaviorList: 'list-all-behaviors',
+  behaviorListResults: 'list-all-behaviors-results',
+  shutdown: 'shutdown',
 };