Cleanup and bug fixes to machine learning code. Have http and https s…

…ervers run in parallel (still sorting some stuff here). Add values to generic removePunctuation and consolidate on its use.
imbrianj · Jan 11, 2018 · b3f56ba · b3f56ba
1 parent 0448208
commit b3f56ba
Show file tree

Hide file tree

Showing 12 changed files with 117 additions and 53 deletions.
diff --git a/app.js b/app.js
@@ -39,14 +39,17 @@ var fs              = require('fs'),
     db              = require(__dirname + '/lib/db'),
     webSockets      = require(__dirname + '/lib/webSockets'),
     controllers,
+    sslServer,
     server,
     wsServer,
+    sslWsServer,
     settings,
     configFile      = __dirname + '/config/config.js',
     arg,
     options,
     connection,
-    startup;
+    startup,
+    sslStartup;
 
 for (arg in process.argv) {
   if (process.argv.hasOwnProperty(arg)) {
@@ -103,6 +106,12 @@ startup = function () {
   }
 };
 
+sslStartup = function () {
+  'use strict';
+
+  console.log('\x1b[36mListening on port ' + settings.config.config.ssl.serverPort + ' for SSL\x1b[0m');
+};
+
 fs.stat(configFile, function (err, data) {
   'use strict';
 
@@ -146,27 +155,39 @@ fs.stat(configFile, function (err, data) {
       else if (process.platform === 'linux') {
         staticAssets.buildCerts(settings.config.config.ssl);
       }
-    }
 
-    if (options) {
-      console.log('\x1b[36mSSL\x1b[0m: Enabled');
-      server = https.createServer(options, connection).listen(settings.config.config.serverPort, startup);
+      if (options) {
+        console.log('\x1b[36mSSL\x1b[0m: Enabled');
+        sslServer = https.createServer(options, connection).listen(settings.config.config.ssl.serverPort, sslStartup);
+
+        // SSL will need it's own Web Sockets server.
+        if (sslServer) {
+          sslWsServer = new webSocketServer({ httpServer : sslServer }, 'echo-protocol');
+
+          sslWsServer.on('request', function (request) {
+            webSockets.newConnection(request, controllers);
+          });
+        }
+      }
     }
 
-    else {
-      console.log('\x1b[31mSSL\x1b[0m: Disabled');
+    if (settings.config.config.serverPort) {
       server = http.createServer(connection).listen(settings.config.config.serverPort, startup);
+
+      // Or you're connecting with Web Sockets
+      wsServer = new webSocketServer({ httpServer : server }, 'echo-protocol');
+
+      wsServer.on('request', function (request) {
+        webSockets.newConnection(request, controllers);
+      });
+    }
+
+    if ((!sslServer) && (!server)) {
+      console.log('\x1b[31mError\x1b[0m: No server started.  Do you have a serverPort configured?');
     }
 
     ai.processFiles();
     db.readDb();
-
-    // Or you're connecting with Web Sockets
-    wsServer = new webSocketServer({ httpServer : server }, 'echo-protocol');
-
-    wsServer.on('request', function (request) {
-      webSockets.newConnection(request, controllers);
-    });
   }
 
   else {

diff --git a/apps/announceTrendingNews.js b/apps/announceTrendingNews.js
@@ -32,7 +32,7 @@ module.exports = (function () {
   var CooldownWords = {};
 
   return {
-    version : 20171127,
+    version : 20180110,
 
     translate : function (token, lang) {
       var translate = require(__dirname + '/../lib/translate');
@@ -41,12 +41,13 @@ module.exports = (function () {
     },
 
     findProperNouns : function (articleParts, blacklist) {
-      var i           = 0,
+      var translate   = require(__dirname + '/../lib/translate'),
+          i           = 0,
           word,
           properNouns = [];
 
       for (i; i < articleParts.length; i += 1) {
-        word = articleParts[i].replace(/[.,\/#!$%\^&\*;:{}=\-_`~()]/g, '');
+        word = translate.stripPunctuation(articleParts[i]);
         word = word.replace(/\s{2,}/g, ' ');
 
         if ((word) && (this.isWordCapitalized(word)) && (blacklist.indexOf(word.toLowerCase()) === -1)) {

diff --git a/apps/gerty/ai.js b/apps/gerty/ai.js
@@ -30,14 +30,17 @@ module.exports = (function () {
   'use strict';
 
   return {
-    version : 20170920,
+    version : 20180109,
 
     /**
-     * Take in a typeClass and return the type of device it's categorized as.
+     * Take in a deviceId, command and set of controllers.  Determines if
+     * there's reasonable intent for a given action and will notify or act on
+     * that implied command.
      */
     findIntent : function (deviceId, command, controllers) {
-      var config = controllers.config,
-          delay  = config.ai.executeDelaySeconds || 1;
+      var config         = controllers.config,
+          trainingWheels = config.ai.trainingWheels      || false,
+          delay          = config.ai.executeDelaySeconds || 1;
 
       if (!config.ai.disable) {
         setTimeout(function() {
@@ -47,24 +50,30 @@ module.exports = (function () {
               translate,
               tempDevice,
               utterance,
-              i      = 0;
+              i      = 0,
+              j      = 0;
 
           if (intent.length) {
+            runCommand = require(__dirname + '/../../lib/runCommand');
+
             for (tempDevice in controllers) {
               if ((tempDevice !== 'config') && (controllers[tempDevice].config.typeClass === 'gerty')) {
-                runCommand = require(__dirname + '/../../lib/runCommand');
                 translate  = require(__dirname + '/../../lib/translate');
 
                 for (i; i < intent.length; i += 1) {
                   utterance = translate.translate('{{i18n_AI_INTENT}}', 'gerty', config.language).replace('{{INPUT}}', command).replace('{{CONFIDENCE}}', intent[i].confidence).replace('{{DEVICE}}', intent[i].subdevice).replace('{{COMMAND}}', intent[i].command);
                   runCommand.runCommand(tempDevice, 'text-' + utterance);
                 }
 
-                // Eventually fired intent should note that this was an AI command,
-                // so the intended event doesn't cascade to others (?)
                 break;
               }
             }
+
+            if (!trainingWheels) {
+              for (j; j < intent.length; j += 1) {
+                runCommand.runCommand(intent[j].device, 'subdevice-' + intent[j].subdevice + '-' + intent[j].command);
+              }
+            }
           }
         }, delay * 1000);
       }

diff --git a/cache/version.txt b/cache/version.txt
@@ -1 +1 @@
-1515546199655
+1515639239743
diff --git a/config/config.js b/config/config.js
@@ -26,12 +26,13 @@ exports.config = {
     // in all (or any) browsers.  Also note that SmartThings cannot contact
     // SwitchBoard for real-time updates.
     ssl          : {
-      country  : 'US',
-      state    : 'Washington',
-      city     : 'Seattle',
-      org      : 'Switchboard',
-      name     : 'Switchboard',
-      disabled : true
+      country    : 'US',
+      state      : 'Washington',
+      city       : 'Seattle',
+      org        : 'SwitchBoard',
+      name       : 'SwitchBoard',
+      serverPort : 8181,
+      disabled   : false
     },
     ai           : {
       eventLogging         : false,
@@ -40,6 +41,9 @@ exports.config = {
       eventCooldownMinutes : 120,
       minimumThreshold     : 100,
       confidence           : 90,
+      // With trainingWheels on, Gerty will notify you of intent.  Set this to
+      // false and he will take action.
+      trainingWheels       : true,
       disable              : false
     }
   },
@@ -750,7 +754,7 @@ exports.config = {
                                                      'I\'m Back'     : 'smartthings=subdevice-mode-Home;nest=Home',
                                                      'Welcome Home'  : 'smartthings=subdevice-mode-Home;nest=Home' },
                                    controllerIds : ['samsung', 'roku', 'ps3', 'panasonic', 'lg', 'pioneer', 'denon', 'speech', 'stocks', 'weather', 'foscam', 'mp3', 'sms', 'pushover', 'smartthings', 'powerView', 'nest', 'switchBoardCI', 'xbmc', 'raspberryRemote', 'wemo', 'activeBuilding', 'clientMp3', 'clientNotify', 'clientSpeech'] } },
-    disabled       : true
+    disabled       : false
   },
 
   /*

diff --git a/devices/nest/controller.js b/devices/nest/controller.js
@@ -29,7 +29,7 @@ module.exports = (function () {
    * @requires querystring, fs, https
    */
   return {
-    version : 20171127,
+    version : 20180109,
 
     inputs : ['command', 'text', 'list', 'subdevice'],
 
@@ -262,6 +262,7 @@ module.exports = (function () {
                 label    : that.findLabel(response.topaz[i].where_id, config.language),
                 type     : 'protect',
                 lastOn   : matched.lastOn,
+                lastOff  : matched.lastOff,
                 duration : matched.duration,
                 readOnly : true
               });
@@ -288,6 +289,7 @@ module.exports = (function () {
                 label        : that.findLabel(response.device[i].where_id),
                 type         : 'thermostat',
                 lastOn       : matched.lastOn,
+                lastOff      : matched.lastOff,
                 duration     : matched.duration,
                 celsius      : celsius
               });

diff --git a/devices/smartthings/controller.js b/devices/smartthings/controller.js
@@ -29,7 +29,7 @@ module.exports = (function () {
    * @fileoverview Basic control of SmartThings endpoint.
    */
   return {
-    version : 20171108,
+    version : 20180109,
 
     inputs  : ['list', 'subdevice'],
 
@@ -213,6 +213,7 @@ module.exports = (function () {
               id       : util.sanitize(device.id),
               label    : util.sanitize(device.label),
               lastOn   : util.sanitize(matched.lastOn),
+              lastOff  : util.sanitize(matched.lastOff),
               duration : util.sanitize(matched.duration)
             };
 

diff --git a/js/combo.min.js b/js/combo.min.js
diff --git a/lib/ai.js b/lib/ai.js
@@ -35,7 +35,7 @@ module.exports = (function () {
   var PROCESSED = null;
 
   return {
-    version : 20171007,
+    version : 20180110,
 
     /**
      * Find all unprocessed, raw DB files that need to be processed.
@@ -257,9 +257,16 @@ module.exports = (function () {
             if (subdeviceId === currentState.value.devices[i].label) {
               now = new Date().getTime();
 
-              if ((currentState.value.devices[i].lastOn + (eventCooldownMinutes * 60000) > now) &&
-                  (!currentState.value.devices[i].readOnly)) {
-                returnState = currentState.value.devices[i];
+              if (!currentState.value.devices[i].readOnly) {
+                if (((!currentState.value.devices[i].lastOn) && (!currentState.value.devices[i].lastOff))                                       ||
+                    ((currentState.value.devices[i].lastOn)  && (now > currentState.value.devices[i].lastOn  + (eventCooldownMinutes * 60000))) ||
+                    ((currentState.value.devices[i].lastOff) && (now > currentState.value.devices[i].lastOff + (eventCooldownMinutes * 60000)))) {
+                  returnState = currentState.value.devices[i];
+                }
+
+                else {
+                  console.log('\x1b[35mAI\x1b[0m: Ignoring ' + currentState.value.devices[i].label + ' since it recently changed state.');
+                }
               }
 
               break;
@@ -315,17 +322,27 @@ module.exports = (function () {
                       if (values[valueType]) {
                         confidence = parseInt(((values[valueType] / total) * 100), 10);
 
-                        if (confidence > config.ai.confidence) {
-                          // Only indicate intent if it differs from the current
-                          // state or is not read only.
-                          intendedDevice = this.findDevice(deviceId, subdeviceId, controllers, eventCooldownMinutes);
+                        // Only indicate intent if it differs from the current
+                        // state or is not read only.
+                        intendedDevice = this.findDevice(deviceId, subdeviceId, controllers, eventCooldownMinutes);
 
+                        if (confidence > config.ai.confidence) {
                           if ((intendedDevice) && (intendedDevice.state) && (intendedDevice.state !== valueType)) {
+                            console.log('\x1b[35mAI\x1b[0m: Intent available with ' + confidence + '% confidence that ' + subdeviceId + ' should be ' + valueType + '.');
+
                             intent.push({ device     : device,
                                           subdevice  : subdeviceId,
                                           command    : valueType,
                                           confidence : confidence });
                           }
+
+                          else if ((intendedDevice) && (intendedDevice.state)) {
+                            console.log('\x1b[35mAI\x1b[0m: Already set, but ' + confidence + '% confident ' + subdeviceId + ' should be ' + valueType + '.');
+                          }
+                        }
+
+                        else if (intendedDevice) {
+                          console.log('\x1b[35mAI\x1b[0m: Only ' + confidence + '% confident ' + subdeviceId + ' should be ' + valueType + '.');
                         }
                       }
                     }
@@ -335,6 +352,10 @@ module.exports = (function () {
             }
           }
         }
+
+        else {
+          console.log('\x1b[35mAI\x1b[0m: Only ' + summary.count + ' events for ' + deviceId + ' ' + command + '.');
+        }
       }
 
       return intent;

diff --git a/lib/deviceState.js b/lib/deviceState.js
@@ -47,13 +47,14 @@ module.exports = (function () {
     },
 
     findDeviceOnTime : function (deviceId, newState, oldState) {
-      var now = Math.round(new Date().getTime() / 1000),
+      var now = new Date().getTime(),
           i   = 0,
           device;
 
       if (oldState.state !== newState.state) {
         if (newState.state === 'ok') {
-          State[deviceId].lastOn = now;
+          State[deviceId].lastOn  = now;
+          State[deviceId].lastOff = null;
         }
 
         else if (newState.state === 'err') {
@@ -62,7 +63,8 @@ module.exports = (function () {
             State[deviceId].duration += (now - State[deviceId].lastOn);
           }
 
-          State[deviceId].lastOn = null;
+          State[deviceId].lastOn  = null;
+          State[deviceId].lastOff = now;
         }
       }
 
@@ -72,7 +74,8 @@ module.exports = (function () {
 
           if ((oldState.value.devices[i]) && (oldState.value.devices[i].state !== device.state)) {
             if (device.state === 'on') {
-              State[deviceId].value.devices[i].lastOn = now;
+              State[deviceId].value.devices[i].lastOn  = now;
+              State[deviceId].value.devices[i].lastOff = null;
             }
 
             else if (device.state === 'off') {
@@ -81,7 +84,8 @@ module.exports = (function () {
                 State[deviceId].value.devices[i].duration += (now - State[deviceId].value.devices[i].lastOn);
               }
 
-              State[deviceId].value.devices[i].lastOn = null;
+              State[deviceId].value.devices[i].lastOff = now;
+              State[deviceId].value.devices[i].lastOn  = null;
             }
           }
         }
@@ -98,7 +102,7 @@ module.exports = (function () {
      */
     updateState : function (deviceId, typeClass, config) {
       var webSockets = require(__dirname + '/webSockets'),
-          now        = Math.round(new Date().getTime() / 1000),
+          now        = new Date().getTime(),
           oldState;
 
       if (deviceId) {