Simplifying and improving structure.

lib/services/classify.js

@@ -1,27 +1,29 @@
 import Q from 'q';
 import debug from 'debug';
 import BrainJSClassifier from 'natural-brain';
+import pos from 'pos';
+import crypto from 'crypto';
 
 import Extractor from '../extractor';
 
 const log = debug('mysam:classifier');
+const lexer = new pos.Lexer();
+const tagger = new pos.Tagger();
 
 export function addPos(hook, next) {
-  let pos = this.app.service('pos');
   let data = hook.result;
-
-  pos.create({
-    text: data.input
-  }, (error, pos) => {
-    data.pos = pos;
-    next();
+  let tokens = lexer.lex(data.input);
+  let tags = tagger.tag(tokens).map(function(touple) {
+    return touple[1];
   });
+
+  data.pos = { tokens, tags };
+  next();
 }
 
 export function extract(hook, next) {
-  let pos = this.app.service('pos');
   let data = hook.result;
-  let e = new Extractor(pos.tokenize(data.text), data.tags || []);
+  let e = new Extractor(lexer.lex(data.text), data.tags || []);
 
   data.extracted = e.extract(data.pos.tokens);
   next();
@@ -37,16 +39,14 @@ export default {
       this.retrain = false;
     }
 
-    let input = data.text;
+    let { input } = data;
+    let id = crypto.createHash('md5').update(input).digest('hex');
     let action = this.classifier.classify(input);
     let classifications = this.classifier.getClassifications(input);
 
     Q.ninvoke(this.actions, 'get', action)
       .then(action => {
-        return Object.assign(action, {
-          input: data.text,
-          classifications
-        });
+        return Object.assign(action, { id, input, classifications });
       })
       .then(result => {
         log(result);
@@ -65,6 +65,9 @@ export default {
   setup(app) {
     let add = this.add.bind(this);
 
+    // We want to classify every word
+    BrainJSClassifier.disableStopWords();
+
     this.classifier = new BrainJSClassifier();
     this.app = app;
 

lib/services/index.js

@@ -1,4 +1,3 @@
-import pos from './pos';
 import classify from './classify';
 import nedb from 'feathers-nedb';
 import plugins from './plugins';
@@ -8,7 +7,6 @@ export default function() {
 
   return app.use('/actions', nedb('actions'))
     .use('/configurations', nedb('configuration'))
-    .use('/pos', pos)
     .use('/plugins', plugins)
     .use('/classify', classify);
 }

lib/services/plugins.js

@@ -1,6 +1,6 @@
 import memory from 'feathers-memory';
 
-export default memory().extend({
+export default memory({ idField: 'name' }).extend({
   remove: null,
   update: null,
   patch: null

package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "start": "node lib/",
     "test": "npm run jshint && npm run mocha",
-    "watch": "supervisor --ignore node_modules/ --no-restart-on error lib/",
+    "watch": "DEBUG=mysam:* supervisor --ignore node_modules/ --no-restart-on error lib/",
     "jshint": "jshint lib/. test/. --config",
     "mocha:watch": "mocha --compilers js:babel/register test/ --watch --recursive --growl --reporter Min",
     "mocha": "mocha --compilers js:babel/register test/ --recursive"
@@ -22,7 +22,7 @@
     "feathers-memory": "^0.3.4",
     "feathers-nedb": "^0.1.0",
     "lodash": "^3.9.3",
-    "natural-brain": "^0.1.0",
+    "natural-brain": "^0.2.0",
     "npm": "^2.14.3",
     "pos": "^0.1.9",
     "q": "^1.4.1"

test/services/classify.test.js

@@ -62,14 +62,18 @@ describe('BrainJS classifier', () => {
 
   it('classifies actions', done => {
     classify.create({
-      text: 'Hey, you hear me?'
+      input: 'hey you hear me'
     }, {}, (error, data) => {
       assert.equal(data._id, 0);
       assert.deepEqual(data.action, fixture[0].action);
       assert.equal(data.classifications.length, 3);
+      assert.deepEqual(data.pos, {
+        tokens: [ 'hey', 'you', 'hear', 'me' ],
+        tags: [ 'UH', 'PRP', 'VB', 'PRP' ]
+      });
 
       classify.create({
-        text: 'What\'s the meaning of my life?'
+        input: 'What\'s the meaning of my life?'
       }, {}, (error, data) => {
         assert.equal(data._id, 2);
         assert.deepEqual(data.action, fixture[2].action);
@@ -80,7 +84,7 @@ describe('BrainJS classifier', () => {
 
   it('extracts tags', done => {
     classify.create({
-      text: 'Can you please say hi to Gabe'
+      input: 'Can you please say hi to Gabe'
     }, {}, (error, data) => {
       assert.equal(data._id, 1);
       assert.deepEqual(data.extracted, { subject: ['Gabe'] });
@@ -99,7 +103,7 @@ describe('BrainJS classifier', () => {
 
     actions.create(newAction, (error, action) => {
       classify.create({
-        text: 'Can you play music for us?'
+        input: 'Can you play music for us?'
       }, {}, (error, data) => {
         assert.equal(data._id, action._id);
         assert.equal(data.text, action.text);