From 58d908785f31ad99cd27604ba8a8831c26aee6e3 Mon Sep 17 00:00:00 2001
From: Gopal Shah <simplecooldude1@gmail.com>
Date: Mon, 22 May 2017 19:03:21 -0700
Subject: [PATCH] correctly tokenize strings

---
 dist/wade.js     | 20 +++++++++++++++++---
 dist/wade.min.js |  2 +-
 src/index.js     | 20 +++++++++++++++++---
 3 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/dist/wade.js b/dist/wade.js
index 801aee7..e556edd 100644
--- a/dist/wade.js
+++ b/dist/wade.js
@@ -77,6 +77,20 @@
       }
     }
     
+    var getWords = function(str) {
+      var lastIndex = str.length - 1;
+    
+      if(str[0] === " ") {
+        str = str.substring(1);
+      }
+    
+      if(str[lastIndex] === " ") {
+        str = str.substring(0, lastIndex);
+      }
+    
+      return str.split(" ");
+    }
+    
     var lowercase = function(str) {
       return str.toLowerCase();
     }
@@ -86,7 +100,7 @@
     }
     
     var removeStopWords = function(str) {
-      var words = str.split(" ");
+      var words = getWords(str);
     
       for(var i = 0; i < words.length; i++) {
         if(stopWords.indexOf(words[i]) !== -1) {
@@ -100,7 +114,7 @@
     var Wade = function(data) {
       var search = function(item) {
         var index = search.index;
-        var keywords = Wade.process(item).split(" ");
+        var keywords = getWords(Wade.process(item));
         var keywordsLength = keywords.length;
         var fullwordsLength = keywordsLength - 1;
         var scoreIncrement = 1 / keywordsLength;
@@ -141,7 +155,7 @@
     Wade.index = function(data) {
       var index = {};
       for(var i = 0; i < data.length; i++) {
-        var str = data[i].split(" ");
+        var str = getWords(data[i]);
         for(var j = 0; j < str.length; j++) {
           var item = str[j];
           var itemLength = item.length - 1;
diff --git a/dist/wade.min.js b/dist/wade.min.js
index be5c475..80b7039 100644
--- a/dist/wade.min.js
+++ b/dist/wade.min.js
@@ -3,4 +3,4 @@
  * Copyright 2017 Kabir Shah
  * Released under the MIT License
  */
-!function(e,o){"object"==typeof module&&module.exports?module.exports=o():e.Wade=o()}(this,function(){var e=["about","after","all","also","am","an","and","another","any","are","as","at","be","because","been","before","being","between","both","but","by","came","can","come","could","did","do","each","for","from","get","got","has","had","he","have","her","here","him","himself","his","how","if","in","into","is","it","like","make","many","me","might","more","most","much","must","my","never","now","of","on","only","or","other","our","out","over","said","same","see","should","since","some","still","such","take","than","that","the","their","them","then","there","these","they","this","those","through","to","too","under","up","very","was","way","we","well","were","what","where","which","while","who","with","would","you","your","a","i"],o=/\.|\,|\!/g,n=function(e,o){for(var n=o,r=0;r<e.length&&void 0!==(n=n[e[r]]);r++);return n},r=function(e,o,n,r){for(var t=null,i=0;i<e.length;i++){var h=e[i];void 0===(t=n[h])?(n[h]=o.length,o.push({index:h,score:r})):o[t].score+=r}},t=function(e,o,t,i,h){var a=n(e,o);void 0!==a&&void 0!==a.id&&r(a.id,t,i,h)},i=function(e,o,t,i,h){var a=n(e,o);if(void 0!==a)for(var u=[a],s=null,l=0;0!==u.length;){void 0!==(s=u[l]).id&&r(s.id,t,i,h),u.pop(),l--;for(var d in s)u.push(s[d]),l++}},h=function(e){return e.toLowerCase()},a=function(e){return e.replace(o,"")},u=function(o){for(var n=o.split(" "),r=0;r<n.length;r++)-1!==e.indexOf(n[r])&&n.splice(r,1);return n.join(" ")},s=function(e){for(var o=function(e){for(var n=o.index,r=s.process(e).split(" "),h=r.length,a=h-1,u=1/h,l=[],d={},f=0;f<a;f++)t(r[f],n,l,d,u);return i(r[a],n,l,d,u),l},n=0;n<e.length;n++)e[n]=s.process(e[n]);return o.index=s.index(e),o.data=e,o};return s.pipeline=[h,a,u],s.process=function(e){for(var o=s.pipeline,n=0;n<o.length;n++)e=o[n](e);return e},s.index=function(e){for(var o={},n=0;n<e.length;n++)for(var r=e[n].split(" "),t=0;t<r.length;t++){for(var i=r[t],h=i.length-1,a=o,u=0;u<h;u++){var s=i[u],l=a[s];l=void 0===l?{}:l,a[s]=l,a=l}var d=i[h];void 0===a[d]?a[d]={id:[n]}:void 0===(a=a[d]).id?a.id=[n]:a.id.push(n)}return o},s.version="0.2.0",s});
\ No newline at end of file
+!function(e,o){"object"==typeof module&&module.exports?module.exports=o():e.Wade=o()}(this,function(){var e=["about","after","all","also","am","an","and","another","any","are","as","at","be","because","been","before","being","between","both","but","by","came","can","come","could","did","do","each","for","from","get","got","has","had","he","have","her","here","him","himself","his","how","if","in","into","is","it","like","make","many","me","might","more","most","much","must","my","never","now","of","on","only","or","other","our","out","over","said","same","see","should","since","some","still","such","take","than","that","the","their","them","then","there","these","they","this","those","through","to","too","under","up","very","was","way","we","well","were","what","where","which","while","who","with","would","you","your","a","i"],o=/\.|\,|\!/g,n=function(e,o){for(var n=o,r=0;r<e.length&&void 0!==(n=n[e[r]]);r++);return n},r=function(e,o,n,r){for(var t=null,i=0;i<e.length;i++){var h=e[i];void 0===(t=n[h])?(n[h]=o.length,o.push({index:h,score:r})):o[t].score+=r}},t=function(e,o,t,i,h){var a=n(e,o);void 0!==a&&void 0!==a.id&&r(a.id,t,i,h)},i=function(e,o,t,i,h){var a=n(e,o);if(void 0!==a)for(var u=[a],s=null,d=0;0!==u.length;){void 0!==(s=u[d]).id&&r(s.id,t,i,h),u.pop(),d--;for(var l in s)u.push(s[l]),d++}},h=function(e){var o=e.length-1;return" "===e[0]&&(e=e.substring(1))," "===e[o]&&(e=e.substring(0,o)),e.split(" ")},a=function(e){return e.toLowerCase()},u=function(e){return e.replace(o,"")},s=function(o){for(var n=h(o),r=0;r<n.length;r++)-1!==e.indexOf(n[r])&&n.splice(r,1);return n.join(" ")},d=function(e){for(var o=function(e){for(var n=o.index,r=h(d.process(e)),a=r.length,u=a-1,s=1/a,l=[],f={},c=0;c<u;c++)t(r[c],n,l,f,s);return i(r[u],n,l,f,s),l},n=0;n<e.length;n++)e[n]=d.process(e[n]);return o.index=d.index(e),o.data=e,o};return d.pipeline=[a,u,s],d.process=function(e){for(var o=d.pipeline,n=0;n<o.length;n++)e=o[n](e);return e},d.index=function(e){for(var o={},n=0;n<e.length;n++)for(var r=h(e[n]),t=0;t<r.length;t++){for(var i=r[t],a=i.length-1,u=o,s=0;s<a;s++){var d=i[s],l=u[d];l=void 0===l?{}:l,u[d]=l,u=l}var f=i[a];void 0===u[f]?u[f]={id:[n]}:void 0===(u=u[f]).id?u.id=[n]:u.id.push(n)}return o},d.version="0.2.0",d});
\ No newline at end of file
diff --git a/src/index.js b/src/index.js
index 90e96f7..f004c07 100644
--- a/src/index.js
+++ b/src/index.js
@@ -67,6 +67,20 @@ var containsPrefix = function(pattern, index, results, resultsLocations, scoreIn
   }
 }
 
+var getWords = function(str) {
+  var lastIndex = str.length - 1;
+
+  if(str[0] === " ") {
+    str = str.substring(1);
+  }
+
+  if(str[lastIndex] === " ") {
+    str = str.substring(0, lastIndex);
+  }
+
+  return str.split(" ");
+}
+
 var lowercase = function(str) {
   return str.toLowerCase();
 }
@@ -76,7 +90,7 @@ var removePunctuation = function(str) {
 }
 
 var removeStopWords = function(str) {
-  var words = str.split(" ");
+  var words = getWords(str);
 
   for(var i = 0; i < words.length; i++) {
     if(stopWords.indexOf(words[i]) !== -1) {
@@ -90,7 +104,7 @@ var removeStopWords = function(str) {
 var Wade = function(data) {
   var search = function(item) {
     var index = search.index;
-    var keywords = Wade.process(item).split(" ");
+    var keywords = getWords(Wade.process(item));
     var keywordsLength = keywords.length;
     var fullwordsLength = keywordsLength - 1;
     var scoreIncrement = 1 / keywordsLength;
@@ -131,7 +145,7 @@ Wade.process = function(item) {
 Wade.index = function(data) {
   var index = {};
   for(var i = 0; i < data.length; i++) {
-    var str = data[i].split(" ");
+    var str = getWords(data[i]);
     for(var j = 0; j < str.length; j++) {
       var item = str[j];
       var itemLength = item.length - 1;