Fix bug in the scanning stage of earley

If a scan is successful, a new earley item should only be added to the
next state if one exists to be added to. This means the current state
should be < tokens.length, not states.length (states.length =
tokens.length + 1).

lib/parser.js

@@ -141,15 +141,15 @@ function scan(tokens, states, i, j) {
   // scan
   if (curr.rule[curr.position] instanceof RegExp) {
     // regex matches token
-    if (curr.rule[curr.position].test(tokens[i]) && i < states.length) {
+    if (curr.rule[curr.position].test(tokens[i]) && i < tokens.length) {
       newItem = Object.assign({}, curr);
       newItem.position += 1;
       states[i + 1].push(newItem);
     }
   }
   if (typeof curr.rule[curr.position] === 'string') {
     // string equals token
-    if (curr.rule[curr.position] === tokens[i] && i < states.length) {
+    if (curr.rule[curr.position] === tokens[i] && i < tokens.length) {
       newItem = Object.assign({}, curr);
       newItem.position += 1;
       states[i + 1].push(newItem);

spec/parser-spec.js

@@ -82,6 +82,19 @@ describe('parser', () => {
       });
     });
 
+    it('does not attempt to insert item beyond number of states', () => {
+      let s = Sym('S')
+        , n = Sym('N')
+        , v = Sym('V')
+        , rules = [
+          Rule(s, [n, v]),
+          Rule(n, [/\w+/]),
+          Rule(v, [/\w+/])
+        ];
+      f = () => earley('hello there'.split(' '), rules);
+      expect(f).not.toThrow();
+    });
+
   });
 
   describe('dfs()', () => {