A few modifications on lexer

1) Rethrown exception from callback_token
2) Separated function for finding individual matches

lib/wolf-lexer.js

@@ -15,6 +15,8 @@ function WolfLexer() {
     this.resumeOnError = false;
     this.errorOccured = false;
     this.defaultRules = false;
+    this.source = '';
+    this.inStepMode = false;
 }
 
 function Rule(p, k) {
@@ -47,64 +49,82 @@ WolfLexer.prototype.reset = function() {
     this.rules = [];
     this.resumeOnError = false;
     this.errorOccured = false;
+    this.source = '';
+    this.inStepMode = false;
 };
 
-WolfLexer.prototype.scan = function(source, callback_token, callback_error) {
+WolfLexer.prototype.findMatch = function(text, position) {
+    for (var index = 0; index < this.rules.length; index++) {
+        var r = this.rules[index];
+        var m = r.pattern.exec(text);
+        if (!m) {
+            continue;
+        }
+
+        if (0 !== m.index) {
+            continue;
+        }
+
+        var t = new Token();
+        t.position = position;
+        t.symbol = m[0];
+        t.kind = r.kind;
+        return t;
+    }
+};
+
+WolfLexer.prototype.scan = function(text, callback_token, callback_error) {
     if (!this.defaultRules) {
         var whiteSpace = new Rule(/\s+/, '$_whiespace');
         this.rules.push(whiteSpace);
     }
 
+    this.source = text;
     this.position = 0;
-    while (0 < source.length) {
-        var matchFound = false;
-        for (var index = 0; index < this.rules.length; index++) {
-            var r = this.rules[index];
-            if (r) {
-                var m = r.pattern.exec(source);
-                if (m) {
-                    if (0 === m.index) {
-                        var t = new Token();
-                        t.position = this.position;
-                        t.symbol = m[0];
-                        t.kind = r.kind;
-                        matchFound = true;
-                        this.errorOccured = false;
-                        this.position += t.symbol.length;
-                        try {
-                            if (t.kind !== '$_whiespace') {
-                                callback_token(t);
-                            }
-                        } catch (ex) {
-                            // todo: Nothing
-                        }
-                        source = source.substr(t.symbol.length);
-                        break;
+
+    while (0 < this.source.length) {
+        var t = this.findMatch(this.source, this.position);
+        if (t) {
+            this.errorOccured = false;
+            this.position += t.symbol.length;
+            try {
+                if (t.kind !== '$_whiespace') {
+                    if (callback_token) {
+                        callback_token(t);
+                    } else {
+                        // todo: in step mode
                     }
+
                 }
+            } catch (ex) {
+                throw new Error('Error on invoking callback function. ' + ex.message);
             }
-        }
-        if (!matchFound) {
-            if (!this.errorOccured) {
-                var terr = null;
-                if ((callback_error) && (!this.errorOccured)) {
-                    terr = new TokenError('Unidentified token.', this.position);
-                    try {
-                        callback_error(terr);
-                    } catch (ex) {
-                        // todo: Nothing
-                    }
-                    if (!this.resumeOnError) {
-                        break;
-                    }
-                    this.errorOccured = true;
-                } else {
-                    terr = new Error('Unidentified token found at ' + this.position + '.');
-                    throw terr;
+            this.source = this.source.substr(t.symbol.length);
+        } else {
+            var currrentPosition = this.position;
+            this.source = this.source.substr(1);
+            this.position++;
+
+            if (this.errorOccured) {
+                continue;
+            }
+
+            var terr = null;
+            if ((callback_error) && (!this.errorOccured)) {
+                terr = new TokenError('Unidentified token.', currrentPosition);
+                try {
+                    callback_error(terr);
+                } catch (ex) {
+                    // todo: Nothing
                 }
+                if (!this.resumeOnError) {
+                    break;
+                }
+                this.errorOccured = true;
+            } else {
+                terr = new Error('Unidentified token found at ' + currrentPosition + '.');
+                throw terr;
             }
-            source = source.substr(1);
-            this.position++;
         }
     }
 };

package.json

@@ -1,7 +1,7 @@
 {
   "name": "wolf-lexer",
   "description": "Yet another lexer module in nodejs.",
-  "version": "0.0.5",
+  "version": "0.0.6",
   "homepage": "https://github.com/benoybose/wolf-lexer",
   "bugs": "https://github.com/benoybose/wolf-lexer/issues",
   "license": "MIT",

test/wolf-lexer_test.js

@@ -97,6 +97,15 @@ describe('WolfLexer', function() {
 			tokens.length.should.be.equal(2);
 		});
 
+        it('should rethrow exception from the callback function', function() {
+            var lexer = new WolfLexer();
+            lexer.addRule(patternWord, kindWord);
+            lexer.addRule(patternWhiteSpace, kindSpace);
+            (function() {lexer.scan(inputTwoDashes, function(t) {
+                throw new Error('Exception from token callback.');
+            }, function(e) {})}).should.throw();
+        });
+
 		it('should not stop on error.', function() {
 		    var lexer = new WolfLexer();
 		    lexer.resumeOnError = true;
@@ -165,4 +174,12 @@ describe('WolfLexer', function() {
 		    lexer.rules.length.should.be.equal(0);
 		});
 	});
-});
+
+    describe('#inStepMode', function() {
+        it('may accept wth no callback to work in step mode.', function () {
+            var lexer = new WolfLexer();
+            lexer.addRule(patternWord, kindWord);
+            lexer.scan('hello world');
+        });
+    });
+});