Returned <null> on end of source while using step mode.

benoybose · Dec 25, 2016 · 4b6ffed · 4b6ffed
1 parent 3d34302
commit 4b6ffed
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 17 deletions.
diff --git a/lib/wolf-lexer.js b/lib/wolf-lexer.js
@@ -81,18 +81,18 @@ WolfLexer.prototype.scan = function(text, callback_token, callback_error) {
     }
 
     if (!callback_token) {
-        var Stepper = function (t,l) {
+        var Stepper = function(t, l) {
             this.text = t;
             this.lexer = l;
             this.position = 0;
             this.errors = [];
         };
 
-        Stepper.prototype.hasErrors = function () {
+        Stepper.prototype.hasErrors = function() {
             return (0 < this.errors.length);
         };
 
-        Stepper.prototype.next = function (skipWhiteSpace) {
+        Stepper.prototype.next = function(skipWhiteSpace) {
             if (typeof(skipWhiteSpace) === 'undefined') {
                 skipWhiteSpace = true;
             }
@@ -106,21 +106,22 @@ WolfLexer.prototype.scan = function(text, callback_token, callback_error) {
                     token = this.next(skipWhiteSpace);
                 }
             } else {
-                if (!this.lexer.resumeOnError) {
-                    throw new Error('Invalid character is found at ' + this.position);
-                } else {
-                    this.errors.push('Invalid character is found at ' + this.position);
-                    while (true) {
-                        this.text = this.text.substr(1);
-                        if (!this.text) {
-                            break;
-                        }
-                        this.position += 1;
-                        token = this.next(skipWhiteSpace);
-                        if (token) {
-                            return token;
+                if (0 < this.text.length) {
+                    if (!this.lexer.resumeOnError) {
+                        throw new Error('Invalid character is found at ' + this.position);
+                    } else {
+                        this.errors.push('Invalid character is found at ' + this.position);
+                        while (true) {
+                            this.text = this.text.substr(1);
+                            this.position += 1;
+                            token = this.next(skipWhiteSpace);
+                            if (token) {
+                                return token;
+                            }
                         }
                     }
+                } else {
+                    return null;
                 }
             }
             return token;

diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "wolf-lexer",
   "description": "Yet another lexer module in nodejs.",
-  "version": "0.0.7",
+  "version": "0.0.8",
   "homepage": "https://github.com/benoybose/wolf-lexer",
   "bugs": "https://github.com/benoybose/wolf-lexer/issues",
   "license": "MIT",

diff --git a/test/wolf-lexer_test.js b/test/wolf-lexer_test.js
@@ -195,6 +195,21 @@ describe('WolfLexer', function() {
             t2.kind.should.be.eql('word');
         });
 
+				it('should return <null> on end of file or source', function() {
+					var lexer = new WolfLexer();
+					lexer.addRule(patternWord, kindWord);
+					var handler = lexer.scan('hello world');
+					handler.text.should.be.eql('hello world');
+					handler.lexer.should.be.eql(lexer);
+
+					var t1 = handler.next();
+					var t2 = handler.next();
+					t1.should.not.equal(null);
+					t2.should.not.equal(null);
+					var t3 = handler.next();
+					assert.equal(t3, null);
+				});
+
         it('should throw exception on encoutering invalid characters', function () {
             var lexer = new WolfLexer();
             lexer.addRule(patternWord, kindWord);