fix lexer & cover with tests

src/lexer.js

@@ -28,10 +28,11 @@ var lexerWhiteSpace = [' ', '\t', '\r', '\n'];
  */
 Lexer.prototype.read = function (input) {
   this._input = input;
-  this.line = 0;
+  this.line = 1;
   this.offset = 0;
   this.text = '';
   this.token = null;
+  this.backup = null;
 };
 
 /**
@@ -89,6 +90,7 @@ Lexer.prototype.unlex = function (state) {
     this.token = state.token;
     this.line = state.line;
   }
+  this.backup = null;
   return this.token;
 };
 

test/lexer.js

@@ -0,0 +1,103 @@
+/*!
+ * Copyright (C) 2017 Glayzzle (BSD3 License)
+ * @authors https://github.com/glayzzle/docblock-parser/graphs/contributors
+ * @url http://glayzzle.com/docblock-parser
+ */
+
+// eslint-disable-next-line no-unused-vars
+var should = require('should');
+var tokens = require('../src/token');
+var Lexer = require('../src/lexer');
+
+describe('Test lexer', function () {
+  var reader = new Lexer(tokens);
+  describe('Test windows lines', function () {
+    reader.read([
+      'hello',
+      'world'
+    ].join('\r\n'));
+
+    reader.lex().should.be.exactly(tokens.T_STRING);
+    reader.text.should.be.exactly('hello');
+    reader.line.should.be.exactly(1);
+
+    reader.next().should.be.exactly(tokens.T_WHITESPACE);
+    reader.text.should.be.exactly('\r\n');
+    reader.line.should.be.exactly(2);
+
+    reader.unput();
+    reader.line.should.be.exactly(1);
+
+    reader.lex().should.be.exactly(tokens.T_STRING);
+    reader.text.should.be.exactly('world');
+    reader.line.should.be.exactly(2);
+  });
+  describe('Test mac lines', function () {
+    reader.read([
+      'hello',
+      'world'
+    ].join('\r'));
+
+    reader.lex().should.be.exactly(tokens.T_STRING);
+    reader.text.should.be.exactly('hello');
+    reader.line.should.be.exactly(1);
+
+    reader.lex().should.be.exactly(tokens.T_STRING);
+    reader.text.should.be.exactly('world');
+    reader.line.should.be.exactly(2);
+  });
+  describe('Test unlex', function () {
+    reader.read([
+      'hello',
+      'world'
+    ].join('\n'));
+
+    // read
+    var start = reader.state();
+    reader.unlex();
+    reader.offset.should.be.exactly(0);
+
+    // next
+    reader.lex().should.be.exactly(tokens.T_STRING);
+    reader.lex().should.be.exactly(tokens.T_STRING);
+    reader.line.should.be.exactly(2);
+
+    // go back
+    reader.unlex(start);
+    reader.lex().should.be.exactly(tokens.T_STRING);
+    reader.line.should.be.exactly(1);
+    reader.text.should.be.exactly('hello');
+
+    // next
+    reader.lex().should.be.exactly(tokens.T_STRING);
+    reader.line.should.be.exactly(2);
+
+    // back
+    reader.unlex();
+    reader.line.should.be.exactly(1);
+    reader.text.should.be.exactly('hello');
+
+  });
+  describe('Test float', function () {
+    reader.read('1.2.3');
+    reader.lex().should.be.exactly(tokens.T_NUM);
+    reader.text.should.be.exactly('1.2');
+    reader.lex().should.be.exactly(tokens.T_NUM);
+    reader.text.should.be.exactly('.3');
+  });
+  describe('Test assign', function () {
+    reader.read('foo => a, bar: b');
+    reader.lex().should.be.exactly(tokens.T_STRING);
+    reader.lex().should.be.exactly('=>');
+    reader.lex().should.be.exactly(tokens.T_STRING);
+    reader.lex().should.be.exactly(',');
+    reader.lex().should.be.exactly(tokens.T_STRING);
+    reader.lex().should.be.exactly('=>');
+    reader.lex().should.be.exactly(tokens.T_STRING);
+  });
+  describe('Test text', function () {
+    reader.read('\'aze\\\'rty\'');
+    reader.lex().should.be.exactly(tokens.T_TEXT);
+    reader.text.should.be.exactly('\'aze\\\'rty\'');
+  });
+});