[api] Improved comment handling in lexer

.gitignore

@@ -1 +1,2 @@
-test/output/*.js
+test/output/*.js
+test/beta-test.js

examples/simple.feature

@@ -1,4 +1,4 @@
-Feature: Addition
+Feature: Addition     #AND A COMMENT!!!
 	In order to avoid silly mistakes
 	As a math idiot
 	I want to be told the sum of two numbers

lib/kyuri/lexer.js

@@ -16,7 +16,8 @@ var MULTI_DENT        = /^(\t+)(\.)?/,
     PYSTRING          = /"""/,
     TAG               = /@(\w+)/i,
     COMMENT           = /#\s*([\S+\s+]+)/i,
-    SENTENCE          = /([\S+\s+]+)/i;
+    SENTENCE          = /([\S+\s+]+)/i,
+    SENTENCE_COMMENT  = /([\S+\s+]+)#\s*([\S+\s+]+)/i;
 
 var Lexer = function (language, i18n) {
   // Set the i18n dictionary on the lexer for later use
@@ -92,9 +93,9 @@ Lexer.prototype = {
   },
 
   extractNextToken: function () {
-    var tokens = ['lineToken', 'commentToken', 'featureToken', 'scenarioToken', 'scenarioOutlineToken', 
-    'backgroundToken', 'examplesToken', 'examplesRowToken', 'operatorToken', 'pystringToken', 
-    'tagToken', 'sentenceToken'];
+    var tokens = ['lineToken', 'featureToken', 'scenarioToken', 'scenarioOutlineToken', 
+    'backgroundToken', 'commentToken', 'examplesToken', 'examplesRowToken', 'operatorToken', 
+    'pystringToken', 'tagToken', 'sentenceToken'];
 
     for (var index in tokens) {
       if (this[tokens[index]].apply(this)) {
@@ -175,8 +176,29 @@ Lexer.prototype = {
     return this.complexMatch(TAG, 'TAG');
   },
 
-  commentToken: function() {
-    return this.complexMatch(COMMENT, 'COMMENT');
+  commentToken: function () {
+    var match;
+    if (!(match = this.match(COMMENT))) {
+      return false;
+    }
+
+    // If the comment is at the beginning of the chunk
+    // then evaluate it normally
+    if(this.chunk.indexOf('#') === 0) { 
+      this.i += match[0].length;
+      this.token('COMMENT', match[1].trim());
+      return true;
+    }
+
+    // Otherwise we have a comment inside of 
+    // a SENTENCE token, so strip off the SENTENCE
+    // and create tokens for both.
+    match = this.match(SENTENCE_COMMENT); 
+    this.i += match[0].length;
+    this.token('SENTENCE', match[1].trim());
+    this.token('COMMENT', match[2].trim());
+
+    return true;
   },
 
   sentenceToken: function () {
@@ -186,7 +208,7 @@ Lexer.prototype = {
     }
 
     this.i += sentence.length;
-    this.token('SENTENCE', sentence);
+    this.token('SENTENCE', sentence.trim());
 
     return true;
   },
@@ -252,7 +274,7 @@ Lexer.prototype = {
       return true;
     }
 
-    this.token(token, match[0]);
+    this.token(token, match[0].trim());
     return true;
   },