Permalink
Browse files

[api] Added comment and tag support to the lexer

  • Loading branch information...
1 parent 0eb8687 commit 276166f5f745f28a740703dd004e0396dd02f98b @indexzero indexzero committed Sep 2, 2010
Showing with 27 additions and 20 deletions.
  1. +2 −0 examples/simple.feature
  2. +23 −18 lib/kyuri/lexer.js
  3. +2 −2 test/lexer-test.js
@@ -3,6 +3,8 @@ Feature: Addition
As a math idiot
I want to be told the sum of two numbers
+ @tag1
+ #AND A COMMENT!!!
Scenario: Add two numbers
"""
Hey I'm a pystring, check me out now
View
@@ -15,7 +15,7 @@ var MULTI_DENT = /^(\t+)(\.)?/,
PARSE_EXAMPLE_ROW = /\|\s*(\S+)/gi,
PYSTRING = /"""/,
TAG = /@(\w+)/i,
- COMMENT = /#([\S+\s+]+)/i,
+ COMMENT = /#\s*([\S+\s+]+)/i,
SENTENCE = /([\S+\s+]+)/i;
var Lexer = function (language, i18n) {
@@ -93,7 +93,8 @@ Lexer.prototype = {
extractNextToken: function () {
var tokens = ['lineToken', 'commentToken', 'featureToken', 'scenarioToken', 'scenarioOutlineToken',
- 'backgroundToken', 'examplesToken', 'examplesRowToken', 'operatorToken', 'pystringToken', 'sentenceToken'];
+ 'backgroundToken', 'examplesToken', 'examplesRowToken', 'operatorToken', 'pystringToken',
+ 'tagToken', 'sentenceToken'];
for (var index in tokens) {
if (this[tokens[index]].apply(this)) {
@@ -167,28 +168,15 @@ Lexer.prototype = {
},
pystringToken: function () {
- var match;
- if (!(match = this.match(PYSTRING))) {
- return false;
- }
-
- eyes.inspect(match);
- eyes.inspect(match[0].length);
-
- this.i += match[0].length;
- this.token('PYSTRING', 'PYSTRING');
-
- return true;
+ return this.complexMatch(PYSTRING, 'PYSTRING', 'PYSTRING');
},
tagToken: function () {
- // TODO: Implement comments [http://github.com/nodejitsu/kyuri/issues/5]
- return false;
+ return this.complexMatch(TAG, 'TAG');
},
commentToken: function() {
- // TODO: Implement comments [http://github.com/nodejitsu/kyuri/issues/6]
- return false;
+ return this.complexMatch(COMMENT, 'COMMENT');
},
sentenceToken: function () {
@@ -251,6 +239,23 @@ Lexer.prototype = {
return tok[0];
},
+ complexMatch: function (regex, token, value) {
+ var match;
+ if (!(match = this.match(regex))) {
+ return false;
+ }
+
+ this.i += match[0].length;
+
+ if (value) {
+ this.token(token, value);
+ return true;
+ }
+
+ this.token(token, match[0]);
+ return true;
+ },
+
match: function(regex, index) {
var m;
if (!(m = this.chunk.match(regex))) {
View
@@ -43,13 +43,13 @@ vows.describe('kyuri/lexer').addBatch({
assert.isNotNull(data.toString());
inspect(kyuri.tokens(data.toString()));
}
- }/*,
+ },
"lexing complex.feature": {
topic: readAllLines(path.join(__dirname, '..', 'examples', 'complex.feature')),
"should lex correctly": function (err, data) {
assert.isNotNull(data.toString());
inspect(kyuri.tokens(data.toString()));
}
- }*/
+ }
}
}).export(module);

0 comments on commit 276166f

Please sign in to comment.