Skip to content
This repository
Browse code

[api] Added comment and tag support to the lexer

  • Loading branch information...
commit 276166f5f745f28a740703dd004e0396dd02f98b 1 parent 0eb8687
Charlie Robbins indexzero authored
2  examples/simple.feature
@@ -3,6 +3,8 @@ Feature: Addition
3 3 As a math idiot
4 4 I want to be told the sum of two numbers
5 5
  6 + @tag1
  7 + #AND A COMMENT!!!
6 8 Scenario: Add two numbers
7 9 """
8 10 Hey I'm a pystring, check me out now
41 lib/kyuri/lexer.js
@@ -15,7 +15,7 @@ var MULTI_DENT = /^(\t+)(\.)?/,
15 15 PARSE_EXAMPLE_ROW = /\|\s*(\S+)/gi,
16 16 PYSTRING = /"""/,
17 17 TAG = /@(\w+)/i,
18   - COMMENT = /#([\S+\s+]+)/i,
  18 + COMMENT = /#\s*([\S+\s+]+)/i,
19 19 SENTENCE = /([\S+\s+]+)/i;
20 20
21 21 var Lexer = function (language, i18n) {
@@ -93,7 +93,8 @@ Lexer.prototype = {
93 93
94 94 extractNextToken: function () {
95 95 var tokens = ['lineToken', 'commentToken', 'featureToken', 'scenarioToken', 'scenarioOutlineToken',
96   - 'backgroundToken', 'examplesToken', 'examplesRowToken', 'operatorToken', 'pystringToken', 'sentenceToken'];
  96 + 'backgroundToken', 'examplesToken', 'examplesRowToken', 'operatorToken', 'pystringToken',
  97 + 'tagToken', 'sentenceToken'];
97 98
98 99 for (var index in tokens) {
99 100 if (this[tokens[index]].apply(this)) {
@@ -167,28 +168,15 @@ Lexer.prototype = {
167 168 },
168 169
169 170 pystringToken: function () {
170   - var match;
171   - if (!(match = this.match(PYSTRING))) {
172   - return false;
173   - }
174   -
175   - eyes.inspect(match);
176   - eyes.inspect(match[0].length);
177   -
178   - this.i += match[0].length;
179   - this.token('PYSTRING', 'PYSTRING');
180   -
181   - return true;
  171 + return this.complexMatch(PYSTRING, 'PYSTRING', 'PYSTRING');
182 172 },
183 173
184 174 tagToken: function () {
185   - // TODO: Implement comments [http://github.com/nodejitsu/kyuri/issues/5]
186   - return false;
  175 + return this.complexMatch(TAG, 'TAG');
187 176 },
188 177
189 178 commentToken: function() {
190   - // TODO: Implement comments [http://github.com/nodejitsu/kyuri/issues/6]
191   - return false;
  179 + return this.complexMatch(COMMENT, 'COMMENT');
192 180 },
193 181
194 182 sentenceToken: function () {
@@ -251,6 +239,23 @@ Lexer.prototype = {
251 239 return tok[0];
252 240 },
253 241
  242 + complexMatch: function (regex, token, value) {
  243 + var match;
  244 + if (!(match = this.match(regex))) {
  245 + return false;
  246 + }
  247 +
  248 + this.i += match[0].length;
  249 +
  250 + if (value) {
  251 + this.token(token, value);
  252 + return true;
  253 + }
  254 +
  255 + this.token(token, match[0]);
  256 + return true;
  257 + },
  258 +
254 259 match: function(regex, index) {
255 260 var m;
256 261 if (!(m = this.chunk.match(regex))) {
4 test/lexer-test.js
@@ -43,13 +43,13 @@ vows.describe('kyuri/lexer').addBatch({
43 43 assert.isNotNull(data.toString());
44 44 inspect(kyuri.tokens(data.toString()));
45 45 }
46   - }/*,
  46 + },
47 47 "lexing complex.feature": {
48 48 topic: readAllLines(path.join(__dirname, '..', 'examples', 'complex.feature')),
49 49 "should lex correctly": function (err, data) {
50 50 assert.isNotNull(data.toString());
51 51 inspect(kyuri.tokens(data.toString()));
52 52 }
53   - }*/
  53 + }
54 54 }
55 55 }).export(module);

0 comments on commit 276166f

Please sign in to comment.
Something went wrong with that request. Please try again.