Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

add built-in rules for common use cases

A rule is an array containing the arguments required for
addRule and are members of the Tokenizer constructor
  • Loading branch information...
commit 57c452f467099cdb7a663f2447e10a1774afac79 1 parent cba1c87
@Floby authored
Showing with 11 additions and 1 deletion.
  1. +11 −1 lib/Tokenizer.js
View
12 lib/Tokenizer.js
@@ -82,8 +82,18 @@ Tokenizer.prototype._gotToken = function _gotToken(str, rule) {
};
Tokenizer.prototype.addRule = function addRule(regex, type) {
- // TODO: check types
+ // this is useful for built-in rules
+ if(typeof regex === 'array') {
+ return this.addRule(regex[0], regex[1]);
+ }
+ assert.ok(regex instanceof RegExp);
+ assert.equal(typeof type, 'string');
this._regexes.push({regex:regex,type:type});
};
module.exports = Tokenizer;
+
+// built-in rules
+Tokenizer.whitespace = [/(\s)+/, 'whitespace'];
+Tokenizer.word = [/\w+/, 'word'];
+Tokenizer.number = [/\d+/, 'number'];
Please sign in to comment.
Something went wrong with that request. Please try again.