Permalink
Browse files

add built-in rules for common use cases

A rule is an array containing the arguments required for
addRule and are members of the Tokenizer constructor
  • Loading branch information...
1 parent cba1c87 commit 57c452f467099cdb7a663f2447e10a1774afac79 @Floby committed Apr 13, 2011
Showing with 11 additions and 1 deletion.
  1. +11 −1 lib/Tokenizer.js
View
@@ -82,8 +82,18 @@ Tokenizer.prototype._gotToken = function _gotToken(str, rule) {
};
Tokenizer.prototype.addRule = function addRule(regex, type) {
- // TODO: check types
+ // this is useful for built-in rules
+ if(typeof regex === 'array') {
+ return this.addRule(regex[0], regex[1]);
+ }
+ assert.ok(regex instanceof RegExp);
+ assert.equal(typeof type, 'string');
this._regexes.push({regex:regex,type:type});
};
module.exports = Tokenizer;
+
+// built-in rules
+Tokenizer.whitespace = [/(\s)+/, 'whitespace'];
+Tokenizer.word = [/\w+/, 'word'];
+Tokenizer.number = [/\d+/, 'number'];

0 comments on commit 57c452f

Please sign in to comment.