Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

add possibility to ignore tokens

  • Loading branch information...
commit 945944cf2233e616dc5981a0454ffaf6d5034273 1 parent 57c452f
@Floby authored
Showing with 13 additions and 0 deletions.
  1. +13 −0 lib/Tokenizer.js
View
13 lib/Tokenizer.js
@@ -9,6 +9,8 @@ function Tokenizer (check_token_cb) {
this._buffered = ""; // we buffer untokenized data between writes
this._regexes = []; // should contain objects
// with regex[RegExp] and type[String]
+ this._ignored = {} // a hash of ignored token types
+ // these will be parsed but not emitted
this._checkToken = check_token_cb || noop;
}
sys.inherits(Tokenizer, EventEmitter);
@@ -77,6 +79,7 @@ Tokenizer.prototype.end = function end(data) {
Tokenizer.prototype._gotToken = function _gotToken(str, rule) {
// notify the token checker
var type = this._checkToken(str, rule) || rule.type;
+ if(this._ignored[type]) return;
this.emit('data', str, type); // act like a readable stream
this.emit('token', str, type);
};
@@ -91,6 +94,16 @@ Tokenizer.prototype.addRule = function addRule(regex, type) {
this._regexes.push({regex:regex,type:type});
};
+Tokenizer.prototype.ignore = function ignore(ignored) {
+ if(typeof ignore === 'array') {
+ for (var i = 0; i < ignored.length; ++i) {
+ this.ignore(ignored[i]);
+ };
+ return;
+ }
+ this._ignored[ignored] = true;
+};
+
module.exports = Tokenizer;
// built-in rules
Please sign in to comment.
Something went wrong with that request. Please try again.