Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

better token

  • Loading branch information...
commit abd6938e5d64fa802f1c04e0e8b60ea6bdd39317 1 parent 7d99cb9
Elmer Bulthuis authored
54  index.js
@@ -61,7 +61,7 @@ module.exports = function(tokenCallback, expressionSet, options) {
61 61
 	/**
62 62
 	matches in the current buffer. Serves as a cache.
63 63
 	*/
64  
-	var matchSet = {};
  64
+	var tokenSet = {};
65 65
 
66 66
 	/**
67 67
 	What categories should we look for?
@@ -95,7 +95,7 @@ module.exports = function(tokenCallback, expressionSet, options) {
95 95
 		flush(0);
96 96
 	    tokenCallback.call(tokenizer, null, buffer);
97 97
 	    buffer = '';
98  
-	    matchSet = {};
  98
+	    tokenSet = {};
99 99
 	}
100 100
 	
101 101
 	/**
@@ -108,7 +108,7 @@ module.exports = function(tokenCallback, expressionSet, options) {
108 108
 			/*
109 109
 			the offset to which we are going to flush.
110 110
 			*/
111  
-        	var offset = token.match.index + token.match[0].length;
  111
+        	var offset = token.index + token[0].length;
112 112
 			
113 113
 			/*
114 114
 			trim the buffer.
@@ -118,20 +118,20 @@ module.exports = function(tokenCallback, expressionSet, options) {
118 118
 			/*
119 119
 			sync the cache with the buffer.
120 120
 			*/
121  
-			for(var category in matchSet)	{
122  
-	            var match = matchSet[category];
  121
+			for(var category in tokenSet)	{
  122
+	            var token = tokenSet[category];
123 123
 				/*
124 124
 				when the index of the cached match is before the
125 125
 				offset, it is trimmed off! so remove it from the
126 126
 				cache
127 127
 				*/
128  
-	            if(match.index < offset) delete matchSet[category];
  128
+	            if(token.index < offset) delete tokenSet[category];
129 129
 	            /*
130 130
 	            if the cached match is not before the offset, it is
131 131
 	            still in the buffer but if moved a litte to the
132 132
 	            beginning. So adjust the index.
133 133
 	            */
134  
-	            else match.index -= offset;
  134
+	            else token.index -= offset;
135 135
 			}
136 136
 			
137 137
 		}
@@ -141,17 +141,17 @@ module.exports = function(tokenCallback, expressionSet, options) {
141 141
 	finds the next token in the buffer.
142 142
 	*/
143 143
     function nextToken() {
144  
-        var token = null;
  144
+		var foundToken = null;
145 145
 		categoryList.forEach(function(category) {
146 146
             var expression = expressionSet[category];
147 147
 			/*
148  
-			look for a cached match
  148
+			look for a cached token
149 149
 			*/
150  
-            var match = matchSet[category];
  150
+           	var token = tokenSet[category];
151 151
 			/*
152  
-			if there is no cached match
  152
+			if there is no cached token
153 153
 			*/
154  
-            if(!match)	{
  154
+            if(!token)	{
155 155
             	/*
156 156
             	if expression is a string we are just going
157 157
             	to look for the string.
@@ -160,47 +160,47 @@ module.exports = function(tokenCallback, expressionSet, options) {
160 160
 	            	var index = buffer.indexOf(expression);
161 161
 	            	/*
162 162
 	            	if we found the string (remember, ~-1 == 0) then
163  
-	            	mimic the match object returned by RegExp
  163
+	            	create a token object
164 164
 	            	*/
165 165
 	            	if(~index)	{
166  
-	            		match = extend([
  166
+	            		token = extend([
167 167
 	            			expression
168 168
 	            		], {
169 169
 	            			index:	index
  170
+	            			, category:	category
170 171
 	            		});
171 172
 		           	}
172 173
             	}
173 174
         		/*
174  
-        		if it's not a string, it should be a RegExp. Just execute
175  
-        		it.
  175
+        		if it's not a string, it must be a RegExp. Execute it
  176
+        		and make it a token
176 177
         		*/
177 178
             	else	{
178  
-	            	match = expression.exec(buffer);
  179
+	            	var match = expression.exec(buffer);
  180
+	            	if(match)	{
  181
+	            		token = extend(match, {
  182
+	            			category:	category
  183
+	            		});
  184
+	            	}
179 185
             	}
180 186
             	/*
181 187
             	if there is a match, cache it!
182 188
             	*/
183  
-            	if(match) matchSet[category] = match;
  189
+            	if(token) tokenSet[category] = token;
184 190
             }
185 191
             
186 192
 			/*
187 193
 			if there is no token or, if there is a token and it is before
188 194
 			the current token.
189 195
 			*/
190  
-            if (match && (!token || match.index < token.match.index)) {
191  
-				/*
192  
-				define a new token.
193  
-				*/
194  
-                token = {
195  
-                    category: category
196  
-                    , match: match
197  
-                };
  196
+            if (token && (!foundToken || token.index < foundToken.index)) {
  197
+                foundToken = token;
198 198
             }
199 199
 		});
200 200
 		/*
201 201
 		if there is no token found, this will return null
202 202
 		*/
203  
-        return token;
  203
+        return foundToken;
204 204
     }
205 205
     
206 206
     /**
2  package.json
... ...
@@ -1,7 +1,7 @@
1 1
 {
2 2
     "name":	"2kenizer"
3 3
     , "description":	"efficient tokenizer"
4  
-    , "version":	"0.0.1"
  4
+    , "version":	"0.0.2"
5 5
     , "author":	"Elmer Bulthuis <elmerbulthuis@gmail.com>"
6 6
     , "repositories":	[{
7 7
         "type": "git",
18  test/html.html
... ...
@@ -1,18 +0,0 @@
1  
-<!DOCTYPE html>
2  
-<html>
3  
-<head>
4  
-<title></title>
5  
-</head>
6  
-<body>
7  
-
8  
-<p>
9  
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed nec sapien varius urna aliquet interdum. Nulla ultricies magna ac purus tincidunt vitae accumsan risus feugiat. Pellentesque lobortis viverra volutpat. Nullam mattis mollis lorem, tincidunt tempor massa rhoncus non. Integer luctus felis at ante scelerisque sed fermentum dolor volutpat. In ut ante at arcu rutrum viverra at non lectus. Aliquam erat volutpat. Curabitur nec risus vel dolor eleifend semper quis ut nisl. Proin neque nibh, malesuada et adipiscing eget, imperdiet sed risus. Curabitur malesuada ullamcorper tortor, quis pulvinar tellus pellentesque eget. Curabitur non sem neque, ut adipiscing odio. Donec sollicitudin feugiat lectus, eu porttitor urna faucibus id. In interdum pharetra semper. Cras in nisi accumsan dui auctor rhoncus.
10  
-</p>
11  
-
12  
-<p>
13  
-Morbi auctor elit nec libero pretium eleifend non non diam. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Fusce accumsan vehicula neque et bibendum. Maecenas laoreet arcu sit amet elit convallis a dapibus turpis tincidunt. Curabitur quis ligula vel orci varius pretium. Sed imperdiet scelerisque arcu vitae placerat. Aenean facilisis nisl sed risus consectetur ac congue diam hendrerit. Fusce imperdiet turpis sapien. Sed erat diam, accumsan sed congue ac, varius eu massa. Sed consequat mauris nec neque fringilla rhoncus. Nam id imperdiet justo. Ut in est non enim ornare varius quis vel sem. Nunc commodo elit dignissim felis elementum mollis.
14  
-</p>
15  
-
16  
-</body>
17  
-</html>
18  
-
4  test/js.js
@@ -6,7 +6,7 @@ function parse(data)	{
6 6
 	var tokenizer = new Tokenizer(function(token, buffer) {
7 7
 		if(!token) return;
8 8
 
9  
-		//console.log(token.category, token.match[0]);
  9
+		//console.log(token.category, token[0]);
10 10
 
11 11
 		if(token.category in tokenActions) tokenActions[token.category](token, buffer);
12 12
 		else throw "invalid category '" + token.category + "' in this context";
@@ -114,7 +114,7 @@ function testFile(filePath, options)	{
114 114
 		return;
115 115
 	}
116 116
 
117  
-	//console.log('[' + match[3] + ']');
  117
+	console.log('[' + match[3] + ']');
118 118
 	try	{
119 119
 		parse(fs.readFileSync(match[0], 'utf-8'));
120 120
 	}
2  test/switching.js
@@ -7,7 +7,7 @@ var tokenizer = new Tokenizer(function(token, buffer) {
7 7
 
8 8
 	switch(token.category)	{
9 9
 		case 'A':
10  
-		if(token.match[0] == 'yz')	{
  10
+		if(token[0] == 'yz')	{
11 11
 			tokenizer.filter('0');
12 12
 		}
13 13
 		break;
12  test/tag.js
@@ -11,11 +11,11 @@ function parse(data)	{
11 11
 	var tokenizer = new Tokenizer(function(token, buffer) {
12 12
 		if(!token) return;
13 13
 
14  
-		for(var index = buffer.indexOf(newline); ~index && index < token.match.index; index = buffer.indexOf(newline, index + newline.length))	{
  14
+		for(var index = buffer.indexOf(newline); ~index && index < token.index; index = buffer.indexOf(newline, index + newline.length))	{
15 15
 	line++;
16 16
 		}
17 17
 
18  
-	//console.log(token.match[0], token.category);
  18
+	//console.log(token[0], token.category);
19 19
 
20 20
 		if(token.category in tokenActions) tokenActions[token.category](token, buffer);
21 21
 		else throw "invalid category '" + token.category + "' in this context";
@@ -50,14 +50,14 @@ function parse(data)	{
50 50
 		, "tag":	function(token, buffer)	{
51 51
 			enterContext({
52 52
 				category:	token.category
53  
-				, tag:	token.match[1]
  53
+				, tag:	token[1]
54 54
 				, filter:	["tag1"]
55 55
 			});
56 56
 		}
57 57
 		, "tag1":	function(token, buffer)	{
58 58
 			var tag = currentContext.tag;
59 59
 			exitContext();
60  
-			if(token.match[1] != "/" && !~voidTags.indexOf(tag.toLowerCase()))	{
  60
+			if(token[1] != "/" && !~voidTags.indexOf(tag.toLowerCase()))	{
61 61
 				enterContext({
62 62
 					category:	token.category
63 63
 					, tag:	tag
@@ -68,7 +68,7 @@ function parse(data)	{
68 68
 		, "tag2":	function(token, buffer)	{
69 69
 			var tag = currentContext.tag;
70 70
 			
71  
-			assert.equal(tag, token.match[1], "<" + tag + "> at line " + line + " should be closed before closing <" + token.match[1] + ">");
  71
+			assert.equal(tag, token[1], "<" + tag + "> at line " + line + " should be closed before closing <" + token[1] + ">");
72 72
 			exitContext();
73 73
 		}
74 74
 
@@ -104,7 +104,7 @@ function testFile(filePath, options)	{
104 104
 		return;
105 105
 	}
106 106
 
107  
-	//console.log('[' + match[3] + ']');
  107
+	console.log('[' + match[3] + ']');
108 108
 	try	{
109 109
 		parse(fs.readFileSync(match[0], 'utf-8'));
110 110
 	}

0 notes on commit abd6938

Please sign in to comment.
Something went wrong with that request. Please try again.