Permalink
Browse files

Merge from smtlaissezfaire at http://github.com/smtlaissezfaire/jslex…

  • Loading branch information...
2 parents 5716875 + f3240e8 commit 084decd175c53e689fc3bdaeb3015cabf58d20ca @ekiru committed Apr 10, 2010
Showing with 122 additions and 109 deletions.
  1. +3 −4 README
  2. +47 −37 jslex.js
  3. +72 −68 jslex_test.js
View
@@ -8,7 +8,6 @@ The TokenDef constructor takes two arguments: a regular expression and a functio
For example, this could be used to recognize a natural number:
- new TokenDef(/[0-9]+/, function (string) {
- return ["int", parseInt(string)];
- })
-
+new TokenDef(/[0-9]+/, function (string) {
+ return ["int", parseInt(string)];
+})
View
@@ -24,54 +24,64 @@
*/
function TokenDef(pattern, callbackfn) {
- this.match = function match(text) {
- var trimmed = text.trimBegin();
- var match = trimmed.match(pattern);
- if (match && match.index == 0) {
- var newText = trimmed.substr(match[0].length);
- var token = callbackfn(match[0]);
- return { success: true,
- token: token,
- text: newText};
- } else {
- return { success: false };
- }
- };
+ this.match = function match(text) {
+ var trimmed = text.trim();
+ var match = trimmed.match(pattern);
+ if (match && match.index == 0) {
+ var newText = trimmed.substr(match[0].length);
+ var token = callbackfn(match[0]);
+
+ return {
+ success: true,
+ token: token,
+ text: newText
+ };
+ } else {
+ return { success: false };
+ }
+ };
}
TokenDef.ignore = function (string) {
- return null;
+ return null;
};
TokenDef.always = function (token) {
- return function (string) {
- return token;
- };
+ return function (string) {
+ return token;
+ };
};
TokenDef.identity = function (string) {
- return string;
-}
+ return string;
+};
function jsLex(tokenDefs, text) {
- text = text.trim();
- var tokenStream = [];
- var tokenCount = tokenDefs.length;
- while (text.length > 0) {
- var matchedAny = false;
- for (var i = 0; i < tokenCount; i++) {
- var tokenType = tokenDefs[i];
- var match = tokenType.match(text);
- if (match.success) {
- match.token && tokenStream.push(match.token);
- text = match.text.trim();
- matchedAny = true;
- }
- }
- if (!matchedAny) {
- throw Error("Can't tokenize string " + text);
- }
+ text = text.trim();
+ var tokenStream = [];
+ var tokenCount = tokenDefs.length;
+
+ while (text.length > 0) {
+ var matchedAny = false;
+ for (var i = 0; i < tokenCount; i++) {
+ var tokenType = tokenDefs[i];
+ var match = tokenType.match(text);
+
+ if (match.success) {
+ match.token && tokenStream.push(match.token);
+ text = match.text.trim();
+ matchedAny = true;
+ }
+ }
+ if (!matchedAny) {
+ throw Error("Can't tokenize string " + text);
}
+ }
+
+ return tokenStream;
+}
- return tokenStream;
+if (typeof(exports) !== "undefined") {
+ exports.TokenDef = TokenDef;
+ exports.jsLex = jsLex;
}
View
@@ -22,99 +22,103 @@
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
-*/
-load("jslex.js");
+*/;
+
+var lexer = require("./jslex");
+
+var TokenDef = lexer.TokenDef;
+var jsLex = lexer.jsLex;
function testTokenMatch() {
- var result = true;
-
- var matchResult = new TokenDef(/[0-9]/, parseInt).match("123");
- result = matchResult.success && result;
- result = matchResult.token == 1 && result;
- result = matchResult.text == "23" && result;
-
- matchResult = new TokenDef(/[0-9]/, parseInt).match("a123");
- result = !matchResult.success && result;
-
- matchResult = new TokenDef(/[0-9]+/, parseInt).match("12,");
- result = matchResult.success && result;
- result = matchResult.token == 12 && result;
- result = matchResult.text == "," && result;
-
- return result;
+ var result = true;
+
+ var matchResult = new TokenDef(/[0-9]/, parseInt).match("123");
+ result = matchResult.success && result;
+ result = matchResult.token == 1 && result;
+ result = matchResult.text == "23" && result;
+
+ matchResult = new TokenDef(/[0-9]/, parseInt).match("a123");
+ result = !matchResult.success && result;
+
+ matchResult = new TokenDef(/[0-9]+/, parseInt).match("12,");
+ result = matchResult.success && result;
+ result = matchResult.token == 12 && result;
+ result = matchResult.text == "," && result;
+
+ return result;
}
function testTokenDef() {
- var result = true;
+ var result = true;
- result = testTokenMatch() && result;
+ result = testTokenMatch() && result;
- if (result) {
- print("Token definition tests succeeded.");
- } else {
- print("Token definition tests failed.");
- }
- return result;
+ if (result) {
+ print("Token definition tests succeeded.");
+ } else {
+ print("Token definition tests failed.");
+ }
+ return result;
}
function testIntLexer() {
- var result = true;
+ var result = true;
+
+ var tokenRules = [new TokenDef(/[0-9]+/, parseInt)];
+ var tokens = jsLex(tokenRules, "123 456 789");
- var tokenRules = [new TokenDef(/[0-9]+/, parseInt)];
- var tokens = jsLex(tokenRules, "123 456 789");
+ result = tokens[0] == 123 && result;
+ result = tokens[1] == 456 && result;
+ result = tokens[2] == 789 && result;
- result = tokens[0] == 123 && result;
- result = tokens[1] == 456 && result;
- result = tokens[2] == 789 && result;
-
- return result;
+ return result;
}
function testIntOrIdentifierLexer() {
- var result = true;
- var tokenRules = [
- new TokenDef(/[0-9]+/, function (string) {
- return ["int", parseInt(string)];
- }),
- new TokenDef(/[a-zA-Z][a-zA-Z0-9_$]*/, function (string) {
- return ["ident", string];
- })
- ];
- var tokens = jsLex(tokenRules, "123 plus P4");
-
- result = tokens[0][0] == "int" && tokens[0][1] == 123 && result;
- result = tokens[1][0] == "ident" && tokens[1][1] == "plus" && result;
- result = tokens[2][0] == "ident" && tokens[2][1] == "P4" && result;
-
- return result;
+ var result = true;
+ var tokenRules = [
+ new TokenDef(/[0-9]+/, function (string) {
+ return ["int", parseInt(string, "10")];
+ }),
+ new TokenDef(/[a-zA-Z][a-zA-Z0-9_$]*/, function (string) {
+ return ["ident", string];
+ })
+ ];
+ var tokens = jsLex(tokenRules, "123 plus P4");
+
+ result = tokens[0][0] == "int" && tokens[0][1] == 123 && result;
+ result = tokens[1][0] == "ident" && tokens[1][1] == "plus" && result;
+ result = tokens[2][0] == "ident" && tokens[2][1] == "P4" && result;
+
+ return result;
}
function testLexer() {
- var result = true;
+ var result = true;
- result = testIntLexer() && result;
- result = testIntOrIdentifierLexer() && result;
+ result = testIntLexer() && result;
+ result = testIntOrIdentifierLexer() && result;
- if (result) {
- print("Lexer tests succeeded.");
- } else {
- print("Lexer tests failed.");
- }
- return result;
+ if (result) {
+ print("Lexer tests succeeded.");
+ } else {
+ print("Lexer tests failed.");
+ }
+ return result;
}
function testAll() {
- var result = true;
+ var result = true;
- result = testTokenDef() && result;
- result = testLexer() && result;
+ result = testTokenDef() && result;
+ result = testLexer() && result;
- if (result) {
- print("All tests succeeded.");
- } else {
- print("Some tests failed.");
- }
- return result;
+ if (result) {
+ print("All tests succeeded.");
+ } else {
+ print("Some tests failed.");
+ }
+ return result;
}
testAll();

0 comments on commit 084decd

Please sign in to comment.