Skip to content
Browse files

initial commit

  • Loading branch information...
0 parents commit 4dc50f3c2891ed86ce0f5bb157bc81f20ff6374b @nickdesaulniers committed
Showing with 57 additions and 0 deletions.
  1. +2 −0 README.md
  2. +2 −0 nick.n
  3. +46 −0 parser.js
  4. +7 −0 test.js
2 README.md
@@ -0,0 +1,2 @@
+# tokenizer #
+A simple tokenizer written in javascript for Node.JS
2 nick.n
@@ -0,0 +1,2 @@
+fn hello_world
+ puts 'hello world'
46 parser.js
@@ -0,0 +1,46 @@
+var regex_queue = [];
+var tokens = [];
+var source = '';
+var regexQueue = [];
+
+function logSrc () {
+ console.log(source);
+}
+
+function logTok () {
+ console.log('Tokens: ' + tokens);
+}
+
+exports.rule = function (tokenType, re) {
+ regexQueue.push(function () {
+ var ret = false;
+ var result = re.exec(source);
+ if (result) {
+ console.log(tokenType + ' token: ' + result[0]);
+ tokens.push(result[0]);
+ source = source.substring(result[0].length);
+ ret = true;
+ }
+ return ret;
+ });
+}
+
+exports.parse = function (src) {
+ source = src;
+ console.log('-- Starting tokenizer --');
+ logSrc();
+ console.log('-- --');
+
+ while (source) {
+ var foundToken = regexQueue.some(function (element, index, array) {
+ return element();
+ });
+ if (!foundToken) {
+ console.error('Did not find any tokens on this pass:');
+ logSrc();
+ process.exit(1);
+ }
+ }
+
+ console.log('-- Tokenizing complete --\n[' + tokens + ']\n-- --');
+}
7 test.js
@@ -0,0 +1,7 @@
+var fs = require('fs');
+var parser = require('./parser');
+
+parser.rule('newline', /^\n/);
+parser.rule('whitespace', /^\s+/);
+parser.rule('word', /^[^\s]+/);
+parser.parse(fs.readFileSync('./nick.n', 'utf8'));

0 comments on commit 4dc50f3

Please sign in to comment.
Something went wrong with that request. Please try again.