Permalink
Browse files

Add new token scanner that will be the refactored version. As of righ…

…t now, it is actually slower though. But much more readable, at least!
  • Loading branch information...
1 parent 4eb1d4f commit 63afc0ebc0dfcccc17dcb3f466f12b6bb2e88434 @olabini committed Jul 23, 2009
View
@@ -41,6 +41,11 @@
</exec>
<exec executable="re2j" failonerror="true">
+ <arg line="-o ${main.src.dir}/org/yecht/TokenScanner2.java"/>
+ <arg line="${main.src.dir}/org/yecht/TokenScanner2.re"/>
+ </exec>
+
+ <exec executable="re2j" failonerror="true">
<arg line="-o ${main.src.dir}/org/yecht/ImplicitScanner.java"/>
<arg line="${main.src.dir}/org/yecht/ImplicitScanner.re"/>
</exec>
Oops, something went wrong.
Oops, something went wrong.
@@ -35,27 +35,27 @@ public static void main(String[] args) throws Exception {
tok = s.yylex();
if(tok == YAMLGrammarTokens.YAML_IOPEN) {
for(int i=0; i < indent; i++) {
- System.err.print(" ");
+ System.out.print(" ");
}
indent++;
} else if(tok == YAMLGrammarTokens.YAML_IEND) {
indent--;
for(int i=0; i < indent; i++) {
- System.err.print(" ");
+ System.out.print(" ");
}
} else {
for(int i=0; i < indent; i++) {
- System.err.print(" ");
+ System.out.print(" ");
}
}
Object lval2 = s.getLVal();
- System.err.print("tok: " + TokenScanner.tnames[tok]);
+ System.out.print("tok: " + TokenScanner.tnames[tok]);
if(lval != lval2) {
- System.err.print(" lval: " + lval2);
+ System.out.print(" lval: " + lval2);
lval = lval2;
}
- System.err.println();
+ System.out.println();
}
}
}
@@ -0,0 +1,61 @@
+package org.yecht;
+
+import java.io.InputStream;
+import java.io.FileInputStream;
+
+public class ScannerOutput2 {
+ public static void main(String[] args) throws Exception {
+ String filename = args[0];
+ int len = 8000;
+ int read = 0;
+ int currRead = 0;
+ byte[] buffer = new byte[1024];
+ byte[] input = new byte[len];
+ InputStream is = new FileInputStream(filename);
+ while((currRead = is.read(buffer, 0, 1024)) != -1) {
+ if(read + currRead >= len) {
+ len *= 2;
+ input = YAML.realloc(input, len);
+ }
+ System.arraycopy(buffer, 0, input, read, currRead);
+ read += currRead;
+ }
+
+ Parser parser = Parser.newParser();
+ parser.str(Pointer.create(input, 0), read, null);
+ parser.handler(new NullNodeHandler());
+ parser.errorHandler(null);
+ parser.implicitTyping(true);
+ parser.taguriExpansion(true);
+ Scanner s = TokenScanner2.createScanner(parser);
+ int tok = -1;
+ Object lval = null;
+ int indent = 0;
+ while(tok != YAMLGrammarTokens.ENDINPUT) {
+ tok = s.yylex();
+ if(tok == YAMLGrammarTokens.YAML_IOPEN) {
+ for(int i=0; i < indent; i++) {
+ System.out.print(" ");
+ }
+ indent++;
+ } else if(tok == YAMLGrammarTokens.YAML_IEND) {
+ indent--;
+ for(int i=0; i < indent; i++) {
+ System.out.print(" ");
+ }
+ } else {
+ for(int i=0; i < indent; i++) {
+ System.out.print(" ");
+ }
+ }
+
+ Object lval2 = s.getLVal();
+ System.out.print("tok: " + TokenScanner2.tnames[tok]);
+ if(lval != lval2) {
+ System.out.print(" lval: " + lval2);
+ lval = lval2;
+ }
+ System.out.println();
+ }
+ }
+}
@@ -20,7 +20,7 @@ public static void main(String[] args) throws Exception {
System.arraycopy(buffer, 0, input, read, currRead);
read += currRead;
}
- int times = 100;
+ int times = 10000;
long before = System.currentTimeMillis();
for(int i=0; i<times; i++) {
Parser parser = Parser.newParser();
Oops, something went wrong.

0 comments on commit 63afc0e

Please sign in to comment.