Permalink
Browse files

- Fixed tests package layout a bit.

- WIP: Tokenizer unit tests.
  • Loading branch information...
1 parent f82a05e commit c2dc24493c8995f8d44c4901df3b1736038808aa @alexarnon committed Sep 25, 2011
@@ -52,7 +52,7 @@ private Expression compile(Iterator<Token> it, boolean openBracketAlreadyConsume
throw new ParseException("unexpected EOF instead of open paren", -1);
}
Token openParen = it.next();
- if (! openParen.type.equals(Tokenizer.TokenType.OPEN_BRACKET)) {
+ if (! openParen.type.equals(Tokenizer.TokenType.OPEN_PAREN)) {
throw new ParseException("expected open paren instead of " + rep(openParen), -1);
}
}
@@ -77,6 +77,7 @@ private Expression compile(Iterator<Token> it, boolean openBracketAlreadyConsume
}
// TADAA.
+ throw new UnsupportedOperationException("unimplemented");
}
@@ -1,6 +1,7 @@
package com.tmp.uint.core.compilation;
import java.io.IOException;
+import java.io.Reader;
import java.io.StringReader;
import java.text.ParseException;
import java.util.ArrayList;
@@ -9,8 +10,8 @@
public class Tokenizer {
public static enum TokenType {
- OPEN_BRACKET,
- CLOSE_BRACKET,
+ OPEN_PAREN,
+ CLOSE_PAREN,
WORD,
STRING,
INTEGER,
@@ -35,15 +36,15 @@ public Token(TokenType type, Object value) {
}
- private final static Token open_bracket = new Token(TokenType.OPEN_BRACKET, "(");
- private final static Token close_bracket = new Token(TokenType.CLOSE_BRACKET, "(");
+ private final static Token open_paren = new Token(TokenType.OPEN_PAREN, "(");
+ private final static Token close_paren = new Token(TokenType.CLOSE_PAREN, "(");
- public static List<Token> tokenize(String input) throws IOException, ParseException {
+ public static List<Token> tokenize(String input) throws ParseException {
return tokenize(new StringReader(input));
}
- private static List<Token> tokenize(StringReader reader) throws IOException, ParseException {
+ private static List<Token> tokenize(StringReader reader) throws ParseException {
List<Token> tokens = new ArrayList<Token>();
State state = State.WS;
@@ -52,7 +53,7 @@ public Token(TokenType type, Object value) {
while (! State.END.equals(state)) {
- final int c = reader.read();
+ final int c = read(reader, pos);
final State nextState;
@@ -65,9 +66,9 @@ public Token(TokenType type, Object value) {
} else if (Character.isWhitespace(c)) {
nextState = state;
} else if (c == '(') {
- nextState = newState(open_bracket, tokens, state);
+ nextState = newState(open_paren, tokens, state);
} else if (c == ')') {
- nextState = newState(close_bracket, tokens, state);
+ nextState = newState(close_paren, tokens, state);
} else if (c == '"') {
nextState = newState(buf, State.STRING);
} else if (Character.isDigit(c)) {
@@ -90,10 +91,10 @@ public Token(TokenType type, Object value) {
nextState = state;
} else if (c == '(') {
nextState = newState(TokenType.WORD, tokens, buf, State.WS);
- tokens.add(open_bracket);
+ tokens.add(open_paren);
} else if (c == ')') {
nextState = newState(TokenType.WORD, tokens, buf, State.WS);
- tokens.add(close_bracket);
+ tokens.add(close_paren);
} else if (Character.isWhitespace(c)) {
nextState = newState(TokenType.WORD, tokens, buf, State.WS);
} else {
@@ -110,7 +111,7 @@ public Token(TokenType type, Object value) {
} else if (c == '"') {
nextState = newState(TokenType.STRING, tokens, buf, State.WS);
} else if (c == '\\') {
- int escaped = reader.read();
+ int escaped = read(reader, pos);
if (escaped == -1) {
throw new ParseException("unexpected end of buffer in state " + state.name(), pos);
}
@@ -133,10 +134,10 @@ public Token(TokenType type, Object value) {
nextState = state;
} else if (c == '(') {
nextState = newState(TokenType.INTEGER, tokens, buf, State.WS);
- tokens.add(open_bracket);
+ tokens.add(open_paren);
} else if (c == ')') {
nextState = newState(TokenType.INTEGER, tokens, buf, State.WS);
- tokens.add(close_bracket);
+ tokens.add(close_paren);
} else if (Character.isWhitespace(c)) {
tokens.add(new Token(TokenType.INTEGER, buf.toString()));
nextState = State.WS;
@@ -159,6 +160,13 @@ public Token(TokenType type, Object value) {
return tokens;
}
+ private static int read(Reader reader, int pos) throws ParseException {
+ try {
+ return reader.read();
+ } catch (IOException e) {
+ throw new ParseException("unexpected IO Error: "+e.getClass().getSimpleName()+": "+e.getMessage(), pos);
+ }
+ }
private static void append(int c, StringBuilder buf) {
addCharacter((char) c, buf);
@@ -1,14 +1,16 @@
-package com.tmp.uint.core;
+package com.tmp.uint.core.compilation;
import com.tmp.uint.core.types.Function;
import org.junit.Test;
+import java.io.IOException;
+import java.text.ParseException;
import java.util.Collections;
public class CompilerTest {
@Test
- public void testNothing() {
+ public void testNothing() throws IOException, ParseException {
if (false) {
new com.tmp.uint.core.compilation.Compiler(Collections.<String, Function>emptyMap()).compile("bla bla");
@@ -0,0 +1,57 @@
+package com.tmp.uint.core.compilation;
+
+import junit.framework.Assert;
+import org.junit.Test;
+import static com.tmp.uint.core.compilation.Tokenizer.Token;
+
+import java.io.IOException;
+import java.text.ParseException;
+import java.util.Arrays;
+import java.util.List;
+
+public class TokenizerTests {
+
+ @Test
+ public void testNothing() throws ParseException {
+ List<Token> noTokens = Tokenizer.tokenize("");
+ Assert.assertTrue(noTokens.isEmpty());
+ }
+
+ @Test
+ public void testSpaces() throws ParseException {
+ List<Token> noTokens = Tokenizer.tokenize(" ");
+ Assert.assertTrue(noTokens.isEmpty());
+ }
+
+ @Test
+ public void testJustOpenParen() throws ParseException {
+ for (String s: Arrays.asList("(", "( ", " ( ", " (")) {
+ List<Token> p1 = Tokenizer.tokenize(s);
+ Assert.assertEquals(Tokenizer.TokenType.OPEN_PAREN, p1.get(0).type);
+ }
+ }
+
+ @Test
+ public void testJustCloseParen() throws ParseException {
+ for (String s: Arrays.asList(")", ") ", " ) ", " )")) {
+ List<Token> p1 = Tokenizer.tokenize(s);
+ Assert.assertEquals(Tokenizer.TokenType.CLOSE_PAREN, p1.get(0).type);
+ }
+ }
+
+ @Test
+ public void testJustInteger() throws ParseException {
+ for (Object[] row: new Object[][]{
+ {"000", 0}, {"000 ", 0}, {" 0 ", 0}, {" 00", 0},
+ {"1", 1}, {"11 ", 11}, {" 12345", 12345}, {" 1322 ", 1322}}) {
+ String s = (String) row[0];
+ Integer i = (Integer) row[1];
+ List<Token> p1 = Tokenizer.tokenize(s);
+ Assert.assertEquals(Tokenizer.TokenType.INTEGER, p1.get(0).type);
+ String is = s.trim();
+ Assert.assertEquals(is, p1.get(0).value);
+ Assert.assertEquals((Object)i, Integer.parseInt(is));
+ }
+ }
+
+}
@@ -1,4 +1,4 @@
-package com.tmp.uint.core;
+package com.tmp.uint.core.execution;
import com.tmp.uint.functions.arithmetic.One;
import com.tmp.uint.functions.arithmetic.Plus;

0 comments on commit c2dc244

Please sign in to comment.