Skip to content

Commit

Permalink
codewritable fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
Klaas Bosteels authored and Klaas Bosteels committed Nov 27, 2008
1 parent d2d469a commit 8585087
Show file tree
Hide file tree
Showing 3 changed files with 62 additions and 36 deletions.
13 changes: 9 additions & 4 deletions src/java/org/apache/hadoop/dumbo/CodeUtils.java
View file
Expand Up @@ -38,7 +38,7 @@
public abstract class CodeUtils {

public static enum CodeType {
NULL, BOOLEAN, INTEGER, LONG, FLOAT, STRING, TUPLE, LIST, DICTIONARY
NULL, BOOLEAN, INTEGER, LONG, FLOAT, STRING, TUPLE, LIST, DICTIONARY, OTHER
}

private CodeUtils() {}
Expand Down Expand Up @@ -153,14 +153,19 @@ public static String[] codesFromDictionary(String code) {
private static String[] findSubcodes(String code) {
List<String> codes = new ArrayList<String>();
boolean inStr = false;
char strChar = 0, prevChar = 0;
int prevIndex = 1;
for (int i = 1; i < code.length()-1; i++) {
char c = code.charAt(i);
if (c == '\'' || c == '"') inStr = !inStr;
if ((c == '\'' || c == '"') && (!inStr || c == strChar) && prevChar != '\\') {
inStr = !inStr;
strChar = c;
}
else if (!inStr && (c == ',' || c == ':')) {
codes.add(code.substring(prevIndex, i).trim());
prevIndex = i+1;
}
prevChar = c;
}
codes.add(code.substring(prevIndex, code.length()-1).trim());
String[] codesArray = new String[codes.size()];
Expand All @@ -187,9 +192,9 @@ public static CodeType deriveType(String code) {
return CodeType.FLOAT;
} else if (code.charAt(code.length()-1) == 'L') {
return CodeType.LONG;
} else {
} else if (Character.isDigit(code.charAt(0))) {
return CodeType.INTEGER;
}
} else return CodeType.OTHER;
}


Expand Down
42 changes: 21 additions & 21 deletions src/java/org/apache/hadoop/dumbo/CodeWritable.java
View file
Expand Up @@ -55,27 +55,27 @@ public CodeType getType() {
}

public void write(DataOutput out) throws IOException {
CodeType type = CodeUtils.deriveType(code);
out.writeByte(type.ordinal());
if (type == CodeType.STRING) {
writeString(out, CodeUtils.codeToString(code));
} else if (type == CodeType.BOOLEAN) {
out.writeBoolean(CodeUtils.codeToBoolean(code));
} else if (type == CodeType.INTEGER) {
WritableUtils.writeVInt(out, CodeUtils.codeToInt(code));
} else if (type == CodeType.LONG) {
WritableUtils.writeVLong(out, CodeUtils.codeToLong(code));
} else if (type == CodeType.FLOAT) {
out.writeFloat(CodeUtils.codeToFloat(code));
} else if (type == CodeType.TUPLE) {
writeSequence(out, CodeUtils.codesFromTuple(code));
} else if (type == CodeType.LIST) {
writeSequence(out, CodeUtils.codesFromList(code));
} else if (type == CodeType.DICTIONARY) {
writeSequence(out, CodeUtils.codesFromDictionary(code));
} else if (type != CodeType.NULL) {
writeString(out, code); // write code itself
}
CodeType type = CodeUtils.deriveType(code);
out.writeByte(type.ordinal());
if (type == CodeType.STRING) {
writeString(out, CodeUtils.codeToString(code));
} else if (type == CodeType.BOOLEAN) {
out.writeBoolean(CodeUtils.codeToBoolean(code));
} else if (type == CodeType.INTEGER) {
WritableUtils.writeVInt(out, CodeUtils.codeToInt(code));
} else if (type == CodeType.LONG) {
WritableUtils.writeVLong(out, CodeUtils.codeToLong(code));
} else if (type == CodeType.FLOAT) {
out.writeFloat(CodeUtils.codeToFloat(code));
} else if (type == CodeType.TUPLE) {
writeSequence(out, CodeUtils.codesFromTuple(code));
} else if (type == CodeType.LIST) {
writeSequence(out, CodeUtils.codesFromList(code));
} else if (type == CodeType.DICTIONARY) {
writeSequence(out, CodeUtils.codesFromDictionary(code));
} else if (type != CodeType.NULL) {
writeString(out, code); // write code itself
}
}

public void readFields(DataInput in) throws IOException {
Expand Down
43 changes: 32 additions & 11 deletions src/test/java/org/apache/hadoop/dumbo/TestCodeWritable.java
View file
Expand Up @@ -68,12 +68,35 @@ public void testContainer() throws Exception {
testCode("( 1, 2, 3, 'random' )","(1,2,3,'random')");
testCode("[1,2,3,'random']");
testCode("{'1':1,'2':2}");
testCode("('test','paul\\'s, mine, and yours')");
testCode("('test',\"paul's, mine, and yours\")","('test','paul\\'s, mine, and yours')");
}

private static void testCode(String before, String goal) throws IOException {
public void testRest() throws Exception {
testCode("MyClass(1,'string')", false);
}

private static void testCode(String code, String goal, boolean strict) throws IOException {
testSerialization(code, goal);
compareWithText(code, strict);
}

private static void testCode(String code, String goal) throws IOException {
testCode(code, goal, true);
}

private static void testCode(String code, boolean strict) throws IOException {
testCode(code, code, strict);
}

private static void testCode(String code) throws IOException {
testCode(code, code, true);
}

private static void testSerialization(String code, String goal) throws IOException {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
DataOutputStream dout = new DataOutputStream(bout);
CodeWritable cw = new CodeWritable(before);
CodeWritable cw = new CodeWritable(code);
cw.write(dout);
dout.close();
bout.close();
Expand All @@ -83,16 +106,11 @@ private static void testCode(String before, String goal) throws IOException {
din.close();
bin.close();
String after = cw.get();
System.out.println("before: " + before + ", after: " + after);
assertTrue("Wrong code deserialized for \"" + before + "\".", goal.equals(after));
compareWithText(before);
}

private static void testCode(String before) throws IOException {
testCode(before, before);
System.out.println("before: " + code + ", after: " + after);
assertTrue("Wrong code deserialized for \"" + code + "\".", goal.equals(after));
}

private static void compareWithText(String code) throws IOException {
private static void compareWithText(String code, boolean strict) throws IOException {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
DataOutputStream dout = new DataOutputStream(bout);
CodeWritable cw = new CodeWritable(code);
Expand All @@ -109,6 +127,9 @@ private static void compareWithText(String code) throws IOException {
int textlen = bout.toByteArray().length;
System.out.println("Number of bytes for \"" + code + "\" when using CodeWritable = " + cwlen);
System.out.println("Number of bytes for \"" + code + "\" when using Text = " + textlen);
assertTrue("CodeWritable does not require less bytes than Text for \"" + code + "\"", cwlen < textlen);
if (strict)
assertTrue("CodeWritable does not require less bytes than Text for \"" + code + "\"", cwlen < textlen);
else
assertTrue("Text requires at least 2 bytes less than CodeWritable for \"" + code + "\"", cwlen - 1 <= textlen);
}
}

0 comments on commit 8585087

Please sign in to comment.