Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

codewritable fixes

  • Loading branch information...
commit 8585087fd1ef653fc26ae01d1d24112c82358b84 1 parent d2d469a
Klaas Bosteels authored
View
13 src/java/org/apache/hadoop/dumbo/CodeUtils.java
@@ -38,7 +38,7 @@
public abstract class CodeUtils {
public static enum CodeType {
- NULL, BOOLEAN, INTEGER, LONG, FLOAT, STRING, TUPLE, LIST, DICTIONARY
+ NULL, BOOLEAN, INTEGER, LONG, FLOAT, STRING, TUPLE, LIST, DICTIONARY, OTHER
}
private CodeUtils() {}
@@ -153,14 +153,19 @@ private static String combineSubcodes(String[] codes, String begin, String end,
private static String[] findSubcodes(String code) {
List<String> codes = new ArrayList<String>();
boolean inStr = false;
+ char strChar = 0, prevChar = 0;
int prevIndex = 1;
for (int i = 1; i < code.length()-1; i++) {
char c = code.charAt(i);
- if (c == '\'' || c == '"') inStr = !inStr;
+ if ((c == '\'' || c == '"') && (!inStr || c == strChar) && prevChar != '\\') {
+ inStr = !inStr;
+ strChar = c;
+ }
else if (!inStr && (c == ',' || c == ':')) {
codes.add(code.substring(prevIndex, i).trim());
prevIndex = i+1;
}
+ prevChar = c;
}
codes.add(code.substring(prevIndex, code.length()-1).trim());
String[] codesArray = new String[codes.size()];
@@ -187,9 +192,9 @@ public static CodeType deriveType(String code) {
return CodeType.FLOAT;
} else if (code.charAt(code.length()-1) == 'L') {
return CodeType.LONG;
- } else {
+ } else if (Character.isDigit(code.charAt(0))) {
return CodeType.INTEGER;
- }
+ } else return CodeType.OTHER;
}
View
42 src/java/org/apache/hadoop/dumbo/CodeWritable.java
@@ -55,27 +55,27 @@ public CodeType getType() {
}
public void write(DataOutput out) throws IOException {
- CodeType type = CodeUtils.deriveType(code);
- out.writeByte(type.ordinal());
- if (type == CodeType.STRING) {
- writeString(out, CodeUtils.codeToString(code));
- } else if (type == CodeType.BOOLEAN) {
- out.writeBoolean(CodeUtils.codeToBoolean(code));
- } else if (type == CodeType.INTEGER) {
- WritableUtils.writeVInt(out, CodeUtils.codeToInt(code));
- } else if (type == CodeType.LONG) {
- WritableUtils.writeVLong(out, CodeUtils.codeToLong(code));
- } else if (type == CodeType.FLOAT) {
- out.writeFloat(CodeUtils.codeToFloat(code));
- } else if (type == CodeType.TUPLE) {
- writeSequence(out, CodeUtils.codesFromTuple(code));
- } else if (type == CodeType.LIST) {
- writeSequence(out, CodeUtils.codesFromList(code));
- } else if (type == CodeType.DICTIONARY) {
- writeSequence(out, CodeUtils.codesFromDictionary(code));
- } else if (type != CodeType.NULL) {
- writeString(out, code); // write code itself
- }
+ CodeType type = CodeUtils.deriveType(code);
+ out.writeByte(type.ordinal());
+ if (type == CodeType.STRING) {
+ writeString(out, CodeUtils.codeToString(code));
+ } else if (type == CodeType.BOOLEAN) {
+ out.writeBoolean(CodeUtils.codeToBoolean(code));
+ } else if (type == CodeType.INTEGER) {
+ WritableUtils.writeVInt(out, CodeUtils.codeToInt(code));
+ } else if (type == CodeType.LONG) {
+ WritableUtils.writeVLong(out, CodeUtils.codeToLong(code));
+ } else if (type == CodeType.FLOAT) {
+ out.writeFloat(CodeUtils.codeToFloat(code));
+ } else if (type == CodeType.TUPLE) {
+ writeSequence(out, CodeUtils.codesFromTuple(code));
+ } else if (type == CodeType.LIST) {
+ writeSequence(out, CodeUtils.codesFromList(code));
+ } else if (type == CodeType.DICTIONARY) {
+ writeSequence(out, CodeUtils.codesFromDictionary(code));
+ } else if (type != CodeType.NULL) {
+ writeString(out, code); // write code itself
+ }
}
public void readFields(DataInput in) throws IOException {
View
43 src/test/java/org/apache/hadoop/dumbo/TestCodeWritable.java
@@ -68,12 +68,35 @@ public void testContainer() throws Exception {
testCode("( 1, 2, 3, 'random' )","(1,2,3,'random')");
testCode("[1,2,3,'random']");
testCode("{'1':1,'2':2}");
+ testCode("('test','paul\\'s, mine, and yours')");
+ testCode("('test',\"paul's, mine, and yours\")","('test','paul\\'s, mine, and yours')");
}
- private static void testCode(String before, String goal) throws IOException {
+ public void testRest() throws Exception {
+ testCode("MyClass(1,'string')", false);
+ }
+
+ private static void testCode(String code, String goal, boolean strict) throws IOException {
+ testSerialization(code, goal);
+ compareWithText(code, strict);
+ }
+
+ private static void testCode(String code, String goal) throws IOException {
+ testCode(code, goal, true);
+ }
+
+ private static void testCode(String code, boolean strict) throws IOException {
+ testCode(code, code, strict);
+ }
+
+ private static void testCode(String code) throws IOException {
+ testCode(code, code, true);
+ }
+
+ private static void testSerialization(String code, String goal) throws IOException {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
DataOutputStream dout = new DataOutputStream(bout);
- CodeWritable cw = new CodeWritable(before);
+ CodeWritable cw = new CodeWritable(code);
cw.write(dout);
dout.close();
bout.close();
@@ -83,16 +106,11 @@ private static void testCode(String before, String goal) throws IOException {
din.close();
bin.close();
String after = cw.get();
- System.out.println("before: " + before + ", after: " + after);
- assertTrue("Wrong code deserialized for \"" + before + "\".", goal.equals(after));
- compareWithText(before);
- }
-
- private static void testCode(String before) throws IOException {
- testCode(before, before);
+ System.out.println("before: " + code + ", after: " + after);
+ assertTrue("Wrong code deserialized for \"" + code + "\".", goal.equals(after));
}
- private static void compareWithText(String code) throws IOException {
+ private static void compareWithText(String code, boolean strict) throws IOException {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
DataOutputStream dout = new DataOutputStream(bout);
CodeWritable cw = new CodeWritable(code);
@@ -109,6 +127,9 @@ private static void compareWithText(String code) throws IOException {
int textlen = bout.toByteArray().length;
System.out.println("Number of bytes for \"" + code + "\" when using CodeWritable = " + cwlen);
System.out.println("Number of bytes for \"" + code + "\" when using Text = " + textlen);
- assertTrue("CodeWritable does not require less bytes than Text for \"" + code + "\"", cwlen < textlen);
+ if (strict)
+ assertTrue("CodeWritable does not require less bytes than Text for \"" + code + "\"", cwlen < textlen);
+ else
+ assertTrue("Text requires at least 2 bytes less than CodeWritable for \"" + code + "\"", cwlen - 1 <= textlen);
}
}
Please sign in to comment.
Something went wrong with that request. Please try again.