Skip to content
Browse files

Add protocol buffers serialization! Applying patch from Ishmael.

  • Loading branch information...
1 parent 1c93169 commit 7df4ece4905dcbf35dfba6b2737fd9504144ad23 @jkreps jkreps committed Jan 28, 2009
View
1 .classpath
@@ -28,5 +28,6 @@
<classpathentry kind="lib" path="lib/velocity-1.5.jar"/>
<classpathentry kind="lib" path="lib/xerces.jar"/>
<classpathentry kind="lib" path="lib/colt-1.2.0.jar"/>
+ <classpathentry kind="lib" path="lib/protobuf-java-2.0.3.jar"/>
<classpathentry kind="output" path="classes"/>
</classpath>
View
2 NOTICE
@@ -18,7 +18,7 @@ This product includes the libraries
xerces
all of which are software developed by The Apache Software Foundation (http://www.apache.org/)
-This product includes google-collections, a library developed by Google (http://www.google.com and http://code.google.com/p/google-collections/).
+This product includes google-collections and protocol buffers, two libraries developed by Google (http://www.google.com, http://code.google.com/p/google-collections/ and http://code.google.com/apis/protocolbuffers/).
This product includes jdom, a library developed by jdom.org.
View
4 build_number.txt
@@ -1,3 +1,3 @@
#Build Number for ANT. Do not edit!
-#Sat Jan 24 21:40:14 PST 2009
-build.number=177
+#Tue Jan 27 10:52:16 PST 2009
+build.number=182
View
BIN lib/protobuf-java-2.0.3.jar
Binary file not shown.
View
6 src/java/voldemort/serialization/DefaultSerializerFactory.java
@@ -21,6 +21,9 @@
import voldemort.serialization.json.JsonTypeDefinition;
import voldemort.serialization.json.JsonTypeSerializer;
+import voldemort.serialization.protobuf.ProtoBufSerializer;
+
+import com.google.protobuf.Message;
/**
* Factory that maps serialization strings to serializers. Used to get a
@@ -35,6 +38,7 @@
private static final String STRING_SERIALIZER_TYPE_NAME = "string";
private static final String IDENTITY_SERIALIZER_TYPE_NAME = "identity";
private static final String JSON_SERIALIZER_TYPE_NAME = "json";
+ private static final String PROTO_BUF_TYPE_NAME = "protobuf";
public Serializer<?> getSerializer(SerializerDefinition serializerDef) {
String name = serializerDef.getName();
@@ -50,6 +54,8 @@
.entrySet())
versions.put(entry.getKey(), JsonTypeDefinition.fromJson(entry.getValue()));
return new JsonTypeSerializer(versions);
+ } else if(name.equals(PROTO_BUF_TYPE_NAME)) {
+ return new ProtoBufSerializer<Message>(serializerDef.getCurrentSchemaInfo());
} else {
throw new IllegalArgumentException("No known serializer type: "
+ serializerDef.getName());
View
5 src/java/voldemort/serialization/json/JsonTypeDefinition.java
@@ -19,8 +19,8 @@
import java.io.Serializable;
import java.io.StringReader;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
-import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -97,7 +97,8 @@ private static Object fromJsonObjects(Object o) {
public JsonTypeDefinition projectionType(String... properties) {
if(this.getType() instanceof Map) {
Map<String, Object> type = (Map<String, Object>) getType();
- Map<String, Object> newType = new HashMap<String, Object>();
+ Arrays.sort(properties);
+ Map<String, Object> newType = new LinkedHashMap<String, Object>();
for(String prop: properties)
newType.put(prop, type.get(prop));
return new JsonTypeDefinition(newType);
View
84 src/java/voldemort/serialization/protobuf/ProtoBufSerializer.java
@@ -0,0 +1,84 @@
+package voldemort.serialization.protobuf;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
+import org.apache.commons.lang.StringUtils;
+
+import voldemort.serialization.Serializer;
+
+import com.google.protobuf.Message;
+
+/**
+ * A serializer that relies on Protocol Buffers for serialization and
+ * deserialization. Only Java clients are supported currently, but there are
+ * plans to support clients in other languages.
+ *
+ * An example configuration of a value-serializer follows.
+ *
+ * <pre>
+ * &lt;value-serializer&gt;
+ * &lt;type&gt;protobuf&lt;/type&gt;
+ * &lt;schema-info&gt;java=com.linkedin.foobar.FooMessage&lt;/schema-info&gt;
+ * &lt;/value-serializer&gt;
+ * </pre>
+ *
+ * Once support for clients in other languages is available, a comma-separated
+ * list will be accepted for the schema-info element (one for each language).
+ */
+public class ProtoBufSerializer<T extends Message> implements Serializer<T> {
+
+ private static final String ONLY_JAVA_CLIENTS_SUPPORTED = "Only Java clients are supported currently, so the format of the schema-info should be: <schema-info>java=foo.Bar</schema-info> where foo.Bar is the fully qualified name of the message.";
+
+ private final Method parseFromMethod;
+ private final Class<T> messageClass;
+
+ @SuppressWarnings("unchecked")
+ public ProtoBufSerializer(String currentSchemaInfo) {
+ try {
+ this.messageClass = (Class<T>) Class.forName(getJavaFqnForMessage(currentSchemaInfo),
+ false,
+ Thread.currentThread()
+ .getContextClassLoader());
+
+ if(!Message.class.isAssignableFrom(messageClass))
+ throw new IllegalArgumentException("Class provided should be a subtype of Message");
+
+ parseFromMethod = messageClass.getMethod("parseFrom", byte[].class);
+ } catch(NoSuchMethodException e) {
+ throw new IllegalArgumentException("No parseFrom static method found, the provided class is not a Message.",
+ e);
+ } catch(ClassNotFoundException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
+ private String getJavaFqnForMessage(String currentSchemaInfo) {
+ if(StringUtils.isEmpty(currentSchemaInfo))
+ throw new IllegalArgumentException("The type protobuf requires a non-empty schema-info.");
+
+ String[] languagePairs = StringUtils.split(currentSchemaInfo, ',');
+ if(languagePairs.length > 1)
+ throw new IllegalArgumentException(ONLY_JAVA_CLIENTS_SUPPORTED);
+
+ String[] javaPair = StringUtils.split(languagePairs[0], '=');
+ if(javaPair.length != 2 || !javaPair[0].trim().equals("java"))
+ throw new IllegalArgumentException(ONLY_JAVA_CLIENTS_SUPPORTED);
+
+ return javaPair[1].trim();
+ }
+
+ public byte[] toBytes(T object) {
+ return object.toByteArray();
+ }
+
+ public T toObject(byte[] bytes) {
+ try {
+ return messageClass.cast(parseFromMethod.invoke(null, bytes));
+ } catch(InvocationTargetException e) {
+ throw new IllegalArgumentException(e);
+ } catch(IllegalAccessException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+}
View
121 test/unit/voldemort/serialization/protobuf/ProtoBufSerializerTest.java
@@ -0,0 +1,121 @@
+package voldemort.serialization.protobuf;
+
+import junit.framework.TestCase;
+import voldemort.serialization.DefaultSerializerFactory;
+import voldemort.serialization.Serializer;
+import voldemort.serialization.SerializerDefinition;
+
+import com.google.protobuf.GeneratedMessage;
+import com.google.protobuf.Message;
+
+public class ProtoBufSerializerTest extends TestCase {
+
+ private abstract static class AbstractMessageStub extends GeneratedMessage {
+
+ byte[] bytes;
+
+ public AbstractMessageStub(byte[] bytes) {
+ this.bytes = bytes;
+ }
+
+ @Override
+ protected FieldAccessorTable internalGetFieldAccessorTable() {
+ throw new UnsupportedOperationException();
+ }
+
+ public Message getDefaultInstanceForType() {
+ throw new UnsupportedOperationException();
+ }
+
+ public com.google.protobuf.Message.Builder newBuilderForType() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public byte[] toByteArray() {
+ return bytes;
+ }
+ }
+
+ private static class MessageWithNoParseFrom extends AbstractMessageStub {
+
+ public MessageWithNoParseFrom(byte[] bytes) {
+ super(bytes);
+ }
+ }
+
+ private static class InvalidMessageWithParseFrom {
+
+ public static InvalidMessageWithParseFrom parseFrom(byte[] bytes) {
+ return new InvalidMessageWithParseFrom();
+ }
+ }
+
+ private static class MessageStub extends AbstractMessageStub {
+
+ public MessageStub(byte[] bytes) {
+ super(bytes);
+ }
+
+ public static MessageStub parseFrom(byte[] bytes) {
+ return new MessageStub(bytes);
+ }
+ }
+
+ public void testValidSchemaInfo() {
+ new ProtoBufSerializer<Message>(" java = " + MessageStub.class.getName());
+ createSerializer();
+ }
+
+ public void testToBytes() {
+ ProtoBufSerializer<MessageStub> serializer = createSerializer();
+ MessageStub message = new MessageStub(new byte[] { 23, 34, 22, 23 });
+ assertEquals(message.bytes, serializer.toBytes(message));
+ }
+
+ private ProtoBufSerializer<MessageStub> createSerializer() {
+ return new ProtoBufSerializer<MessageStub>("java=" + MessageStub.class.getName());
+ }
+
+ public void testToObject() {
+ ProtoBufSerializer<MessageStub> serializer = createSerializer();
+ byte[] bytes = new byte[] { 3, 5, 123 };
+ assertEquals(bytes, serializer.toObject(bytes).bytes);
+ }
+
+ public void testNonExistentClass() {
+ try {
+ new ProtoBufSerializer<Message>("java=com.foo.Bar");
+ } catch(IllegalArgumentException e) {
+ assertEquals(ClassNotFoundException.class, e.getCause().getClass());
+ return;
+ }
+ fail("IllegalArgumentException should have been thrown");
+ }
+
+ public void testClassWithNoParseFrom() {
+ try {
+ new ProtoBufSerializer<Message>("java=" + MessageWithNoParseFrom.class.getName());
+ } catch(IllegalArgumentException e) {
+ assertEquals(NoSuchMethodException.class, e.getCause().getClass());
+ return;
+ }
+ fail("IllegalArgumentException should have been thrown");
+ }
+
+ public void testNoMessageWithParseFrom() {
+ try {
+ new ProtoBufSerializer<Message>("java=" + InvalidMessageWithParseFrom.class.getName());
+ } catch(IllegalArgumentException e) {
+ return;
+ }
+ fail("IllegalArgumentException should have been thrown");
+ }
+
+ public void testGetSerializer() {
+ SerializerDefinition def = new SerializerDefinition("protobuf",
+ "java=" + MessageStub.class.getName());
+ Serializer<?> serializer = new DefaultSerializerFactory().getSerializer(def);
+ assertEquals(ProtoBufSerializer.class, serializer.getClass());
+ }
+}

0 comments on commit 7df4ece

Please sign in to comment.
Something went wrong with that request. Please try again.