From 8bc08f9044fca1084e3a8d420dddadadcd804071 Mon Sep 17 00:00:00 2001 From: Song Kun Date: Fri, 26 Oct 2018 11:06:09 +0800 Subject: [PATCH] [Dubbo-1983] Support Protobuf Serialization (#2618) * finish support protobuf * polish * fix code review * use the general test for serialization --- .../dubbo-serialization-protobuf/pom.xml | 60 +++++++ .../protobuf/ProtobufObjectInput.java | 158 ++++++++++++++++++ .../protobuf/ProtobufObjectOutput.java | 121 ++++++++++++++ .../protobuf/ProtobufSerialization.java | 49 ++++++ .../common/serialize/protobuf/Wrapper.java | 33 ++++ .../protobuf/utils/WrapperUtils.java | 67 ++++++++ ...pache.dubbo.common.serialize.Serialization | 1 + .../dubbo-serialization-test/pom.xml | 5 + .../protobuf/ProtobufPersonOkTest.java | 26 +++ .../protobuf/ProtobufSerializationTest.java | 26 +++ dubbo-serialization/pom.xml | 1 + 11 files changed, 547 insertions(+) create mode 100644 dubbo-serialization/dubbo-serialization-protobuf/pom.xml create mode 100644 dubbo-serialization/dubbo-serialization-protobuf/src/main/java/org/apache/dubbo/common/serialize/protobuf/ProtobufObjectInput.java create mode 100644 dubbo-serialization/dubbo-serialization-protobuf/src/main/java/org/apache/dubbo/common/serialize/protobuf/ProtobufObjectOutput.java create mode 100644 dubbo-serialization/dubbo-serialization-protobuf/src/main/java/org/apache/dubbo/common/serialize/protobuf/ProtobufSerialization.java create mode 100644 dubbo-serialization/dubbo-serialization-protobuf/src/main/java/org/apache/dubbo/common/serialize/protobuf/Wrapper.java create mode 100644 dubbo-serialization/dubbo-serialization-protobuf/src/main/java/org/apache/dubbo/common/serialize/protobuf/utils/WrapperUtils.java create mode 100644 dubbo-serialization/dubbo-serialization-protobuf/src/main/resources/META-INF.dubbo.internal/org.apache.dubbo.common.serialize.Serialization create mode 100644 dubbo-serialization/dubbo-serialization-test/src/test/java/org/apache/dubbo/common/serialize/protobuf/ProtobufPersonOkTest.java create mode 100644 dubbo-serialization/dubbo-serialization-test/src/test/java/org/apache/dubbo/common/serialize/protobuf/ProtobufSerializationTest.java diff --git a/dubbo-serialization/dubbo-serialization-protobuf/pom.xml b/dubbo-serialization/dubbo-serialization-protobuf/pom.xml new file mode 100644 index 000000000000..d1843b0e9b39 --- /dev/null +++ b/dubbo-serialization/dubbo-serialization-protobuf/pom.xml @@ -0,0 +1,60 @@ + + + + 4.0.0 + + dubbo-serialization + org.apache.dubbo + 2.7.0-SNAPSHOT + + + dubbo-serialization-protobuf + jar + ${project.artifactId} + The protobuf serialization module of dubbo project + + + 1.5.9 + + + + + org.apache.dubbo + dubbo-serialization-api + ${project.parent.version} + + + io.protostuff + protostuff-core + ${protobuf.version} + + + io.protostuff + protostuff-runtime + ${protobuf.version} + + + org.objenesis + objenesis + 2.6 + + + + \ No newline at end of file diff --git a/dubbo-serialization/dubbo-serialization-protobuf/src/main/java/org/apache/dubbo/common/serialize/protobuf/ProtobufObjectInput.java b/dubbo-serialization/dubbo-serialization-protobuf/src/main/java/org/apache/dubbo/common/serialize/protobuf/ProtobufObjectInput.java new file mode 100644 index 000000000000..64773e9cff7f --- /dev/null +++ b/dubbo-serialization/dubbo-serialization-protobuf/src/main/java/org/apache/dubbo/common/serialize/protobuf/ProtobufObjectInput.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dubbo.common.serialize.protobuf; + +import io.protostuff.ProtobufIOUtil; +import io.protostuff.Schema; +import io.protostuff.runtime.RuntimeSchema; +import org.apache.dubbo.common.serialize.ObjectInput; +import org.apache.dubbo.common.serialize.protobuf.utils.WrapperUtils; + +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Type; + +public class ProtobufObjectInput implements ObjectInput { + + private DataInputStream dis; + + public ProtobufObjectInput(InputStream inputStream) { + dis = new DataInputStream(inputStream); + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + @Override + public Object readObject() throws IOException, ClassNotFoundException { + int classNameLength = dis.readInt(); + int bytesLength = dis.readInt(); + + if (classNameLength < 0 || bytesLength < 0) { + throw new IOException(); + } + + byte[] classNameBytes = new byte[classNameLength]; + dis.readFully(classNameBytes, 0, classNameLength); + + byte[] bytes = new byte[bytesLength]; + dis.readFully(bytes, 0, bytesLength); + + String className = new String(classNameBytes); + Class clazz = Class.forName(className); + + Object result; + if (WrapperUtils.needWrapper(clazz)) { + Schema schema = RuntimeSchema.getSchema(Wrapper.class); + Wrapper wrapper = schema.newMessage(); + ProtobufIOUtil.mergeFrom(bytes, wrapper, schema); + result = wrapper.getData(); + } else { + Schema schema = RuntimeSchema.getSchema(clazz); + result = schema.newMessage(); + ProtobufIOUtil.mergeFrom(bytes, result, schema); + } + + return result; + } + + @Override + public T readObject(Class clazz) throws IOException, ClassNotFoundException { + int classNameLength = dis.readInt(); + int bytesLength = dis.readInt(); + + if (classNameLength < 0 || bytesLength < 0) { + throw new IOException(); + } + + byte[] classNameBytes = new byte[classNameLength]; + dis.read(classNameBytes, 0, classNameLength); + + byte[] bytes = new byte[bytesLength]; + dis.read(bytes, 0, bytesLength); + + T result; + if (WrapperUtils.needWrapper(clazz)) { + Schema schema = RuntimeSchema.getSchema(Wrapper.class); + Wrapper wrapper = schema.newMessage(); + ProtobufIOUtil.mergeFrom(bytes, wrapper, schema); + result = (T) wrapper.getData(); + } else { + Schema schema = RuntimeSchema.getSchema(clazz); + result = schema.newMessage(); + ProtobufIOUtil.mergeFrom(bytes, result, schema); + } + + return result; + } + + @Override + public T readObject(Class cls, Type type) throws IOException, ClassNotFoundException { + return readObject(cls); + } + + @Override + public boolean readBool() throws IOException { + return dis.readBoolean(); + } + + @Override + public byte readByte() throws IOException { + return dis.readByte(); + } + + @Override + public short readShort() throws IOException { + return dis.readShort(); + } + + @Override + public int readInt() throws IOException { + return dis.readInt(); + } + + @Override + public long readLong() throws IOException { + return dis.readLong(); + } + + @Override + public float readFloat() throws IOException { + return dis.readFloat(); + } + + @Override + public double readDouble() throws IOException { + return dis.readDouble(); + } + + @Override + public String readUTF() throws IOException { + int length = dis.readInt(); + byte[] bytes = new byte[length]; + dis.read(bytes, 0, length); + return new String(bytes); + } + + @Override + public byte[] readBytes() throws IOException { + int length = dis.readInt(); + byte[] bytes = new byte[length]; + dis.read(bytes, 0, length); + return bytes; + } +} diff --git a/dubbo-serialization/dubbo-serialization-protobuf/src/main/java/org/apache/dubbo/common/serialize/protobuf/ProtobufObjectOutput.java b/dubbo-serialization/dubbo-serialization-protobuf/src/main/java/org/apache/dubbo/common/serialize/protobuf/ProtobufObjectOutput.java new file mode 100644 index 000000000000..8672a6619736 --- /dev/null +++ b/dubbo-serialization/dubbo-serialization-protobuf/src/main/java/org/apache/dubbo/common/serialize/protobuf/ProtobufObjectOutput.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dubbo.common.serialize.protobuf; + +import io.protostuff.*; +import io.protostuff.runtime.RuntimeSchema; +import org.apache.dubbo.common.serialize.ObjectOutput; +import org.apache.dubbo.common.serialize.protobuf.utils.WrapperUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +public class ProtobufObjectOutput implements ObjectOutput { + + private DataOutputStream dos; + + public ProtobufObjectOutput(OutputStream outputStream) { + dos = new DataOutputStream(outputStream); + } + + @Override + public void writeObject(Object obj) throws IOException { + LinkedBuffer buffer = LinkedBuffer.allocate(); + + byte[] bytes; + byte[] classNameBytes; + + if (WrapperUtils.needWrapper(obj)) { + Schema schema = RuntimeSchema.getSchema(Wrapper.class); + Wrapper wrapper = new Wrapper(obj); + bytes = ProtobufIOUtil.toByteArray(wrapper, schema, buffer); + classNameBytes = Wrapper.class.getName().getBytes(); + } else { + Schema schema = RuntimeSchema.getSchema(obj.getClass()); + bytes = ProtobufIOUtil.toByteArray(obj, schema, buffer); + classNameBytes = obj.getClass().getName().getBytes(); + } + + dos.writeInt(classNameBytes.length); + dos.writeInt(bytes.length); + dos.write(classNameBytes); + dos.write(bytes); + } + + @Override + public void writeBool(boolean v) throws IOException { + dos.writeBoolean(v); + } + + @Override + public void writeByte(byte v) throws IOException { + dos.writeByte(v); + } + + @Override + public void writeShort(short v) throws IOException { + dos.writeShort(v); + } + + @Override + public void writeInt(int v) throws IOException { + dos.writeInt(v); + } + + @Override + public void writeLong(long v) throws IOException { + dos.writeLong(v); + } + + @Override + public void writeFloat(float v) throws IOException { + dos.writeFloat(v); + } + + @Override + public void writeDouble(double v) throws IOException { + dos.writeDouble(v); + } + + @Override + public void writeUTF(String v) throws IOException { + byte[] bytes = v.getBytes(); + dos.writeInt(bytes.length); + dos.write(bytes); + } + + @Override + public void writeBytes(byte[] v) throws IOException { + dos.writeInt(v.length); + dos.write(v); + } + + @Override + public void writeBytes(byte[] v, int off, int len) throws IOException { + dos.writeInt(len); + byte[] bytes = new byte[len]; + System.arraycopy(v, off, bytes, 0, len); + dos.write(bytes); + } + + @Override + public void flushBuffer() throws IOException { + dos.flush(); + } +} diff --git a/dubbo-serialization/dubbo-serialization-protobuf/src/main/java/org/apache/dubbo/common/serialize/protobuf/ProtobufSerialization.java b/dubbo-serialization/dubbo-serialization-protobuf/src/main/java/org/apache/dubbo/common/serialize/protobuf/ProtobufSerialization.java new file mode 100644 index 000000000000..e87d25d6f735 --- /dev/null +++ b/dubbo-serialization/dubbo-serialization-protobuf/src/main/java/org/apache/dubbo/common/serialize/protobuf/ProtobufSerialization.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dubbo.common.serialize.protobuf; + +import org.apache.dubbo.common.URL; +import org.apache.dubbo.common.serialize.ObjectInput; +import org.apache.dubbo.common.serialize.ObjectOutput; +import org.apache.dubbo.common.serialize.Serialization; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +public class ProtobufSerialization implements Serialization { + @Override + public byte getContentTypeId() { + return 10; + } + + @Override + public String getContentType() { + return "x-application/protobuf"; + } + + @Override + public ObjectOutput serialize(URL url, OutputStream output) throws IOException { + return new ProtobufObjectOutput(output); + } + + @Override + public ObjectInput deserialize(URL url, InputStream input) throws IOException { + return new ProtobufObjectInput(input); + } +} diff --git a/dubbo-serialization/dubbo-serialization-protobuf/src/main/java/org/apache/dubbo/common/serialize/protobuf/Wrapper.java b/dubbo-serialization/dubbo-serialization-protobuf/src/main/java/org/apache/dubbo/common/serialize/protobuf/Wrapper.java new file mode 100644 index 000000000000..498c6c99e487 --- /dev/null +++ b/dubbo-serialization/dubbo-serialization-protobuf/src/main/java/org/apache/dubbo/common/serialize/protobuf/Wrapper.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dubbo.common.serialize.protobuf; + +/** + * Protostuff can only serialize/deserialize POJOs, for those it can't deal with, use this Wrapper. + */ +public class Wrapper { + private T data; + + Wrapper(T data) { + this.data = data; + } + + Object getData() { + return data; + } +} diff --git a/dubbo-serialization/dubbo-serialization-protobuf/src/main/java/org/apache/dubbo/common/serialize/protobuf/utils/WrapperUtils.java b/dubbo-serialization/dubbo-serialization-protobuf/src/main/java/org/apache/dubbo/common/serialize/protobuf/utils/WrapperUtils.java new file mode 100644 index 000000000000..32806d142a0f --- /dev/null +++ b/dubbo-serialization/dubbo-serialization-protobuf/src/main/java/org/apache/dubbo/common/serialize/protobuf/utils/WrapperUtils.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dubbo.common.serialize.protobuf.utils; + +import org.apache.dubbo.common.serialize.protobuf.Wrapper; + +import java.math.BigDecimal; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +public class WrapperUtils { + private static final Set> WRAPPER_SET = new HashSet<>(); + + static { + WRAPPER_SET.add(Map.class); + WRAPPER_SET.add(HashMap.class); + WRAPPER_SET.add(TreeMap.class); + WRAPPER_SET.add(Hashtable.class); + WRAPPER_SET.add(SortedMap.class); + WRAPPER_SET.add(LinkedHashMap.class); + WRAPPER_SET.add(ConcurrentHashMap.class); + + WRAPPER_SET.add(List.class); + WRAPPER_SET.add(ArrayList.class); + WRAPPER_SET.add(LinkedList.class); + + WRAPPER_SET.add(Vector.class); + + WRAPPER_SET.add(Set.class); + WRAPPER_SET.add(HashSet.class); + WRAPPER_SET.add(TreeSet.class); + WRAPPER_SET.add(BitSet.class); + + WRAPPER_SET.add(StringBuffer.class); + WRAPPER_SET.add(StringBuilder.class); + + WRAPPER_SET.add(BigDecimal.class); + WRAPPER_SET.add(Date.class); + WRAPPER_SET.add(Calendar.class); + + WRAPPER_SET.add(Wrapper.class); + } + + public static boolean needWrapper(Class clazz) { + return WrapperUtils.WRAPPER_SET.contains(clazz) || clazz.isArray() || clazz.isEnum(); + } + + public static boolean needWrapper(Object obj) { + return needWrapper(obj.getClass()); + } + +} diff --git a/dubbo-serialization/dubbo-serialization-protobuf/src/main/resources/META-INF.dubbo.internal/org.apache.dubbo.common.serialize.Serialization b/dubbo-serialization/dubbo-serialization-protobuf/src/main/resources/META-INF.dubbo.internal/org.apache.dubbo.common.serialize.Serialization new file mode 100644 index 000000000000..c276c91610e5 --- /dev/null +++ b/dubbo-serialization/dubbo-serialization-protobuf/src/main/resources/META-INF.dubbo.internal/org.apache.dubbo.common.serialize.Serialization @@ -0,0 +1 @@ +protobuf=org.apache.dubbo.common.serialize.protobuf.ProtobufSerialization \ No newline at end of file diff --git a/dubbo-serialization/dubbo-serialization-test/pom.xml b/dubbo-serialization/dubbo-serialization-test/pom.xml index 8c8b8fadca09..4577f7fb1881 100644 --- a/dubbo-serialization/dubbo-serialization-test/pom.xml +++ b/dubbo-serialization/dubbo-serialization-test/pom.xml @@ -59,6 +59,11 @@ dubbo-serialization-kryo ${project.parent.version} + + org.apache.dubbo + dubbo-serialization-protobuf + ${project.parent.version} + org.apache.dubbo dubbo-serialization-api diff --git a/dubbo-serialization/dubbo-serialization-test/src/test/java/org/apache/dubbo/common/serialize/protobuf/ProtobufPersonOkTest.java b/dubbo-serialization/dubbo-serialization-test/src/test/java/org/apache/dubbo/common/serialize/protobuf/ProtobufPersonOkTest.java new file mode 100644 index 000000000000..53e021f38ef1 --- /dev/null +++ b/dubbo-serialization/dubbo-serialization-test/src/test/java/org/apache/dubbo/common/serialize/protobuf/ProtobufPersonOkTest.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dubbo.common.serialize.protobuf; + +import org.apache.dubbo.common.serialize.base.AbstractSerializationPersonOkTest; + +public class ProtobufPersonOkTest extends AbstractSerializationPersonOkTest { + { + serialization = new ProtobufSerialization(); + } +} diff --git a/dubbo-serialization/dubbo-serialization-test/src/test/java/org/apache/dubbo/common/serialize/protobuf/ProtobufSerializationTest.java b/dubbo-serialization/dubbo-serialization-test/src/test/java/org/apache/dubbo/common/serialize/protobuf/ProtobufSerializationTest.java new file mode 100644 index 000000000000..ffb5fc66e2e9 --- /dev/null +++ b/dubbo-serialization/dubbo-serialization-test/src/test/java/org/apache/dubbo/common/serialize/protobuf/ProtobufSerializationTest.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dubbo.common.serialize.protobuf; + +import org.apache.dubbo.common.serialize.base.AbstractSerializationTest; + +public class ProtobufSerializationTest extends AbstractSerializationTest { + { + serialization = new ProtobufSerialization(); + } +} diff --git a/dubbo-serialization/pom.xml b/dubbo-serialization/pom.xml index 9c2e8694d77b..8546a1b98744 100644 --- a/dubbo-serialization/pom.xml +++ b/dubbo-serialization/pom.xml @@ -35,6 +35,7 @@ dubbo-serialization-kryo dubbo-serialization-fst dubbo-serialization-jdk + dubbo-serialization-protobuf dubbo-serialization-test