From dd5211e16b21ee691616ffcca71e076d54ca5c43 Mon Sep 17 00:00:00 2001 From: Dionysios Logothetis Date: Wed, 30 May 2018 15:39:40 -0700 Subject: [PATCH 1/2] Register RegularImmutableBiMap in HadoopKryo --- .../giraph/writable/kryo/HadoopKryo.java | 2 + .../kryo/KryoWritableWrapperTest.java | 38 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java index 27133168f..d5dba4544 100644 --- a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java +++ b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java @@ -339,6 +339,8 @@ private static HadoopKryo createKryo(boolean trackReferences, registerSerializer(kryo, "com.google.common.collect.RegularImmutableMap", new ImmutableMapSerializer()); + registerSerializer(kryo, "com.google.common.collect.RegularImmutableBiMap", + new ImmutableMapSerializer()); registerSerializer(kryo, "com.google.common.collect.SingletonImmutableBiMap", new ImmutableMapSerializer()); diff --git a/giraph-core/src/test/java/org/apache/giraph/writable/kryo/KryoWritableWrapperTest.java b/giraph-core/src/test/java/org/apache/giraph/writable/kryo/KryoWritableWrapperTest.java index 6e0eb2fbb..9129a4e3e 100644 --- a/giraph-core/src/test/java/org/apache/giraph/writable/kryo/KryoWritableWrapperTest.java +++ b/giraph-core/src/test/java/org/apache/giraph/writable/kryo/KryoWritableWrapperTest.java @@ -25,8 +25,11 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Random; +import com.google.common.collect.ImmutableBiMap; +import com.google.common.collect.ImmutableMap; import org.apache.giraph.conf.GiraphConfiguration; import org.apache.giraph.conf.GiraphConfigurationSettable; import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration; @@ -170,6 +173,41 @@ public void testImmutableList() throws IOException { } } + @Test + public void testImmutableMapSerialization() throws IOException { + Map original = ImmutableMap.of("x", "y", "y", "z"); + Map copy = kryoSerDeser(original); + assertEquals(original, copy); + } + + @Test + public void testImmutableMapSinglePairSerialization() throws IOException { + Map original = ImmutableMap.of("x", "y"); + Map copy = kryoSerDeser(original); + assertEquals(original, copy); + } + + @Test + public void testImmutableBiMap() throws IOException { + Map original = ImmutableBiMap.of("x", "y", "z", "w"); + Map copy = kryoSerDeser(original); + assertEquals(original, copy); + } + + @Test + public void testSingletonImmutableBiMapSerialization() throws IOException { + Map original = ImmutableBiMap.of("x", "y"); + Map copy = kryoSerDeser(original); + assertEquals(original, copy); + } + + @Test + public void testEmptyImmutableBiMap() throws IOException { + Map original = ImmutableBiMap.of(); + Map copy = kryoSerDeser(original); + assertEquals(original, copy); + } + @Test public void testFastutilSet() throws ClassNotFoundException, IOException { LongOpenHashSet set = new LongOpenHashSet(); From c9637aa9d5cf00616795bff666fe0c7ef5fc5120 Mon Sep 17 00:00:00 2001 From: Dionysios Logothetis Date: Wed, 30 May 2018 18:06:47 -0700 Subject: [PATCH 2/2] Register Map serializers in a more reliable way --- .../giraph/writable/kryo/HadoopKryo.java | 16 ++--- .../ImmutableBiMapSerializerUtils.java | 60 +++++++++++++++++++ .../serializers/ImmutableMapSerializer.java | 40 ------------- pom.xml | 12 +++- 4 files changed, 75 insertions(+), 53 deletions(-) create mode 100644 giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableBiMapSerializerUtils.java delete mode 100644 giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableMapSerializer.java diff --git a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java index d5dba4544..5d51a3f3d 100644 --- a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java +++ b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java @@ -27,6 +27,7 @@ import java.util.Random; import com.esotericsoftware.kryo.util.DefaultClassResolver; +import de.javakaffee.kryoserializers.guava.ImmutableMapSerializer; import org.apache.giraph.conf.GiraphConfigurationSettable; import com.esotericsoftware.kryo.ClassResolver; import com.esotericsoftware.kryo.ReferenceResolver; @@ -39,7 +40,7 @@ import org.apache.giraph.writable.kryo.serializers.CollectionsNCopiesSerializer; import org.apache.giraph.writable.kryo.serializers.DirectWritableSerializer; import org.apache.giraph.writable.kryo.serializers.FastUtilSerializer; -import org.apache.giraph.writable.kryo.serializers.ImmutableMapSerializer; +import org.apache.giraph.writable.kryo.serializers.ImmutableBiMapSerializerUtils; import org.apache.giraph.writable.kryo.serializers.ReusableFieldSerializer; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; @@ -322,7 +323,8 @@ private static HadoopKryo createKryo(boolean trackReferences, if (minor >= '8') { try { kryo.register(Class.forName("java.lang.invoke.SerializedLambda")); - kryo.register(Class.forName("com.esotericsoftware.kryo.Kryo$Closure"), + kryo.register(Class.forName( + "com.esotericsoftware.kryo.serializers.ClosureSerializer$Closure"), new ClosureSerializer()); } catch (ClassNotFoundException e) { throw new IllegalStateException( @@ -336,14 +338,8 @@ private static HadoopKryo createKryo(boolean trackReferences, new CollectionsNCopiesSerializer()); ImmutableListSerializer.registerSerializers(kryo); - - registerSerializer(kryo, "com.google.common.collect.RegularImmutableMap", - new ImmutableMapSerializer()); - registerSerializer(kryo, "com.google.common.collect.RegularImmutableBiMap", - new ImmutableMapSerializer()); - registerSerializer(kryo, - "com.google.common.collect.SingletonImmutableBiMap", - new ImmutableMapSerializer()); + ImmutableMapSerializer.registerSerializers(kryo); + ImmutableBiMapSerializerUtils.registerSerializers(kryo); // There are many fastutil classes, register them at the end, // so they don't use up small registration numbers diff --git a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableBiMapSerializerUtils.java b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableBiMapSerializerUtils.java new file mode 100644 index 000000000..6e66b47a9 --- /dev/null +++ b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableBiMapSerializerUtils.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.giraph.writable.kryo.serializers; + + +import com.esotericsoftware.kryo.Kryo; +import com.google.common.collect.ImmutableBiMap; +import de.javakaffee.kryoserializers.guava.ImmutableMapSerializer; + +/** + * Helper class used to register serializers for the different classes + * return by the {@link ImmutableBiMap} methods. + */ +public class ImmutableBiMapSerializerUtils { + + /** + * Private default constructor + */ + private ImmutableBiMapSerializerUtils() { + } + + /** + * Registers different {@link ImmutableBiMap} implementations with Kryo. + * @param kryo {@link Kryo} instance to register class with. + */ + public static void registerSerializers(Kryo kryo) { + ImmutableMapSerializer serializer = new ImmutableMapSerializer(); + kryo.register(ImmutableBiMap.class, serializer); + kryo.register(ImmutableBiMap.of().getClass(), serializer); + Object o1 = new Object(); + Object o2 = new Object(); + Object o3 = new Object(); + Object o4 = new Object(); + Object o5 = new Object(); + kryo.register(ImmutableBiMap.of(o1, o1).getClass(), serializer); + kryo.register(ImmutableBiMap.of(o1, o1, o2, o2).getClass(), serializer); + kryo.register(ImmutableBiMap.of(o1, o1, o2, o2, o3, o3) + .getClass(), serializer); + kryo.register(ImmutableBiMap.of(o1, o1, o2, o2, o3, o3, o4, o4) + .getClass(), serializer); + kryo.register(ImmutableBiMap.of(o1, o1, o2, o2, o3, o3, o4, o4, o5, o5) + .getClass(), serializer); + } +} diff --git a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableMapSerializer.java b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableMapSerializer.java deleted file mode 100644 index 8dd2d1cda..000000000 --- a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableMapSerializer.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.giraph.writable.kryo.serializers; - - -import com.esotericsoftware.kryo.Kryo; -import com.esotericsoftware.kryo.io.Input; -import com.esotericsoftware.kryo.serializers.MapSerializer; -import com.google.common.collect.ImmutableMap; - -import java.util.HashMap; -import java.util.Map; - -/** - * Serializer for {@link ImmutableMap} - */ -public class ImmutableMapSerializer extends MapSerializer { - @Override - public Map read(Kryo kryo, Input input, Class type) { - Map map = super.read(kryo, input, - (Class) ((Object) HashMap.class)); - return ImmutableMap.copyOf(map); - } -} diff --git a/pom.xml b/pom.xml index fd752fbea..1522fd4d0 100644 --- a/pom.xml +++ b/pom.xml @@ -343,15 +343,16 @@ under the License. 20160810 4.11 2.5.3 - 3.0.1 - 0.29 + 4.0.0 + 1.11.3 + 0.42 0.9.0 1.2.17 1.9.5 4.0.14.Final 3.2.2.Final - 2.1 + 2.2 2.2.1 2.5.2 1.7.6 @@ -2049,6 +2050,11 @@ under the License. kryo ${dep.kryo.version} + + com.esotericsoftware.reflectasm + reflectasm + ${dep.reflectasm.version} + de.javakaffee kryo-serializers