Skip to content
Permalink
Browse files
GIRAPH-1195
closes #76
  • Loading branch information
dlogothetis committed May 31, 2018
1 parent 505c98d commit 0c1e2ebce6e4f6e1a47d4dec5c0b8475433ef8df
Show file tree
Hide file tree
Showing 5 changed files with 113 additions and 51 deletions.
@@ -27,6 +27,7 @@
import java.util.Random;

import com.esotericsoftware.kryo.util.DefaultClassResolver;
import de.javakaffee.kryoserializers.guava.ImmutableMapSerializer;
import org.apache.giraph.conf.GiraphConfigurationSettable;
import com.esotericsoftware.kryo.ClassResolver;
import com.esotericsoftware.kryo.ReferenceResolver;
@@ -39,7 +40,7 @@
import org.apache.giraph.writable.kryo.serializers.CollectionsNCopiesSerializer;
import org.apache.giraph.writable.kryo.serializers.DirectWritableSerializer;
import org.apache.giraph.writable.kryo.serializers.FastUtilSerializer;
import org.apache.giraph.writable.kryo.serializers.ImmutableMapSerializer;
import org.apache.giraph.writable.kryo.serializers.ImmutableBiMapSerializerUtils;
import org.apache.giraph.writable.kryo.serializers.ReusableFieldSerializer;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
@@ -322,7 +323,8 @@ private static HadoopKryo createKryo(boolean trackReferences,
if (minor >= '8') {
try {
kryo.register(Class.forName("java.lang.invoke.SerializedLambda"));
kryo.register(Class.forName("com.esotericsoftware.kryo.Kryo$Closure"),
kryo.register(Class.forName(
"com.esotericsoftware.kryo.serializers.ClosureSerializer$Closure"),
new ClosureSerializer());
} catch (ClassNotFoundException e) {
throw new IllegalStateException(
@@ -336,12 +338,8 @@ private static HadoopKryo createKryo(boolean trackReferences,
new CollectionsNCopiesSerializer());

ImmutableListSerializer.registerSerializers(kryo);

registerSerializer(kryo, "com.google.common.collect.RegularImmutableMap",
new ImmutableMapSerializer());
registerSerializer(kryo,
"com.google.common.collect.SingletonImmutableBiMap",
new ImmutableMapSerializer());
ImmutableMapSerializer.registerSerializers(kryo);
ImmutableBiMapSerializerUtils.registerSerializers(kryo);

// There are many fastutil classes, register them at the end,
// so they don't use up small registration numbers
@@ -0,0 +1,60 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.giraph.writable.kryo.serializers;


import com.esotericsoftware.kryo.Kryo;
import com.google.common.collect.ImmutableBiMap;
import de.javakaffee.kryoserializers.guava.ImmutableMapSerializer;

/**
* Helper class used to register serializers for the different classes
* return by the {@link ImmutableBiMap} methods.
*/
public class ImmutableBiMapSerializerUtils {

/**
* Private default constructor
*/
private ImmutableBiMapSerializerUtils() {
}

/**
* Registers different {@link ImmutableBiMap} implementations with Kryo.
* @param kryo {@link Kryo} instance to register class with.
*/
public static void registerSerializers(Kryo kryo) {
ImmutableMapSerializer serializer = new ImmutableMapSerializer();
kryo.register(ImmutableBiMap.class, serializer);
kryo.register(ImmutableBiMap.of().getClass(), serializer);
Object o1 = new Object();
Object o2 = new Object();
Object o3 = new Object();
Object o4 = new Object();
Object o5 = new Object();
kryo.register(ImmutableBiMap.of(o1, o1).getClass(), serializer);
kryo.register(ImmutableBiMap.of(o1, o1, o2, o2).getClass(), serializer);
kryo.register(ImmutableBiMap.of(o1, o1, o2, o2, o3, o3)
.getClass(), serializer);
kryo.register(ImmutableBiMap.of(o1, o1, o2, o2, o3, o3, o4, o4)
.getClass(), serializer);
kryo.register(ImmutableBiMap.of(o1, o1, o2, o2, o3, o3, o4, o4, o5, o5)
.getClass(), serializer);
}
}

This file was deleted.

@@ -25,8 +25,11 @@
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;

import com.google.common.collect.ImmutableBiMap;
import com.google.common.collect.ImmutableMap;
import org.apache.giraph.conf.GiraphConfiguration;
import org.apache.giraph.conf.GiraphConfigurationSettable;
import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
@@ -170,6 +173,41 @@ public void testImmutableList() throws IOException {
}
}

@Test
public void testImmutableMapSerialization() throws IOException {
Map original = ImmutableMap.of("x", "y", "y", "z");
Map copy = kryoSerDeser(original);
assertEquals(original, copy);
}

@Test
public void testImmutableMapSinglePairSerialization() throws IOException {
Map original = ImmutableMap.of("x", "y");
Map copy = kryoSerDeser(original);
assertEquals(original, copy);
}

@Test
public void testImmutableBiMap() throws IOException {
Map original = ImmutableBiMap.of("x", "y", "z", "w");
Map copy = kryoSerDeser(original);
assertEquals(original, copy);
}

@Test
public void testSingletonImmutableBiMapSerialization() throws IOException {
Map original = ImmutableBiMap.of("x", "y");
Map copy = kryoSerDeser(original);
assertEquals(original, copy);
}

@Test
public void testEmptyImmutableBiMap() throws IOException {
Map original = ImmutableBiMap.of();
Map copy = kryoSerDeser(original);
assertEquals(original, copy);
}

@Test
public void testFastutilSet() throws ClassNotFoundException, IOException {
LongOpenHashSet set = new LongOpenHashSet();
12 pom.xml
@@ -343,15 +343,16 @@ under the License.
<dep.json.version>20160810</dep.json.version>
<dep.junit.version>4.11</dep.junit.version>
<dep.jython.version>2.5.3</dep.jython.version>
<dep.kryo.version>3.0.1</dep.kryo.version>
<dep.kryo-serializers.version>0.29</dep.kryo-serializers.version>
<dep.kryo.version>4.0.0</dep.kryo.version>
<dep.reflectasm.version>1.11.3</dep.reflectasm.version>
<dep.kryo-serializers.version>0.42</dep.kryo-serializers.version>
<dep.libthrift.version>0.9.0</dep.libthrift.version>
<dep.log4j.version>1.2.17</dep.log4j.version>
<dep.mockito.version>1.9.5</dep.mockito.version>
<!-- note: old version of netty is required by hadoop_facebook for tests to succeed -->
<dep.netty.version>4.0.14.Final</dep.netty.version>
<dep.oldnetty.version>3.2.2.Final</dep.oldnetty.version>
<dep.objenesis.version>2.1</dep.objenesis.version>
<dep.objenesis.version>2.2</dep.objenesis.version>
<dep.openhft-compiler.version>2.2.1</dep.openhft-compiler.version>
<dep.paranamer.version>2.5.2</dep.paranamer.version>
<dep.slf4j.version>1.7.6</dep.slf4j.version>
@@ -2049,6 +2050,11 @@ under the License.
<artifactId>kryo</artifactId>
<version>${dep.kryo.version}</version>
</dependency>
<dependency>
<groupId>com.esotericsoftware.reflectasm</groupId>
<artifactId>reflectasm</artifactId>
<version>${dep.reflectasm.version}</version>
</dependency>
<dependency>
<groupId>de.javakaffee</groupId>
<artifactId>kryo-serializers</artifactId>

0 comments on commit 0c1e2eb

Please sign in to comment.