[HIVEMALL-311] Upgrade Kryo version from 2.21 to 2.24.0

## What changes were proposed in this pull request?

xgboost4j and xgboost module used Kryo version 2.21 but it has a bug in serializing generic collections. So, update Kryo version to 2.24.0 just in case.

## What type of PR is it?

Bug Fix

## What is the Jira issue?

https://issues.apache.org/jira/browse/HIVEMALL-311

## How was this patch tested?

unit tests

## Checklist

(Please remove this section if not needed; check `x` for YES, blank for NO)

- [x] Did you apply source code formatter, i.e., `./bin/format_code.sh`, for your commit?
- [ ] Did you run system tests on Hive (or Spark)?

Author: Makoto Yui <myui@apache.org>

Closes #241 from myui/kryo_update.

core/pom.xml

@@ -163,6 +163,12 @@
 			<artifactId>mockito-core</artifactId>
 			<scope>test</scope>
 		</dependency>
+		<dependency>
+			<groupId>com.esotericsoftware.kryo</groupId>
+			<artifactId>kryo</artifactId>
+			<version>${kryo.version}</version>
+			<scope>test</scope>
+		</dependency>
 	</dependencies>
 
 </project>

core/src/test/java/hivemall/TestUtils.java

@@ -32,6 +32,7 @@
 import org.apache.hive.com.esotericsoftware.kryo.Kryo;
 import org.apache.hive.com.esotericsoftware.kryo.io.Input;
 import org.apache.hive.com.esotericsoftware.kryo.io.Output;
+import org.objenesis.strategy.StdInstantiatorStrategy;
 
 public final class TestUtils {
 
@@ -48,8 +49,11 @@ public static <T extends GenericUDF> void testGenericUDFSerialization(@Nonnull C
         udf.initialize(ois);
 
         // serialization after initialization
-        byte[] serialized = serializeObjectByKryo(udf);
-        deserializeObjectByKryo(serialized, clazz);
+        byte[] serialized1 = serializeObjectByKryo(udf);
+        deserializeObjectByKryo(serialized1, clazz);
+
+        byte[] serialized2 = serializeObjectByOriginalKryo(udf);
+        deserializeObjectByOriginalKryo(serialized2, clazz);
 
         int size = row.length;
         GenericUDF.DeferredObject[] rowDeferred = new GenericUDF.DeferredObject[size];
@@ -60,8 +64,11 @@ public static <T extends GenericUDF> void testGenericUDFSerialization(@Nonnull C
         udf.evaluate(rowDeferred);
 
         // serialization after evaluating row
-        serialized = serializeObjectByKryo(udf);
-        TestUtils.deserializeObjectByKryo(serialized, clazz);
+        serialized1 = serializeObjectByKryo(udf);
+        TestUtils.deserializeObjectByKryo(serialized1, clazz);
+
+        serialized2 = serializeObjectByOriginalKryo(udf);
+        TestUtils.deserializeObjectByOriginalKryo(serialized2, clazz);
 
         udf.close();
     }
@@ -82,6 +89,9 @@ public static <T extends GenericUDTF> void testGenericUDTFSerialization(@Nonnull
         byte[] serialized = serializeObjectByKryo(udtf);
         deserializeObjectByKryo(serialized, clazz);
 
+        byte[] serialized2 = serializeObjectByOriginalKryo(udtf);
+        deserializeObjectByOriginalKryo(serialized2, clazz);
+
         udtf.setCollector(new Collector() {
             public void collect(Object input) throws HiveException {
                 // noop
@@ -99,9 +109,12 @@ public void collect(Object input) throws HiveException {
         udtf.close();
     }
 
+    // -------------------------------
+    // Hive version of Kryo
+
     @Nonnull
     public static byte[] serializeObjectByKryo(@Nonnull Object obj) {
-        Kryo kryo = getKryo();
+        Kryo kryo = getHiveKryo();
         ByteArrayOutputStream bos = new ByteArrayOutputStream();
         Output output = new Output(bos);
         kryo.writeObject(output, obj);
@@ -111,16 +124,53 @@ public static byte[] serializeObjectByKryo(@Nonnull Object obj) {
 
     @Nonnull
     public static <T> T deserializeObjectByKryo(@Nonnull byte[] in, @Nonnull Class<T> clazz) {
-        Kryo kryo = getKryo();
+        Kryo kryo = getHiveKryo();
         Input inp = new Input(in);
         T t = kryo.readObject(inp, clazz);
         inp.close();
         return t;
     }
 
     @Nonnull
-    private static Kryo getKryo() {
+    private static Kryo getHiveKryo() {
         return Utilities.runtimeSerializationKryo.get();
     }
 
+    // -------------------------------
+    // esotericsoftware's original version of Kryo
+
+    @Nonnull
+    public static byte[] serializeObjectByOriginalKryo(@Nonnull Object obj) {
+        com.esotericsoftware.kryo.Kryo kryo = getOriginalKryo();
+        ByteArrayOutputStream bos = new ByteArrayOutputStream();
+        com.esotericsoftware.kryo.io.Output output = new com.esotericsoftware.kryo.io.Output(bos);
+        kryo.writeObject(output, obj);
+        output.close();
+        return bos.toByteArray();
+    }
+
+    @Nonnull
+    public static <T> T deserializeObjectByOriginalKryo(@Nonnull byte[] in,
+            @Nonnull Class<T> clazz) {
+        com.esotericsoftware.kryo.Kryo kryo = getOriginalKryo();
+        com.esotericsoftware.kryo.io.Input inp = new com.esotericsoftware.kryo.io.Input(in);
+        T t = kryo.readObject(inp, clazz);
+        inp.close();
+        return t;
+    }
+
+    @Nonnull
+    private static com.esotericsoftware.kryo.Kryo getOriginalKryo() {
+        com.esotericsoftware.kryo.Kryo kryo = new com.esotericsoftware.kryo.Kryo();
+
+        // kryo.setReferences(true);
+        // kryo.setRegistrationRequired(false);
+
+        // see https://stackoverflow.com/a/23962797/5332768
+        ((com.esotericsoftware.kryo.Kryo.DefaultInstantiatorStrategy) kryo.getInstantiatorStrategy()).setFallbackInstantiatorStrategy(
+            new StdInstantiatorStrategy());
+
+        return kryo;
+    }
+
 }

pom.xml

@@ -272,6 +272,7 @@
 		<maven-enforcer.requireMavenVersion>[3.3.1,)</maven-enforcer.requireMavenVersion>
 		<surefire.version>2.19.1</surefire.version>
 		<netty.version>4.1.42.Final</netty.version>
+		<kryo.version>2.24.0</kryo.version>
 	</properties>
 
 	<distributionManagement>

xgboost/pom.xml

@@ -97,7 +97,7 @@
 			<!-- NOTE: This dependency needed for xgboost4j -->
 			<groupId>com.esotericsoftware.kryo</groupId>
 			<artifactId>kryo</artifactId>
-			<version>2.21</version>
+			<version>${kryo.version}</version>
 			<scope>compile</scope>
 		</dependency>
 		<dependency>