Skip to content
Permalink
Browse files
[HIVEMALL-311] Upgrade Kryo version from 2.21 to 2.24.0
## What changes were proposed in this pull request?

xgboost4j and xgboost module used Kryo version 2.21 but it has a bug in serializing generic collections. So, update Kryo version to 2.24.0 just in case.

## What type of PR is it?

Bug Fix

## What is the Jira issue?

https://issues.apache.org/jira/browse/HIVEMALL-311

## How was this patch tested?

unit tests

## Checklist

(Please remove this section if not needed; check `x` for YES, blank for NO)

- [x] Did you apply source code formatter, i.e., `./bin/format_code.sh`, for your commit?
- [ ] Did you run system tests on Hive (or Spark)?

Author: Makoto Yui <myui@apache.org>

Closes #241 from myui/kryo_update.
  • Loading branch information
myui committed May 2, 2021
1 parent 6ae5e4a commit 2621ca984eb4724be9eea2b87ded8a98e4e3391a
Showing 4 changed files with 65 additions and 8 deletions.
@@ -163,6 +163,12 @@
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.esotericsoftware.kryo</groupId>
<artifactId>kryo</artifactId>
<version>${kryo.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

</project>
@@ -32,6 +32,7 @@
import org.apache.hive.com.esotericsoftware.kryo.Kryo;
import org.apache.hive.com.esotericsoftware.kryo.io.Input;
import org.apache.hive.com.esotericsoftware.kryo.io.Output;
import org.objenesis.strategy.StdInstantiatorStrategy;

public final class TestUtils {

@@ -48,8 +49,11 @@ public static <T extends GenericUDF> void testGenericUDFSerialization(@Nonnull C
udf.initialize(ois);

// serialization after initialization
byte[] serialized = serializeObjectByKryo(udf);
deserializeObjectByKryo(serialized, clazz);
byte[] serialized1 = serializeObjectByKryo(udf);
deserializeObjectByKryo(serialized1, clazz);

byte[] serialized2 = serializeObjectByOriginalKryo(udf);
deserializeObjectByOriginalKryo(serialized2, clazz);

int size = row.length;
GenericUDF.DeferredObject[] rowDeferred = new GenericUDF.DeferredObject[size];
@@ -60,8 +64,11 @@ public static <T extends GenericUDF> void testGenericUDFSerialization(@Nonnull C
udf.evaluate(rowDeferred);

// serialization after evaluating row
serialized = serializeObjectByKryo(udf);
TestUtils.deserializeObjectByKryo(serialized, clazz);
serialized1 = serializeObjectByKryo(udf);
TestUtils.deserializeObjectByKryo(serialized1, clazz);

serialized2 = serializeObjectByOriginalKryo(udf);
TestUtils.deserializeObjectByOriginalKryo(serialized2, clazz);

udf.close();
}
@@ -82,6 +89,9 @@ public static <T extends GenericUDTF> void testGenericUDTFSerialization(@Nonnull
byte[] serialized = serializeObjectByKryo(udtf);
deserializeObjectByKryo(serialized, clazz);

byte[] serialized2 = serializeObjectByOriginalKryo(udtf);
deserializeObjectByOriginalKryo(serialized2, clazz);

udtf.setCollector(new Collector() {
public void collect(Object input) throws HiveException {
// noop
@@ -99,9 +109,12 @@ public void collect(Object input) throws HiveException {
udtf.close();
}

// -------------------------------
// Hive version of Kryo

@Nonnull
public static byte[] serializeObjectByKryo(@Nonnull Object obj) {
Kryo kryo = getKryo();
Kryo kryo = getHiveKryo();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
Output output = new Output(bos);
kryo.writeObject(output, obj);
@@ -111,16 +124,53 @@ public static byte[] serializeObjectByKryo(@Nonnull Object obj) {

@Nonnull
public static <T> T deserializeObjectByKryo(@Nonnull byte[] in, @Nonnull Class<T> clazz) {
Kryo kryo = getKryo();
Kryo kryo = getHiveKryo();
Input inp = new Input(in);
T t = kryo.readObject(inp, clazz);
inp.close();
return t;
}

@Nonnull
private static Kryo getKryo() {
private static Kryo getHiveKryo() {
return Utilities.runtimeSerializationKryo.get();
}

// -------------------------------
// esotericsoftware's original version of Kryo

@Nonnull
public static byte[] serializeObjectByOriginalKryo(@Nonnull Object obj) {
com.esotericsoftware.kryo.Kryo kryo = getOriginalKryo();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
com.esotericsoftware.kryo.io.Output output = new com.esotericsoftware.kryo.io.Output(bos);
kryo.writeObject(output, obj);
output.close();
return bos.toByteArray();
}

@Nonnull
public static <T> T deserializeObjectByOriginalKryo(@Nonnull byte[] in,
@Nonnull Class<T> clazz) {
com.esotericsoftware.kryo.Kryo kryo = getOriginalKryo();
com.esotericsoftware.kryo.io.Input inp = new com.esotericsoftware.kryo.io.Input(in);
T t = kryo.readObject(inp, clazz);
inp.close();
return t;
}

@Nonnull
private static com.esotericsoftware.kryo.Kryo getOriginalKryo() {
com.esotericsoftware.kryo.Kryo kryo = new com.esotericsoftware.kryo.Kryo();

// kryo.setReferences(true);
// kryo.setRegistrationRequired(false);

// see https://stackoverflow.com/a/23962797/5332768
((com.esotericsoftware.kryo.Kryo.DefaultInstantiatorStrategy) kryo.getInstantiatorStrategy()).setFallbackInstantiatorStrategy(
new StdInstantiatorStrategy());

return kryo;
}

}
@@ -272,6 +272,7 @@
<maven-enforcer.requireMavenVersion>[3.3.1,)</maven-enforcer.requireMavenVersion>
<surefire.version>2.19.1</surefire.version>
<netty.version>4.1.42.Final</netty.version>
<kryo.version>2.24.0</kryo.version>
</properties>

<distributionManagement>
@@ -97,7 +97,7 @@
<!-- NOTE: This dependency needed for xgboost4j -->
<groupId>com.esotericsoftware.kryo</groupId>
<artifactId>kryo</artifactId>
<version>2.21</version>
<version>${kryo.version}</version>
<scope>compile</scope>
</dependency>
<dependency>

0 comments on commit 2621ca9

Please sign in to comment.