Merge 47489fc into 72c2361

apache · Oct 10, 2018 · d40f006 · d40f006
2 parents 72c2361 + 47489fc
commit d40f006
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 3 deletions.
diff --git a/build/script/download-spark.sh b/build/script/download-spark.sh
@@ -27,8 +27,8 @@ if [[ `uname -a` =~ "Darwin" ]]; then
     alias md5cmd="md5 -q"
 fi
 
-spark_version="2.1.2"
-spark_pkg_md5="b47cf199bf78042a0471b9a5ae0ceeac"
+spark_version="2.3.2"
+spark_pkg_md5="01b3e6aef0ef574e066d15e035b9b2dd"
 
 if [ ! -f "build/spark-${spark_version}-bin-hadoop2.7.tgz" ]
 then

diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
@@ -111,6 +111,9 @@ public void registerClasses(Kryo kryo) {
         addClassQuitely(kyroClasses, "org.apache.spark.sql.catalyst.expressions.GenericInternalRow");
         addClassQuitely(kyroClasses, "org.apache.spark.unsafe.types.UTF8String");
 
+        addClassQuitely(kyroClasses, "org.apache.spark.internal.io.FileCommitProtocol$TaskCommitMessage");
+        addClassQuitely(kyroClasses, "scala.collection.immutable.Set$EmptySet$");
+
         for (Class kyroClass : kyroClasses) {
             kryo.register(kyroClass);
         }

diff --git a/pom.xml b/pom.xml
@@ -60,7 +60,7 @@
     <kafka.version>1.0.0</kafka.version>
 
     <!-- Spark versions -->
-    <spark.version>2.1.2</spark.version>
+    <spark.version>2.3.2</spark.version>
     <kryo.version>4.0.0</kryo.version>
 
     <!-- mysql versions -->