[BEAM-3875] Update Spark runner to use Spark version 2.3.0

apache · Mar 28, 2018 · d62db87 · d62db87
1 parent 25e4a5b
commit d62db87
Show file tree

Hide file tree

Showing 9 changed files with 8 additions and 32 deletions.
diff --git a/build.gradle b/build.gradle
@@ -39,7 +39,7 @@ def grpc_google_common_protos = "0.1.9"
 def hamcrest_version = "1.3"
 def hadoop_version = "2.7.3"
 def jackson_version = "2.8.9"
-def spark_version = "2.2.1"
+def spark_version = "2.3.0"
 def pubsub_grpc_version = "0.1.18"
 def apex_core_version = "3.6.0"
 def apex_malhar_version = "3.4.0"

diff --git a/examples/java/pom.xml b/examples/java/pom.xml
@@ -112,7 +112,7 @@
       <id>spark-runner</id>
       <properties>
         <!-- Spark uses a different netty version than Beam's. -->
-        <netty.version>4.0.43.Final</netty.version>
+        <netty.version>4.1.17.Final</netty.version>
       </properties>
       <dependencies>
         <dependency>

diff --git a/pom.xml b/pom.xml
@@ -148,7 +148,7 @@
     <pubsub.version>v1-rev10-1.22.0</pubsub.version>
     <slf4j.version>1.7.25</slf4j.version>
     <spanner.version>0.20.0b-beta</spanner.version>
-    <spark.version>2.2.1</spark.version>
+    <spark.version>2.3.0</spark.version>
     <spring.version>4.3.5.RELEASE</spring.version>
     <stax2.version>3.1.4</stax2.version>
     <storage.version>v1-rev71-1.22.0</storage.version>

diff --git a/...s/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/package-info.java b/...s/spark/src/main/java/org/apache/beam/runners/spark/aggregators/metrics/package-info.java
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TranslationUtils.java
@@ -385,11 +385,9 @@ public static Map<TupleTag<?>, Coder<WindowedValue<?>>> getTupleTagCoders(
   /**
    * checking if we can avoid Serialization - relevant to RDDs. DStreams are memory ser in spark.
    * @param level StorageLevel required
-   * @return
+   * @return true if the level is memory only
    */
   public static boolean avoidRddSerialization(StorageLevel level) {
     return level.equals(StorageLevel.MEMORY_ONLY()) || level.equals(StorageLevel.MEMORY_ONLY_2());
   }
-
-
 }
diff --git a/sdks/java/io/hadoop-input-format/pom.xml b/sdks/java/io/hadoop-input-format/pom.xml
@@ -69,7 +69,7 @@
     <profile>
       <id>spark-runner</id>
       <properties>
-        <netty.version>4.0.43.Final</netty.version>
+        <netty.version>4.1.17.Final</netty.version>
       </properties>
       <dependencies>
         <dependency>

diff --git a/sdks/java/io/jdbc/pom.xml b/sdks/java/io/jdbc/pom.xml
@@ -41,7 +41,7 @@
     <profile>
       <id>spark-runner</id>
       <properties>
-        <netty.version>4.0.43.Final</netty.version>
+        <netty.version>4.1.17.Final</netty.version>
       </properties>
       <dependencies>
         <dependency>

diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/pom.xml
@@ -244,7 +244,7 @@
       <!-- Makes the SparkRunner available when running a pipeline. Additionally,
            overrides some Spark dependencies to Beam-compatible versions. -->
       <properties>
-        <netty.version>4.0.43.Final</netty.version>
+        <netty.version>4.1.17.Final</netty.version>
       </properties>
       <dependencies>
         <dependency>

diff --git a/sdks/java/nexmark/pom.xml b/sdks/java/nexmark/pom.xml
@@ -75,7 +75,7 @@
       <id>spark-runner</id>
       <properties>
         <!-- Spark uses a different netty version than Beam's. -->
-        <netty.version>4.0.43.Final</netty.version>
+        <netty.version>4.1.17.Final</netty.version>
       </properties>
       <dependencies>
         <dependency>