From 77cb7b9056416869fd6182b2b3f424ae8f38235a Mon Sep 17 00:00:00 2001
From: Lee moon soo <moon@apache.org>
Date: Wed, 20 May 2015 13:34:27 +0900
Subject: [PATCH] Spark 1.4 support

---
 README.md                                     |  4 ++
 pom.xml                                       | 60 ++++++++++++++++++-
 spark/pom.xml                                 |  1 -
 .../zeppelin/spark/PySparkInterpreter.java    |  5 +-
 .../zeppelin/spark/SparkInterpreter.java      | 10 ++++
 .../zeppelin/spark/SparkSqlInterpreter.java   |  2 +
 6 files changed, 77 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 46e188ca09b..44f50eedd4d 100644
--- a/README.md
+++ b/README.md
@@ -54,6 +54,10 @@ Spark 1.3.x
 ```
 mvn clean package -Pspark-1.3 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests
 ```
+Spark 1.4.x
+```
+mvn clean package -Pspark-1.4 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests
+```
 CDH 5.X
 ```
 mvn clean package -Pspark-1.2 -Dhadoop.version=2.5.0-cdh5.3.0 -Phadoop-2.4 -DskipTests
diff --git a/pom.xml b/pom.xml
index b4f78dfc272..e43ccabbff7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,6 +126,7 @@
     <jets3t.version>0.7.1</jets3t.version>
     <commons.math3.version>3.2</commons.math3.version>
     <commons.httpclient.version>4.3.6</commons.httpclient.version>
+    <fasterxml.jackson.version>2.4.4</fasterxml.jackson.version>
     <codehaus.jackson.version>1.8.8</codehaus.jackson.version>
     <snappy.version>1.0.5</snappy.version>
     <io.netty.version>4.0.17.Final</io.netty.version>
@@ -811,6 +812,26 @@
         <version>${codehaus.jackson.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>com.fasterxml.jackson.core</groupId>
+        <artifactId>jackson-databind</artifactId>
+        <version>${fasterxml.jackson.version}</version>
+      </dependency>
+
+      <!-- Guava is excluded because of SPARK-6149.  The Guava version referenced in this module is
+           15.0, which causes runtime incompatibility issues. -->
+      <dependency>
+        <groupId>com.fasterxml.jackson.module</groupId>
+        <artifactId>jackson-module-scala_2.10</artifactId>
+        <version>${fasterxml.jackson.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
+
       <dependency>
         <groupId>com.thoughtworks.paranamer</groupId>
         <artifactId>paranamer</artifactId>
@@ -1317,8 +1338,6 @@
 
     <profile>
       <id>spark-1.3</id>
-      <dependencies>
-      </dependencies>
       <properties>
         <akka.version>2.3.4-spark</akka.version>
         <spark.version>1.3.0</spark.version>
@@ -1343,8 +1362,45 @@
         <snappy.version>1.1.1.6</snappy.version>
         <mesos.version>0.21.0</mesos.version>
       </properties>
+
+      <dependencies>
+      </dependencies>
+
     </profile>
 
+    <profile>
+      <id>spark-1.4</id>
+      <properties>
+        <akka.version>2.3.4-spark</akka.version>
+        <spark.version>1.4.0</spark.version>
+        <mesos.version>0.21.1</mesos.version>
+        <hbase.version>0.98.7-hadoop2</hbase.version>
+        <hbase.artifact>hbase</hbase.artifact>
+        <hive.group>org.spark-project.hive</hive.group>
+        <hive.version>0.13.1a</hive.version>
+        <derby.version>10.10.1.1</derby.version>
+        <orbit.version>3.0.0.v201112011016</orbit.version>
+        <parquet.version>1.6.0rc3</parquet.version>
+        <chill.version>0.5.0</chill.version>
+        <ivy.version>2.4.0</ivy.version>
+        <oro.version>2.0.8</oro.version>
+        <avro.mapred.classifier>hadoop2</avro.mapred.classifier>
+        <codahale.metrics.version>3.1.0</codahale.metrics.version>
+        <commons.httpclient.version>4.3.2</commons.httpclient.version>
+        <commons.math3.version>3.4.1</commons.math3.version>
+        <io.netty.version>4.0.23.Final</io.netty.version>
+        <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
+        <fasterxml.jackson.version>2.4.4</fasterxml.jackson.version>
+        <snappy.version>1.1.1.7</snappy.version>
+        <mesos.version>0.21.0</mesos.version>
+      </properties>
+
+      <dependencies>
+      </dependencies>
+
+    </profile>
+
+
     <profile>
       <id>hadoop-0.23</id>
       <!-- SPARK-1121: Adds an explicit dependency on Avro to work around a
diff --git a/spark/pom.xml b/spark/pom.xml
index 72f216dee82..b4c05e595b8 100644
--- a/spark/pom.xml
+++ b/spark/pom.xml
@@ -76,7 +76,6 @@
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
-      <version>2.3.0</version>
     </dependency>
 
     <dependency>
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
index 9fce0933a07..95eefd809c8 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
@@ -48,9 +48,9 @@
 import org.apache.zeppelin.interpreter.InterpreterGroup;
 import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
 import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
 import org.apache.zeppelin.interpreter.WrappedInterpreter;
-import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -298,7 +298,8 @@ public InterpreterResult interpret(String st, InterpreterContext context) {
 
     SparkInterpreter sparkInterpreter = getSparkInterpreter();
     if (!sparkInterpreter.getSparkContext().version().startsWith("1.2") &&
-        !sparkInterpreter.getSparkContext().version().startsWith("1.3")) {
+        !sparkInterpreter.getSparkContext().version().startsWith("1.3") &&
+        !sparkInterpreter.getSparkContext().version().startsWith("1.4")) {
       return new InterpreterResult(Code.ERROR, "pyspark "
           + sparkInterpreter.getSparkContext().version() + " is not supported");
     }
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
index c875e8557b8..731068e48ed 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
@@ -415,6 +415,10 @@ public void open() {
         Method loadFiles = this.interpreter.getClass().getMethod(
             "org$apache$spark$repl$SparkILoop$$loadFiles", Settings.class);
         loadFiles.invoke(this.interpreter, settings);
+      } else if (sc.version().startsWith("1.4")) {
+        Method loadFiles = this.interpreter.getClass().getMethod(
+            "org$apache$spark$repl$SparkILoop$$loadFiles", Settings.class);
+        loadFiles.invoke(this.interpreter, settings);
       }
     } catch (NoSuchMethodException | SecurityException | IllegalAccessException
         | IllegalArgumentException | InvocationTargetException e) {
@@ -447,6 +451,10 @@ public void open() {
       intp.interpret("import sqlContext.implicits._");
       intp.interpret("import sqlContext.sql");
       intp.interpret("import org.apache.spark.sql.functions._");
+    } else if (sc.version().startsWith("1.4")) {
+      intp.interpret("import sqlContext.implicits._");
+      intp.interpret("import sqlContext.sql");
+      intp.interpret("import org.apache.spark.sql.functions._");
     }
 
     // add jar
@@ -621,6 +629,8 @@ public int getProgress(InterpreterContext context) {
           progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage());
         } else if (sc.version().startsWith("1.3")) {
           progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage());
+        } else if (sc.version().startsWith("1.4")) {
+          progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage());
         } else {
           continue;
         }
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java
index 618579d0eaa..3138fbc4102 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java
@@ -171,6 +171,8 @@ public int getProgress(InterpreterContext context) {
           progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage());
         } else if (sc.version().startsWith("1.3")) {
           progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage());
+        } else if (sc.version().startsWith("1.4")) {
+          progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage());
         } else {
           logger.warn("Spark {} getting progress information not supported" + sc.version());
           continue;