From 693939cda31f67d4bab2351901d0dcebb40eb405 Mon Sep 17 00:00:00 2001
From: Marcelo Vanzin <vanzin@cloudera.com>
Date: Fri, 1 Jul 2016 15:31:43 -0700
Subject: [PATCH 1/2] [SPARK-16349][sql] Fall back to isolated class loader
 when classes not found.

Some Hadoop classes needed by the Hive metastore client jars are not present
in Spark's packaging (for example, "org/apache/hadoop/mapred/MRVersion"). So
if the parent class loader fails to find a class, try to load it from the
isolated class loader, in case it's available there.

I also took the opportunity to remove the HIVE_EXECUTION_VERSION constant since
it's not used anywhere.

Tested by setting spark.sql.hive.metastore.jars to local paths with Hive/Hadoop
libraries and verifying that Spark can talk to the metastore.
---
 .../scala/org/apache/spark/sql/hive/HiveUtils.scala  |  5 -----
 .../spark/sql/hive/client/IsolatedClientLoader.scala | 12 +++++++++---
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
index 9ed357c587c35..c094646b25a24 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
@@ -61,11 +61,6 @@ private[spark] object HiveUtils extends Logging {
     .stringConf
     .createWithDefault(hiveExecutionVersion)
 
-  val HIVE_EXECUTION_VERSION = SQLConfigBuilder("spark.sql.hive.version")
-    .doc("Version of Hive used internally by Spark SQL.")
-    .stringConf
-    .createWithDefault(hiveExecutionVersion)
-
   val HIVE_METASTORE_JARS = SQLConfigBuilder("spark.sql.hive.metastore.jars")
     .doc(s"""
       | Location of the jars that should be used to instantiate the HiveMetastoreClient.
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
index e1950d181d10e..a72a13b778e20 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
@@ -220,9 +220,15 @@ private[hive] class IsolatedClientLoader(
               logDebug(s"hive class: $name - ${getResource(classToPath(name))}")
               super.loadClass(name, resolve)
             } else {
-              // For shared classes, we delegate to baseClassLoader.
+              // For shared classes, we delegate to baseClassLoader, but fall back in case the
+              // class is not found.
               logDebug(s"shared class: $name")
-              baseClassLoader.loadClass(name)
+              try {
+                baseClassLoader.loadClass(name)
+              } catch {
+                case _: ClassNotFoundException =>
+                  super.loadClass(name, resolve)
+              }
             }
           }
         }
@@ -264,7 +270,7 @@ private[hive] class IsolatedClientLoader(
           throw new ClassNotFoundException(
             s"$cnf when creating Hive client using classpath: ${execJars.mkString(", ")}\n" +
             "Please make sure that jars for your version of hive and hadoop are included in the " +
-            s"paths passed to ${HiveUtils.HIVE_METASTORE_JARS}.", e)
+            s"paths passed to ${HiveUtils.HIVE_METASTORE_JARS.key}.", e)
         } else {
           throw e
         }

From 68f5a2357f592bbc969fe9effb95ab4f28b550f9 Mon Sep 17 00:00:00 2001
From: Marcelo Vanzin <vanzin@cloudera.com>
Date: Tue, 5 Jul 2016 10:48:43 -0700
Subject: [PATCH 2/2] Restore HIVE_EXECUTION_VERSION since tests rely on it
 implicitly.

---
 .../src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
index c094646b25a24..9ed357c587c35 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
@@ -61,6 +61,11 @@ private[spark] object HiveUtils extends Logging {
     .stringConf
     .createWithDefault(hiveExecutionVersion)
 
+  val HIVE_EXECUTION_VERSION = SQLConfigBuilder("spark.sql.hive.version")
+    .doc("Version of Hive used internally by Spark SQL.")
+    .stringConf
+    .createWithDefault(hiveExecutionVersion)
+
   val HIVE_METASTORE_JARS = SQLConfigBuilder("spark.sql.hive.metastore.jars")
     .doc(s"""
       | Location of the jars that should be used to instantiate the HiveMetastoreClient.