From 0fab8c74977927be9a505025754b39fcbef9d614 Mon Sep 17 00:00:00 2001 From: huangzhaowei Date: Thu, 8 Oct 2015 15:38:10 +0800 Subject: [PATCH 1/2] [SPARK-11000][YARN]Bug fix: Derby have booted the database twice in yarn security mode. --- .../org/apache/spark/deploy/yarn/Client.scala | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index eb3b7fb885087..6bb4552483f1c 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -1272,11 +1272,24 @@ object Client extends Logging { val mirror = universe.runtimeMirror(getClass.getClassLoader) try { - val hiveClass = mirror.classLoader.loadClass("org.apache.hadoop.hive.ql.metadata.Hive") - val hive = hiveClass.getMethod("get").invoke(null) - - val hiveConf = hiveClass.getMethod("getConf").invoke(hive) val hiveConfClass = mirror.classLoader.loadClass("org.apache.hadoop.hive.conf.HiveConf") + val hiveConf = hiveConfClass.newInstance() + + // Set metastore to be a local temp directory to avoid conflict of the `metaStore client` + // in `HiveContext` which will use the same derby dataBase by default. + val hiveConfSet = (param: String, value: String) => hiveConfClass + .getMethod("set", classOf[Unit]) + .invoke(hiveConf, param, value) + val tempDir = Utils.createTempDir() + val localMetastore = new File(tempDir, "metastore") + hiveConfSet("hive.metastore.warehouse.dir", localMetastore.toURI.toString) + hiveConfSet("javax.jdo.option.ConnectionURL", + s"jdbc:derby:;databaseName=${localMetastore.getAbsolutePath};create=true") + hiveConfSet("datanucleus.rdbms.datastoreAdapterClassName", + "org.datanucleus.store.rdbms.adapter.DerbyAdapter") + + val hiveClass = mirror.classLoader.loadClass("org.apache.hadoop.hive.ql.metadata.Hive") + val hive = hiveClass.getMethod("get").invoke(null, hiveConf.asInstanceOf[Object]) val hiveConfGet = (param: String) => Option(hiveConfClass .getMethod("get", classOf[java.lang.String]) From 58dcf4da1d0680e4e2ccc0fc9b62f232ccb1b560 Mon Sep 17 00:00:00 2001 From: huangzhaowei Date: Fri, 16 Oct 2015 10:01:16 +0800 Subject: [PATCH 2/2] Load meta.Hive in the 'if' condition. --- .../org/apache/spark/deploy/yarn/Client.scala | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 6bb4552483f1c..791d1f3ff649d 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -1275,22 +1275,6 @@ object Client extends Logging { val hiveConfClass = mirror.classLoader.loadClass("org.apache.hadoop.hive.conf.HiveConf") val hiveConf = hiveConfClass.newInstance() - // Set metastore to be a local temp directory to avoid conflict of the `metaStore client` - // in `HiveContext` which will use the same derby dataBase by default. - val hiveConfSet = (param: String, value: String) => hiveConfClass - .getMethod("set", classOf[Unit]) - .invoke(hiveConf, param, value) - val tempDir = Utils.createTempDir() - val localMetastore = new File(tempDir, "metastore") - hiveConfSet("hive.metastore.warehouse.dir", localMetastore.toURI.toString) - hiveConfSet("javax.jdo.option.ConnectionURL", - s"jdbc:derby:;databaseName=${localMetastore.getAbsolutePath};create=true") - hiveConfSet("datanucleus.rdbms.datastoreAdapterClassName", - "org.datanucleus.store.rdbms.adapter.DerbyAdapter") - - val hiveClass = mirror.classLoader.loadClass("org.apache.hadoop.hive.ql.metadata.Hive") - val hive = hiveClass.getMethod("get").invoke(null, hiveConf.asInstanceOf[Object]) - val hiveConfGet = (param: String) => Option(hiveConfClass .getMethod("get", classOf[java.lang.String]) .invoke(hiveConf, param)) @@ -1299,6 +1283,9 @@ object Client extends Logging { // Check for local metastore if (metastore_uri != None && metastore_uri.get.toString.size > 0) { + val hiveClass = mirror.classLoader.loadClass("org.apache.hadoop.hive.ql.metadata.Hive") + val hive = hiveClass.getMethod("get").invoke(null, hiveConf.asInstanceOf[Object]) + val metastore_kerberos_principal_conf_var = mirror.classLoader .loadClass("org.apache.hadoop.hive.conf.HiveConf$ConfVars") .getField("METASTORE_KERBEROS_PRINCIPAL").get("varname").toString