Only set datanucleus.schema.autoCreateAll=true in non-production envi…

…ronment
apache · May 10, 2019 · 4372759 · 4372759
1 parent 994b5a7
commit 4372759
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 7 deletions.
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+from __future__ import print_function
 from functools import total_ordering
 import itertools
 import re
@@ -24,10 +25,11 @@
     hadoop_version = os.environ.get("AMPLAB_JENKINS_BUILD_PROFILE", "hadoop2.7")
 else:
     hadoop_version = os.environ.get("HADOOP_PROFILE", "hadoop2.7")
-print "[info] Choosing supported modules with Hadoop profile", hadoop_version
+print("[info] Choosing supported modules with Hadoop profile", hadoop_version)
 
 all_modules = []
 
+
 @total_ordering
 class Module(object):
     """
@@ -78,8 +80,9 @@ def __init__(self, name, dependencies, source_file_regexes, build_profile_flags=
         self.dependent_modules = set()
         for dep in dependencies:
             dep.dependent_modules.add(self)
+        # TODO: Skip hive-thriftserver module for hadoop-3.2. remove this once hadoop-3.2 support it
         if name == "hive-thriftserver" and hadoop_version == "hadoop3.2":
-            print "[info] Skip unsupported module:", name
+            print("[info] Skip unsupported module:", name)
         else:
             all_modules.append(self)
 

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -1371,7 +1371,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
       // if (isUsingHiveMetastore) {
       //  assert(storageFormat.properties.get("path") === expected)
       // }
-      assert(Some(storageFormat.locationUri.get.getPath) === Some(expected.getPath))
+      assert(storageFormat.locationUri.map(_.getPath) === Some(expected.getPath))
     }
     // set table location
     sql("ALTER TABLE dbx.tab1 SET LOCATION '/path/to/your/lovely/heart'")

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.Path
 import org.apache.hadoop.hive.common.StatsSetupConst
 import org.apache.hadoop.hive.conf.HiveConf
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars.{METASTORECONNECTURLKEY, METASTOREURIS}
 import org.apache.hadoop.hive.metastore.{TableType => HiveTableType}
 import org.apache.hadoop.hive.metastore.api.{Database => HiveDatabase, FieldSchema, Order}
 import org.apache.hadoop.hive.metastore.api.{SerDeInfo, StorageDescriptor}
@@ -196,10 +197,16 @@ private[hive] class HiveClientImpl(
     // Hive changed the default of datanucleus.schema.autoCreateAll from true to false
     // and hive.metastore.schema.verification from false to true since Hive 2.0.
     // For details, see the JIRA HIVE-6113, HIVE-12463 and HIVE-1841.
-    // We hard-code these configurations here to allow bin/spark-shell, bin/spark-sql
-    // and sbin/start-thriftserver.sh automatically creates Derby metastore.
-    hiveConf.setBoolean("hive.metastore.schema.verification", false)
-    hiveConf.setBoolean("datanucleus.schema.autoCreateAll", true)
+    // For the production environment. Either isDefaultMSUri or isDerbyMS should not be true.
+    // We hard-code hive.metastore.schema.verification and datanucleus.schema.autoCreateAll to allow
+    // bin/spark-shell, bin/spark-sql and sbin/start-thriftserver.sh to automatically create the
+    // Derby Metastore when running Spark in the non-production environment.
+    val isDefaultMSUri = hiveConf.getVar(METASTOREURIS).equals(METASTOREURIS.defaultStrVal)
+    val isDerbyMS = hiveConf.getVar(METASTORECONNECTURLKEY).startsWith("jdbc:derby")
+    if (isDefaultMSUri && isDerbyMS) {
+      hiveConf.setBoolean("hive.metastore.schema.verification", false)
+      hiveConf.setBoolean("datanucleus.schema.autoCreateAll", true)
+    }
     hiveConf
   }