Merge 6047621 into 9a8e636

pom.xml

@@ -19,8 +19,9 @@
   </scm>
 
   <properties>
-    <hadoop.version>2.7.2</hadoop.version>
-    <hive.version>2.3.7</hive.version>
+    <hadoop.version>3.1.0</hadoop.version>
+    <hive.version>3.1.2</hive.version>
+    <tez.version>0.9.1</tez.version>
     <jdk.version>1.8</jdk.version>
     <junit.jupiter.version>5.7.0</junit.jupiter.version>
     <junit.platform.version>1.3.2</junit.platform.version>
@@ -94,6 +95,16 @@
       <artifactId>hive-jdbc</artifactId>
       <version>${hive.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-common</artifactId>
+      <version>${tez.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-dag</artifactId>
+      <version>${tez.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.hamcrest</groupId>
       <artifactId>hamcrest</artifactId>
@@ -174,6 +185,16 @@
           </dependency>
         </dependencies>
       </plugin>
+      <plugin>
+        <groupId>org.jacoco</groupId>
+        <artifactId>jacoco-maven-plugin</artifactId>
+        <configuration>
+          <excludes>
+            <!-- If this class is not excluded, Jacoco throws a MethodTooLargeException when instrumenting it -->
+            <exclude>org/apache/hadoop/hive/ql/parse/HiveParser</exclude>
+          </excludes>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 

src/main/java/com/hotels/beeju/BeejuJUnitRule.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2015-2020 Expedia, Inc.
+ * Copyright (C) 2015-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/java/com/hotels/beeju/HiveMetaStoreJUnitRule.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2015-2020 Expedia, Inc.
+ * Copyright (C) 2015-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,6 +15,9 @@
  */
 package com.hotels.beeju;
 
+
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.CONNECT_URL_KEY;
+
 import java.util.Map;
 
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
@@ -72,6 +75,7 @@ public HiveMetaStoreJUnitRule(
 
   @Override
   protected void before() throws Throwable {
+    System.clearProperty(CONNECT_URL_KEY.getVarname());
     super.before();
     hiveMetaStoreCore.initialise();
   }

src/main/java/com/hotels/beeju/HiveServer2JUnitRule.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2015-2020 Expedia, Inc.
+ * Copyright (C) 2015-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,6 +15,9 @@
  */
 package com.hotels.beeju;
 
+
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.CONNECT_URL_KEY;
+
 import java.util.Map;
 
 import org.apache.hive.jdbc.HiveDriver;
@@ -64,6 +67,7 @@ public HiveServer2JUnitRule(String databaseName, Map<String, String> configurati
 
   @Override
   protected void before() throws Throwable {
+    System.clearProperty(CONNECT_URL_KEY.getVarname());
     hiveServer2Core.startServerSocket();
     super.before();
     hiveServer2Core.initialise();

src/main/java/com/hotels/beeju/ThriftHiveMetaStoreJUnitRule.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2015-2020 Expedia, Inc.
+ * Copyright (C) 2015-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,6 +15,9 @@
  */
 package com.hotels.beeju;
 
+
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.CONNECT_URL_KEY;
+
 import java.util.Map;
 
 import com.hotels.beeju.core.ThriftHiveMetaStoreCore;
@@ -69,6 +72,7 @@ public ThriftHiveMetaStoreJUnitRule(String databaseName, Map<String, String> pre
 
   @Override
   protected void before() throws Throwable {
+    System.clearProperty(CONNECT_URL_KEY.getVarname());
     thriftHiveMetaStoreCore.initialise();
     super.before();
   }

src/main/java/com/hotels/beeju/core/BeejuCore.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2015-2020 Expedia, Inc.
+ * Copyright (C) 2015-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,6 +15,17 @@
  */
 package com.hotels.beeju.core;
 
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_SERVER2_MATERIALIZED_VIEWS_REGISTRY_IMPL;
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT;
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.AUTO_CREATE_ALL;
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.CONNECTION_DRIVER;
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.CONNECTION_USER_NAME;
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.CONNECT_URL_KEY;
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.EVENT_DB_NOTIFICATION_API_AUTH;
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.HMS_HANDLER_FORCE_RELOAD_CONF;
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.PWD;
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.SCHEMA_VERIFICATION;
+
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import java.io.File;
@@ -27,13 +38,15 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.UUID;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.derby.jdbc.EmbeddedDriver;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.thrift.TException;
 
 public class BeejuCore {
@@ -78,20 +91,50 @@ public BeejuCore(String databaseName, Map<String, String> preConfiguration, Map<
     checkNotNull(databaseName, "databaseName is required");
     this.databaseName = databaseName;
     configure(preConfiguration);
-    
+
     driverClassName = EmbeddedDriver.class.getName();
-    conf.setBoolean("hcatalog.hive.client.cache.disabled", true);
     connectionURL = "jdbc:derby:memory:" + UUID.randomUUID() + ";create=true";
-    conf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, connectionURL);
-    conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, driverClassName);
-    conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME, METASTORE_DB_USER);
-    conf.setVar(HiveConf.ConfVars.METASTOREPWD, METASTORE_DB_PASSWORD);
-    conf.setBoolVar(HiveConf.ConfVars.HMSHANDLERFORCERELOADCONF, true);
+
+    System.setProperty("derby.system.home", "metastore_db_parent_" + UUID.randomUUID());
+
+    // This should NOT be set as a system property too
+    // conf.set(CONNECT_URL_KEY.getVarname(), connectionURL);
+    setMetastoreAndSystemProperty(CONNECT_URL_KEY, connectionURL);
+
+    setMetastoreAndSystemProperty(CONNECTION_DRIVER, driverClassName);
+    setMetastoreAndSystemProperty(CONNECTION_USER_NAME, METASTORE_DB_USER);
+    setMetastoreAndSystemProperty(PWD, METASTORE_DB_PASSWORD);
+
+    conf.setBoolean("hcatalog.hive.client.cache.disabled", true);
+
+    setMetastoreAndSystemProperty(HMS_HANDLER_FORCE_RELOAD_CONF, "true");
     // Hive 2.x compatibility
-    conf.setBoolean("datanucleus.schema.autoCreateAll", true);
-    conf.setBoolean("hive.metastore.schema.verification", false);
-    // override default port as some of our test environments claim it is in use.
-    conf.setInt("hive.server2.webui.port", 0); // ConfVars.HIVE_SERVER2_WEBUI_PORT
+    setMetastoreAndSystemProperty(AUTO_CREATE_ALL, "true");
+    setMetastoreAndSystemProperty(SCHEMA_VERIFICATION, "false");
+
+    // Used to prevent "Not authorized to make the get_current_notificationEventId call" errors
+    setMetastoreAndSystemProperty(EVENT_DB_NOTIFICATION_API_AUTH, "false");
+
+    // Used to prevent "Error polling for notification events" error
+    conf.setTimeVar(HiveConf.ConfVars.HIVE_NOTFICATION_EVENT_POLL_INTERVAL, 0, TimeUnit.MILLISECONDS);
+
+    // Has to be added to exclude failures related to the HiveMaterializedViewsRegistry
+    conf.set(HIVE_SERVER2_MATERIALIZED_VIEWS_REGISTRY_IMPL.varname, "DUMMY");
+    System.setProperty(HIVE_SERVER2_MATERIALIZED_VIEWS_REGISTRY_IMPL.varname, "DUMMY");
+
+    // Override default port as some of our test environments claim it is in use.
+    // This should not be equal to 0 as this would actually disable the WebUI and potentially
+    // cause errors.
+    conf.setInt(HIVE_SERVER2_WEBUI_PORT.varname, 0); // ConfVars.HIVE_SERVER2_WEBUI_PORT
+
+    // TODO: check if necessary or not
+//    setMetastoreAndSystemProperty(HIVE_IN_TEST, "true");
+//    setMetastoreAndSystemProperty(CONNECTION_POOLING_TYPE, "NONE");
+//    setMetastoreAndSystemProperty(HIVE_SUPPORT_CONCURRENCY, "false");
+
+//    setMetastoreAndSystemProperty(MULTITHREADED, "false");
+//    setMetastoreAndSystemProperty(NON_TRANSACTIONAL_READ, "false");
+//    setMetastoreAndSystemProperty(DATANUCLEUS_TRANSACTION_ISOLATION, "serializable");
 
     try {
       // overriding default derby log path to go to tmp
@@ -106,6 +149,14 @@ public BeejuCore(String databaseName, Map<String, String> preConfiguration, Map<
 
     configure(postConfiguration);
   }
+
+  private void setMetastoreAndSystemProperty(MetastoreConf.ConfVars key, String value) {
+    conf.set(key.getVarname(), value);
+    conf.set(key.getHiveName(), value);
+
+    System.setProperty(key.getVarname(), value);
+    System.setProperty(key.getHiveName(), value);
+  }
 
   private void configure(Map<String, String> customConfiguration) {
     if (customConfiguration != null) {

src/main/java/com/hotels/beeju/core/HiveMetaStoreCore.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2015-2019 Expedia, Inc.
+ * Copyright (C) 2015-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/java/com/hotels/beeju/core/HiveServer2Core.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2015-2019 Expedia, Inc.
+ * Copyright (C) 2015-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/java/com/hotels/beeju/core/ThriftHiveMetaStoreCore.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2015-2020 Expedia, Inc.
+ * Copyright (C) 2015-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,11 +27,12 @@
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.HiveMetaStore;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
-import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge23;
+import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
+import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge23;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+
 public class ThriftHiveMetaStoreCore {
 
   private static final Logger LOG = LoggerFactory.getLogger(ThriftHiveMetaStoreCore.class);
@@ -57,15 +58,12 @@ public void initialise() throws Exception {
     }
     beejuCore.setHiveVar(HiveConf.ConfVars.METASTOREURIS, getThriftConnectionUri());
     final HiveConf hiveConf = new HiveConf(beejuCore.conf(), HiveMetaStoreClient.class);
-    thriftServer.execute(new Runnable() {
-      @Override
-      public void run() {
-        try {
-          HadoopThriftAuthBridge bridge = new HadoopThriftAuthBridge23();
-          HiveMetaStore.startMetaStore(thriftPort, bridge, hiveConf, startLock, startCondition, startedServing);
-        } catch (Throwable e) {
-          LOG.error("Unable to start a Thrift server for Hive Metastore", e);
-        }
+    thriftServer.execute(() -> {
+      try {
+        HadoopThriftAuthBridge bridge = HadoopThriftAuthBridge23.getBridge();
+        HiveMetaStore.startMetaStore(thriftPort, bridge, hiveConf, startLock, startCondition, startedServing);
+      } catch (Throwable e) {
+        LOG.error("Unable to start a Thrift server for Hive Metastore", e);
       }
     });
     int i = 0;

src/main/java/com/hotels/beeju/extensions/BeejuJUnitExtension.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2015-2019 Expedia, Inc.
+ * Copyright (C) 2015-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/java/com/hotels/beeju/extensions/HiveMetaStoreJUnitExtension.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2015-2019 Expedia, Inc.
+ * Copyright (C) 2015-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,6 +15,9 @@
  */
 package com.hotels.beeju.extensions;
 
+
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.CONNECT_URL_KEY;
+
 import java.util.Map;
 
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
@@ -61,6 +64,7 @@ public HiveMetaStoreJUnitExtension(String databaseName, Map<String, String> conf
 
   @Override
   public void beforeEach(ExtensionContext context) throws Exception {
+    System.clearProperty(CONNECT_URL_KEY.getVarname());
     super.beforeEach(context);
     hiveMetaStoreCore.initialise();
   }

src/main/java/com/hotels/beeju/extensions/HiveServer2JUnitExtension.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2015-2019 Expedia, Inc.
+ * Copyright (C) 2015-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,6 +15,9 @@
  */
 package com.hotels.beeju.extensions;
 
+
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.CONNECT_URL_KEY;
+
 import java.util.Map;
 
 import org.apache.hive.jdbc.HiveDriver;
@@ -55,6 +58,7 @@ public HiveServer2JUnitExtension(String databaseName, Map<String, String> config
 
   @Override
   public void beforeEach(ExtensionContext context) throws Exception {
+    System.clearProperty(CONNECT_URL_KEY.getVarname());
     hiveServer2Core.startServerSocket();
     super.beforeEach(context);
     hiveServer2Core.initialise();

src/main/java/com/hotels/beeju/extensions/ThriftHiveMetaStoreJUnitExtension.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2015-2020 Expedia, Inc.
+ * Copyright (C) 2015-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,6 +15,9 @@
  */
 package com.hotels.beeju.extensions;
 
+
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.CONNECT_URL_KEY;
+
 import java.util.Map;
 
 import org.junit.jupiter.api.extension.ExtensionContext;
@@ -60,7 +63,8 @@ public ThriftHiveMetaStoreJUnitExtension(String databaseName, Map<String, String
   }
 
   @Override
-  public void beforeEach(ExtensionContext context) throws Exception{
+  public void beforeEach(ExtensionContext context) throws Exception {
+    System.clearProperty(CONNECT_URL_KEY.getVarname());
     thriftHiveMetaStoreCore.initialise();
     super.beforeEach(context);
   }