Merge 1cf09a8 into 9a8e636

pom.xml

@@ -19,8 +19,9 @@
   </scm>
 
   <properties>
-    <hadoop.version>2.7.2</hadoop.version>
-    <hive.version>2.3.7</hive.version>
+    <hadoop.version>3.1.0</hadoop.version>
+    <hive.version>3.1.2</hive.version>
+    <tez.version>0.9.1</tez.version>
     <jdk.version>1.8</jdk.version>
     <junit.jupiter.version>5.7.0</junit.jupiter.version>
     <junit.platform.version>1.3.2</junit.platform.version>
@@ -94,6 +95,16 @@
       <artifactId>hive-jdbc</artifactId>
       <version>${hive.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-common</artifactId>
+      <version>${tez.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-dag</artifactId>
+      <version>${tez.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.hamcrest</groupId>
       <artifactId>hamcrest</artifactId>

src/main/java/com/hotels/beeju/BeejuJUnitRule.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2015-2020 Expedia, Inc.
+ * Copyright (C) 2015-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -47,11 +47,19 @@ public BeejuJUnitRule(
 
   @Override
   protected void before() throws Throwable {
+    NoExitSecurityManager securityManager = new NoExitSecurityManager();
+    securityManager.setPolicy();
+    System.setSecurityManager(securityManager);
+
     createDatabase(databaseName());
   }
 
   @Override
   protected void after() {
+    NoExitSecurityManager securityManager = new NoExitSecurityManager();
+    securityManager.setPolicy();
+    System.setSecurityManager(securityManager);
+
     try {
       core.cleanUp();
     } catch (IOException e) {

src/main/java/com/hotels/beeju/HiveMetaStoreJUnitRule.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2015-2020 Expedia, Inc.
+ * Copyright (C) 2015-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,6 +15,8 @@
  */
 package com.hotels.beeju;
 
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.CONNECT_URL_KEY;
+
 import java.util.Map;
 
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
@@ -72,6 +74,7 @@ public HiveMetaStoreJUnitRule(
 
   @Override
   protected void before() throws Throwable {
+    System.clearProperty(CONNECT_URL_KEY.getVarname());
     super.before();
     hiveMetaStoreCore.initialise();
   }

src/main/java/com/hotels/beeju/HiveServer2JUnitRule.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2015-2020 Expedia, Inc.
+ * Copyright (C) 2015-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,6 +15,8 @@
  */
 package com.hotels.beeju;
 
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.CONNECT_URL_KEY;
+
 import java.util.Map;
 
 import org.apache.hive.jdbc.HiveDriver;
@@ -64,6 +66,7 @@ public HiveServer2JUnitRule(String databaseName, Map<String, String> configurati
 
   @Override
   protected void before() throws Throwable {
+    System.clearProperty(CONNECT_URL_KEY.getVarname());
     hiveServer2Core.startServerSocket();
     super.before();
     hiveServer2Core.initialise();

src/main/java/com/hotels/beeju/NoExitSecurityManager.java

@@ -0,0 +1,60 @@
+/**
+ * Copyright (C) 2015-2021 Expedia, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.hotels.beeju;
+
+import java.security.Permission;
+import java.security.Policy;
+import java.security.ProtectionDomain;
+
+public class NoExitSecurityManager extends SecurityManager {
+
+  private boolean isExitAllowedFlag;
+
+  public NoExitSecurityManager() {
+    super();
+    isExitAllowedFlag = false;
+  }
+
+  public boolean isExitAllowed() {
+    return isExitAllowedFlag;
+  }
+
+  @Override
+  public void checkExit(int status) {
+    if (!isExitAllowed()) {
+      System.out.println("System.exit call intercepted and ignored.");
+      return;
+    }
+    super.checkExit(status);
+  }
+
+  public void setExitAllowed(boolean f) {
+    isExitAllowedFlag = f;
+  }
+
+  public void setPolicy() {
+    Policy.getPolicy();
+
+    Policy allPermissionPolicy = new Policy() {
+      @Override
+      public boolean implies(ProtectionDomain domain, Permission permission) {
+        return true;
+      }
+    };
+
+    Policy.setPolicy(allPermissionPolicy);
+  }
+}

src/main/java/com/hotels/beeju/ThriftHiveMetaStoreJUnitRule.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2015-2020 Expedia, Inc.
+ * Copyright (C) 2015-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,6 +15,8 @@
  */
 package com.hotels.beeju;
 
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.CONNECT_URL_KEY;
+
 import java.util.Map;
 
 import com.hotels.beeju.core.ThriftHiveMetaStoreCore;
@@ -69,6 +71,7 @@ public ThriftHiveMetaStoreJUnitRule(String databaseName, Map<String, String> pre
 
   @Override
   protected void before() throws Throwable {
+    System.clearProperty(CONNECT_URL_KEY.getVarname());
     thriftHiveMetaStoreCore.initialise();
     super.before();
   }

src/main/java/com/hotels/beeju/core/BeejuCore.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2015-2020 Expedia, Inc.
+ * Copyright (C) 2015-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,6 +15,15 @@
  */
 package com.hotels.beeju.core;
 
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.AUTO_CREATE_ALL;
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.CONNECTION_DRIVER;
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.CONNECTION_USER_NAME;
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.CONNECT_URL_KEY;
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.EVENT_DB_NOTIFICATION_API_AUTH;
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.HMS_HANDLER_FORCE_RELOAD_CONF;
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.PWD;
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.SCHEMA_VERIFICATION;
+
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import java.io.File;
@@ -27,13 +36,15 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.UUID;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.derby.jdbc.EmbeddedDriver;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.thrift.TException;
 
 public class BeejuCore {
@@ -78,18 +89,47 @@ public BeejuCore(String databaseName, Map<String, String> preConfiguration, Map<
     checkNotNull(databaseName, "databaseName is required");
     this.databaseName = databaseName;
     configure(preConfiguration);
-    
+
     driverClassName = EmbeddedDriver.class.getName();
-    conf.setBoolean("hcatalog.hive.client.cache.disabled", true);
     connectionURL = "jdbc:derby:memory:" + UUID.randomUUID() + ";create=true";
-    conf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, connectionURL);
-    conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, driverClassName);
-    conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME, METASTORE_DB_USER);
-    conf.setVar(HiveConf.ConfVars.METASTOREPWD, METASTORE_DB_PASSWORD);
-    conf.setBoolVar(HiveConf.ConfVars.HMSHANDLERFORCERELOADCONF, true);
+
+    System.setProperty("derby.system.home", "metastore_db_parent_" + UUID.randomUUID());
+
+    // This should NOT be set as a system property too
+    // conf.set(CONNECT_URL_KEY.getVarname(), connectionURL);
+    setMetastoreAndSystemProperty(CONNECT_URL_KEY, connectionURL);
+
+    setMetastoreAndSystemProperty(CONNECTION_DRIVER, driverClassName);
+    setMetastoreAndSystemProperty(CONNECTION_USER_NAME, METASTORE_DB_USER);
+    setMetastoreAndSystemProperty(PWD, METASTORE_DB_PASSWORD);
+
+     conf.setTimeVar(HiveConf.ConfVars.HIVE_NOTFICATION_EVENT_POLL_INTERVAL, 0, TimeUnit.MILLISECONDS);
+
+//    conf.setInt(HMS_HANDLER_ATTEMPTS.getHiveName(), 12);
+//    conf.setInt(HMS_HANDLER_ATTEMPTS.getVarname(), 12);
+
+    conf.setBoolean("hcatalog.hive.client.cache.disabled", true);
+
+    conf.set("hive.server2.materializedviews.registry.impl", "DUMMY");
+    System.setProperty("hive.server2.materializedviews.registry.impl", "DUMMY");
+
+    setMetastoreAndSystemProperty(HMS_HANDLER_FORCE_RELOAD_CONF, "true");
     // Hive 2.x compatibility
-    conf.setBoolean("datanucleus.schema.autoCreateAll", true);
-    conf.setBoolean("hive.metastore.schema.verification", false);
+    setMetastoreAndSystemProperty(AUTO_CREATE_ALL, "true");
+    setMetastoreAndSystemProperty(SCHEMA_VERIFICATION, "false");
+
+    // Used to prevent "Not authorized to make the get_current_notificationEventId call" errors
+    setMetastoreAndSystemProperty(EVENT_DB_NOTIFICATION_API_AUTH, "false");
+
+    // TODO: check if necessary or not
+//    setMetastoreAndSystemProperty(HIVE_IN_TEST, "true");
+//    setMetastoreAndSystemProperty(CONNECTION_POOLING_TYPE, "NONE");
+//    setMetastoreAndSystemProperty(HIVE_SUPPORT_CONCURRENCY, "false");
+
+//    setMetastoreAndSystemProperty(MULTITHREADED, "false");
+//    setMetastoreAndSystemProperty(NON_TRANSACTIONAL_READ, "false");
+//    setMetastoreAndSystemProperty(DATANUCLEUS_TRANSACTION_ISOLATION, "serializable");
+
     // override default port as some of our test environments claim it is in use.
     conf.setInt("hive.server2.webui.port", 0); // ConfVars.HIVE_SERVER2_WEBUI_PORT
 
@@ -106,6 +146,14 @@ public BeejuCore(String databaseName, Map<String, String> preConfiguration, Map<
 
     configure(postConfiguration);
   }
+
+  private void setMetastoreAndSystemProperty(MetastoreConf.ConfVars key, String value) {
+    conf.set(key.getVarname(), value);
+    conf.set(key.getHiveName(), value);
+
+    System.setProperty(key.getVarname(), value);
+    System.setProperty(key.getHiveName(), value);
+  }
 
   private void configure(Map<String, String> customConfiguration) {
     if (customConfiguration != null) {
@@ -149,6 +197,10 @@ public void createDatabase(String databaseName) throws TException {
     String databaseFolder = new File(tempFile, databaseName).toURI().toString();
     try {
       client.createDatabase(new Database(databaseName, null, databaseFolder, null));
+    } catch (Exception ex) {
+      // TODO: remove sout and improve
+      System.out.println("### BeejuCore: Retrying after exception " + ex.getClass() + " ...");
+      client.createDatabase(new Database(databaseName, null, databaseFolder, null));
     } finally {
       client.close();
     }

src/main/java/com/hotels/beeju/core/HiveMetaStoreCore.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2015-2019 Expedia, Inc.
+ * Copyright (C) 2015-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/java/com/hotels/beeju/core/HiveServer2Core.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2015-2019 Expedia, Inc.
+ * Copyright (C) 2015-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,6 +15,9 @@
  */
 package com.hotels.beeju.core;
 
+
+
+
 import java.io.IOException;
 import java.net.ServerSocket;
 

src/main/java/com/hotels/beeju/core/ThriftHiveMetaStoreCore.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2015-2020 Expedia, Inc.
+ * Copyright (C) 2015-2021 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,11 +27,13 @@
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.HiveMetaStore;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
-import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge23;
+import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
+import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge23;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.hotels.beeju.NoExitSecurityManager;
+
 public class ThriftHiveMetaStoreCore {
 
   private static final Logger LOG = LoggerFactory.getLogger(ThriftHiveMetaStoreCore.class);
@@ -57,15 +59,12 @@ public void initialise() throws Exception {
     }
     beejuCore.setHiveVar(HiveConf.ConfVars.METASTOREURIS, getThriftConnectionUri());
     final HiveConf hiveConf = new HiveConf(beejuCore.conf(), HiveMetaStoreClient.class);
-    thriftServer.execute(new Runnable() {
-      @Override
-      public void run() {
-        try {
-          HadoopThriftAuthBridge bridge = new HadoopThriftAuthBridge23();
-          HiveMetaStore.startMetaStore(thriftPort, bridge, hiveConf, startLock, startCondition, startedServing);
-        } catch (Throwable e) {
-          LOG.error("Unable to start a Thrift server for Hive Metastore", e);
-        }
+    thriftServer.execute(() -> {
+      try {
+        HadoopThriftAuthBridge bridge = HadoopThriftAuthBridge23.getBridge();
+        HiveMetaStore.startMetaStore(thriftPort, bridge, hiveConf, startLock, startCondition, startedServing);
+      } catch (Throwable e) {
+        LOG.error("Unable to start a Thrift server for Hive Metastore", e);
       }
     });
     int i = 0;
@@ -85,6 +84,10 @@ public void run() {
   }
 
   public void shutdown() {
+    NoExitSecurityManager securityManager = new NoExitSecurityManager();
+    securityManager.setPolicy();
+    System.setSecurityManager(securityManager);
+
     thriftServer.shutdown();
   }