Skip to content

HIVE-25266: Fix TestWarehouseExternalDir #4297

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 24, 2023
Merged

HIVE-25266: Fix TestWarehouseExternalDir #4297

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3fd2a13
HIVE-25266 : Fix TestWarehouseExternalDir
Apr 19, 2023
647a9d8
HIVE-25266 : Fix TestWarehouseExternalDir
May 5, 2023
d502961
HIVE-25266 : Fix TestWarehouseExternalDir
May 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
149 changes: 64 additions & 85 deletions itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestWarehouseExternalDir.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,14 @@

package org.apache.hadoop.hive.ql;

import java.net.URI;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.HashMap;
import java.util.Map;

import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
Expand All @@ -34,7 +34,7 @@
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hive.jdbc.miniHS2.MiniHS2;
import org.apache.hive.jdbc.miniHS2.MiniHS2.MiniClusterType;

import org.apache.hadoop.hive.ql.session.SessionState;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
Expand All @@ -44,114 +44,92 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;

import org.apache.hadoop.hive.metastore.api.Database;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.metastore.Warehouse;


@org.junit.Ignore("HIVE-25266")
public class TestWarehouseExternalDir {
private static final Logger LOG = LoggerFactory.getLogger(TestWarehouseExternalDir.class);
private static final Logger LOG = LoggerFactory.getLogger(TestWarehouseExternalDir.class);

static MiniHS2 miniHS2;
static Hive db;
static Connection conn;
private static MiniHS2 miniHS2;
private static Hive db;

String whRootExternal = "/wh_ext";
Path whRootExternalPath;
Path whRootManagedPath;
FileSystem fs;
private static HiveConf conf;
private static Connection conn;
private static String whRootExternal = "/wh_ext";
private static String dbName = "twed_db1";
private static Path whRootExternalPath;
private static Path whRootManagedPath;

private Statement stmt;

@BeforeClass
public static void beforeTest() throws Exception {

HiveConf conf = new HiveConf();

conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false);

conf.setVar(ConfVars.HIVE_METASTORE_WAREHOUSE_EXTERNAL, whRootExternal);

miniHS2 = new MiniHS2.Builder().withMiniMR().withRemoteMetastore().withConf(conf).build();
miniHS2.start(new HashMap<String, String>());

FileSystem fs = miniHS2.getDfs().getFileSystem();
whRootExternalPath = fs.makeQualified(new Path(whRootExternal));
whRootManagedPath = fs.makeQualified(new Path(MetastoreConf.getVar(conf, MetastoreConf.ConfVars.WAREHOUSE)));

LOG.info("fs: {}", miniHS2.getDfs().getFileSystem().getUri());
LOG.info("warehouse location: {}", whRootManagedPath);
LOG.info("whRootExternalPath: {}", whRootExternalPath);
db = Hive.get(conf);
createDb();
}

@AfterClass
public static void afterTest() throws Exception {
public static void afterTest() throws Exception {

if (db != null) {
db.closeCurrent();
db = null;
}

if (conn != null) {
// TODO: delete tables/databases?
try (Statement stmt = conn.createStatement()) {
stmt.execute("drop database if exists twed_db1 cascade");
}
conn.close();
conn = null;
}

if (miniHS2 != null) {
miniHS2.stop();
miniHS2.cleanup();
MiniHS2.cleanupLocalDir();
miniHS2 = null;
}
}

@Before
public void setUp() throws Exception {
MiniHS2.cleanupLocalDir();
Hive.closeCurrent();
if (conn != null) {
conn.close();
}
}

@After
public void tearDown() throws Exception {
}

public TestWarehouseExternalDir() throws Exception {
HiveConf conf = new HiveConf();

// Specify the external warehouse root
conf.setVar(ConfVars.HIVE_METASTORE_WAREHOUSE_EXTERNAL, whRootExternal);

// Settings borrowed from TestJdbcWithMiniHS2
conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
conf.setBoolVar(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED, false);
conf.setBoolVar(ConfVars.HIVESTATSCOLAUTOGATHER, false);

MiniHS2.Builder builder = new MiniHS2.Builder()
.withConf(conf)
.cleanupLocalDirOnStartup(true)
.withMiniMR()
.withRemoteMetastore();
miniHS2 = builder.build();

Map<String, String> confOverlay = new HashMap<String, String>();
miniHS2.start(confOverlay);

HiveConf dbConf = miniHS2.getHiveConf();
db = Hive.get(dbConf);

fs = miniHS2.getDfs().getFileSystem();
whRootExternalPath = fs.makeQualified(new Path(whRootExternal));
whRootManagedPath = fs.makeQualified(new Path(dbConf.getVar(ConfVars.METASTOREWAREHOUSE)));

LOG.info("fs: {}", miniHS2.getDfs().getFileSystem().getUri());
LOG.info("warehouse location: {}", whRootManagedPath);
LOG.info("whRootExternalPath: {}", whRootExternalPath);

conn = getConnection();
try (Statement stmt = conn.createStatement()) {
stmt.execute("create database if not exists twed_db1");
if (conn != null) {
conn.close();
}
}

private static Connection getConnection() throws Exception {
return getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar");
}

private static Connection getConnection(String dbName) throws Exception {
return getConnection(miniHS2.getJdbcURL(dbName), System.getProperty("user.name"), "bar");
@Before
public void setUp() throws Exception {
conn = DriverManager.getConnection(miniHS2.getJdbcURL(dbName),
System.getProperty("user.name"), "bar");
assertNotNull(conn);
}

private static Connection getConnection(String jdbcURL, String user, String pwd)
throws SQLException {
Connection conn = DriverManager.getConnection(jdbcURL, user, pwd);
private static void createDb() throws Exception {
conn = DriverManager.getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar");
assertNotNull(conn);
return conn;
Statement stmt2 = conn.createStatement();
stmt2.execute("DROP DATABASE IF EXISTS " + dbName + " CASCADE");
stmt2.execute("CREATE DATABASE " + dbName);
stmt2.close();
}

static void checkTableLocation(Table table, Path expectedPath) throws Exception {
void checkTableLocation(Table table, Path expectedPath) throws Exception {
LOG.info("Table {}: location {}", table.getTableName(), table.getDataLocation());
assertEquals(table.getTableName(), expectedPath, table.getDataLocation());
assertTrue(miniHS2.getDfs().getFileSystem().exists(table.getDataLocation()));
Expand All @@ -161,8 +139,9 @@ static void checkTableLocation(Table table, Path expectedPath) throws Exception
public void testManagedPaths() throws Exception {
try (Statement stmt = conn.createStatement()) {
// Confirm default managed table paths
stmt.execute("create table default.twed_1(c1 string)");
Table tab = db.getTable("default", "twed_1");
stmt.execute("create table IF NOT EXISTS default.twed_1(c1 string)");

Table tab = db.getTable("default","twed_1");
checkTableLocation(tab, new Path(whRootManagedPath, "twed_1"));

stmt.execute("create table twed_db1.tab1(c1 string, c2 string)");
Expand All @@ -174,19 +153,19 @@ public void testManagedPaths() throws Exception {
@Test
public void testExternalDefaultPaths() throws Exception {
try (Statement stmt = conn.createStatement()) {
stmt.execute("create external table default.twed_ext1(c1 string)");
Table tab = db.getTable("default", "twed_ext1");
stmt.execute("create external table IF NOT EXISTS default.twed_ext1(c1 string)");
Table tab = db.getTable("default","twed_ext1");
checkTableLocation(tab, new Path(whRootExternalPath, "twed_ext1"));

stmt.execute("create external table twed_db1.twed_ext2(c1 string)");
stmt.execute("create external table twed_db1.twed_ext2(c1 string) ");
tab = db.getTable("twed_db1", "twed_ext2");
checkTableLocation(tab, new Path(new Path(whRootExternalPath, "twed_db1.db"), "twed_ext2"));

stmt.execute("create external table default.twed_ext3 like default.twed_ext1");
stmt.execute("create external table default.twed_ext3 like default.twed_ext1 ");
tab = db.getTable("default", "twed_ext3");
checkTableLocation(tab, new Path(whRootExternalPath, "twed_ext3"));

stmt.execute("create external table twed_db1.twed_ext4 like default.twed_ext1");
stmt.execute("create external table twed_db1.twed_ext4 like default.twed_ext1 ");
tab = db.getTable("twed_db1", "twed_ext4");
checkTableLocation(tab, new Path(new Path(whRootExternalPath, "twed_db1.db"), "twed_ext4"));
}
Expand Down