Skip to content
Permalink
Browse files
HIVE-26258: Provide an option for enable locking of external tables (#…
…3313) (Peter Vary reviewed by Denys Kuzmenko)
  • Loading branch information
pvary committed May 24, 2022
1 parent 2500b2d commit dee417faa7341b827f8289814f0aacdeeed82abb
Showing 3 changed files with 51 additions and 6 deletions.
@@ -3008,6 +3008,12 @@ public static enum ConfVars {
"and hive.exec.dynamic.partition.mode (nonstrict).\n" +
"The default DummyTxnManager replicates pre-Hive-0.13 behavior and provides\n" +
"no transactions."),
HIVE_TXN_EXT_LOCKING_ENABLED("hive.txn.ext.locking.enabled", false,
"When enabled use standard R/W lock semantics based on hive.txn.strict.locking.mode for external resources,\n" +
"e.g. INSERT will acquire lock based on hive.txn.strict.locking.mode\n" +
"(exclusive if it is true, shared if that is false),\n" +
"SELECT will acquire shared lock based on hive.txn.nonacid.read.locks.\n" +
"When disabled no locks are acquired for external resources."),
HIVE_TXN_STRICT_LOCKING_MODE("hive.txn.strict.locking.mode", true, "In strict mode non-ACID\n" +
"resources use standard R/W lock semantics, e.g. INSERT will acquire exclusive lock.\n" +
"In nonstrict mode, for non-ACID resources, INSERT will only acquire shared lock, which\n" +
@@ -2844,12 +2844,12 @@ public static void setNonTransactional(Map<String, String> tblProps) {
tblProps.remove(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
}

private static boolean needsLock(Entity entity) {
private static boolean needsLock(Entity entity, boolean isExternalEnabled) {
switch (entity.getType()) {
case TABLE:
return isLockableTable(entity.getTable());
return isLockableTable(entity.getTable(), isExternalEnabled);
case PARTITION:
return isLockableTable(entity.getPartition().getTable());
return isLockableTable(entity.getPartition().getTable(), isExternalEnabled);
default:
return true;
}
@@ -2863,14 +2863,16 @@ private static Table getTable(WriteEntity we) {
return t;
}

private static boolean isLockableTable(Table t) {
private static boolean isLockableTable(Table t, boolean isExternalEnabled) {
if (t.isTemporary()) {
return false;
}
switch (t.getTableType()) {
case MANAGED_TABLE:
case MATERIALIZED_VIEW:
return true;
case EXTERNAL_TABLE:
return isExternalEnabled;
default:
return false;
}
@@ -2890,6 +2892,7 @@ public static List<LockComponent> makeLockComponents(Set<WriteEntity> outputs, S
boolean skipReadLock = !conf.getBoolVar(ConfVars.HIVE_TXN_READ_LOCKS);
boolean skipNonAcidReadLock = !conf.getBoolVar(ConfVars.HIVE_TXN_NONACID_READ_LOCKS);
boolean sharedWrite = !conf.getBoolVar(HiveConf.ConfVars.TXN_WRITE_X_LOCK);
boolean isExternalEnabled = conf.getBoolVar(HiveConf.ConfVars.HIVE_TXN_EXT_LOCKING_ENABLED);
boolean isMerge = operation == Context.Operation.MERGE;

// We don't want to acquire read locks during update or delete as we'll be acquiring write
@@ -2898,7 +2901,7 @@ public static List<LockComponent> makeLockComponents(Set<WriteEntity> outputs, S
.filter(input -> !input.isDummy()
&& input.needsLock()
&& !input.isUpdateOrDelete()
&& AcidUtils.needsLock(input)
&& AcidUtils.needsLock(input, isExternalEnabled)
&& !skipReadLock)
.collect(Collectors.toList());

@@ -2959,7 +2962,7 @@ public static List<LockComponent> makeLockComponents(Set<WriteEntity> outputs, S
for (WriteEntity output : outputs) {
LOG.debug("output is null " + (output == null));
if (output.getType() == Entity.Type.DFS_DIR || output.getType() == Entity.Type.LOCAL_DIR || !AcidUtils
.needsLock(output)) {
.needsLock(output, isExternalEnabled)) {
// We don't lock files or directories. We also skip locking temp tables.
continue;
}
@@ -864,6 +864,42 @@ public void testLockingOnInsertOverwriteNonNativeTables() throws Exception {
dropTable(new String[] {"tab_not_acid"});
}

@Test
public void testLockingExternalInStrictModeInsert() throws Exception {
dropTable(new String[] {"tab_not_acid"});
driver.run("create external table if not exists tab_not_acid (na int, nb int) partitioned by (np string) " +
"clustered by (na) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='false')");
driver.run("insert into tab_not_acid partition(np) (na,nb,np) values(1,2,'blah'),(3,4,'doh')");

conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_EXT_LOCKING_ENABLED, true);
HiveTxnManager txnMgr = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
txnMgr.openTxn(ctx, "T1");
driver.compileAndRespond("insert into tab_not_acid partition(np='blah') values(7,8)", true);
((DbTxnManager)txnMgr).acquireLocks(driver.getPlan(), ctx, "T1", false);
List<ShowLocksResponseElement> locks = getLocks(txnMgr);
Assert.assertEquals("Unexpected lock count", 1, locks.size());
checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks);
conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_EXT_LOCKING_ENABLED, false);
}

@Test
public void testLockingExternalInStrictModeSelect() throws Exception {
dropTable(new String[] {"tab_not_acid"});
driver.run("create external table if not exists tab_not_acid (na int, nb int) " +
"stored as orc TBLPROPERTIES ('transactional'='false')");
driver.run("insert into tab_not_acid values(1,2),(3,4)");

conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_EXT_LOCKING_ENABLED, true);
HiveTxnManager txnMgr = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
txnMgr.openTxn(ctx, "T1");
driver.compileAndRespond("select * from tab_not_acid", true);
((DbTxnManager)txnMgr).acquireLocks(driver.getPlan(), ctx, "T1", false);
List<ShowLocksResponseElement> locks = getLocks(txnMgr);
Assert.assertEquals("Unexpected lock count", 1, locks.size());
checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null, locks);
conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_EXT_LOCKING_ENABLED, false);
}

/** The list is small, and the object is generated, so we don't use sets/equals/etc. */
public static ShowLocksResponseElement checkLock(LockType expectedType, LockState expectedState, String expectedDb,
String expectedTable, String expectedPartition, List<ShowLocksResponseElement> actuals) {

0 comments on commit dee417f

Please sign in to comment.