diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 903a8030b2c6..cf96fff73ad8 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -5593,6 +5593,10 @@ public static enum ConfVars { HIVE_SERVER2_ICEBERG_METADATA_GENERATOR_THREADS("hive.server2.iceberg.metadata.generator.threads", 10, "Number of threads used to scan partition directories for data files and update/generate iceberg metadata"), + HIVE_ICEBERG_METADATA_REFRESH_MAX_RETRIES("hive.iceberg.metadata.refresh.max.retries", 2, + "Max retry count for trying to access the metadata location in order to refresh metadata during " + + " Iceberg table load."), + /* BLOBSTORE section */ HIVE_BLOBSTORE_SUPPORTED_SCHEMES("hive.blobstore.supported.schemes", "s3,s3a,s3n", diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java index d7533e09044b..386d9ff40513 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java @@ -32,6 +32,7 @@ import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; @@ -180,7 +181,8 @@ protected void doRefresh() { throw new RuntimeException("Interrupted during refresh", e); } - refreshFromMetadataLocation(metadataLocation); + refreshFromMetadataLocation(metadataLocation, HiveConf.getIntVar(conf, + HiveConf.ConfVars.HIVE_ICEBERG_METADATA_REFRESH_MAX_RETRIES)); } @SuppressWarnings("checkstyle:CyclomaticComplexity") diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveTableBaseTest.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveTableBaseTest.java index 76f21924058b..b584b9e73c14 100644 --- a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveTableBaseTest.java +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveTableBaseTest.java @@ -79,7 +79,7 @@ protected static String getTableLocation(String tableName) { return getTableLocationPath(tableName).toString(); } - private static String metadataLocation(String tableName) { + protected static String metadataLocation(String tableName) { return Paths.get(getTableBasePath(tableName), "metadata").toString(); } diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveTableTest.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveTableTest.java index fa67e5d971f4..6804fc80e5b0 100644 --- a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveTableTest.java +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/HiveTableTest.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.SerDeInfo; @@ -48,6 +49,7 @@ import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.CommitFailedException; +import org.apache.iceberg.exceptions.NotFoundException; import org.apache.iceberg.hadoop.ConfigProperties; import org.apache.iceberg.io.FileAppender; import org.apache.iceberg.relocated.com.google.common.collect.Lists; @@ -434,6 +436,22 @@ public void testEngineHiveEnabledTableProperty() throws TException { assertHiveEnabled(hmsTable, false); } + @Test(timeout = 60000, expected = NotFoundException.class) + public void testMissingMetadataWontCauseHang() throws Exception { + catalog.loadTable(TABLE_IDENTIFIER); + HiveConf.setIntVar(catalog.getConf(), HiveConf.ConfVars.HIVE_ICEBERG_METADATA_REFRESH_MAX_RETRIES, 3); + + File realLocation = new File(metadataLocation(TABLE_NAME)); + File fakeLocation = new File(metadataLocation(TABLE_NAME) + "_dummy"); + realLocation.renameTo(fakeLocation); + + try { + catalog.loadTable(TABLE_IDENTIFIER); + } finally { + realLocation.renameTo(realLocation); + } + } + private void assertHiveEnabled(org.apache.hadoop.hive.metastore.api.Table hmsTable, boolean expected) { if (expected) { Assert.assertEquals("org.apache.iceberg.mr.hive.HiveIcebergStorageHandler",