Skip to content
Permalink
Browse files
HIVE-26203: Implement alter iceberg table metadata location (#3270) (…
…Laszlo Pinter, reviewed by Peter Vary)
  • Loading branch information
lcspinter committed May 11, 2022
1 parent 7e4220f commit faa3f62923ac0b5a68a2b1e16736f1a32d38bb8e
Showing 2 changed files with 95 additions and 0 deletions.
@@ -336,6 +336,31 @@ public void preAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable, E
// that users can change data types or reorder columns too with this alter op type, so its name is misleading..)
assertNotMigratedTable(hmsTable.getParameters(), "CHANGE COLUMN");
handleChangeColumn(hmsTable);
} else if (AlterTableType.ADDPROPS.equals(currentAlterTableOp)) {
assertNotCrossTableMetadataLocationChange(hmsTable.getParameters());
}
}

/**
* Perform a check on the current iceberg table whether a metadata change can be performed. A table is eligible if
* the current metadata uuid and the new metadata uuid matches.
* @param tblParams hms table properties, must be non-null
*/
private void assertNotCrossTableMetadataLocationChange(Map<String, String> tblParams) {
if (tblParams.containsKey(BaseMetastoreTableOperations.METADATA_LOCATION_PROP)) {
Preconditions.checkArgument(icebergTable != null,
"Cannot perform table migration to Iceberg and setting the snapshot location in one step. " +
"Please migrate the table first");
String newMetadataLocation = tblParams.get(BaseMetastoreTableOperations.METADATA_LOCATION_PROP);
FileIO io = ((BaseTable) icebergTable).operations().io();
TableMetadata newMetadata = TableMetadataParser.read(io, newMetadataLocation);
TableMetadata currentMetadata = ((BaseTable) icebergTable).operations().current();
if (!currentMetadata.uuid().equals(newMetadata.uuid())) {
throw new UnsupportedOperationException(
String.format("Cannot change iceberg table %s metadata location pointing to another table's metadata %s",
icebergTable.name(), newMetadataLocation)
);
}
}
}

@@ -1456,6 +1456,76 @@ public void testCreateTableWithMetadataLocation() throws IOException {
HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS.stream()).collect(Collectors.toList()), records, 0);
}

@Test
public void testAlterTableWithMetadataLocation() throws IOException {
Assume.assumeTrue("Alter table with metadata location is only supported for Hive Catalog tables",
testTableType.equals(TestTables.TestTableType.HIVE_CATALOG));
TableIdentifier tableIdentifier = TableIdentifier.of("default", "source");
// create a test table with some dummy data
Table table =
testTables.createTable(shell, tableIdentifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA,
PartitionSpec.unpartitioned(), FileFormat.PARQUET, Collections.emptyList(), 1, Collections.emptyMap());
testTables.appendIcebergTable(shell.getHiveConf(), table, FileFormat.PARQUET, null,
HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS);
String firstMetadataLocation = ((BaseTable) table).operations().current().metadataFileLocation();
testTables.appendIcebergTable(shell.getHiveConf(), table, FileFormat.PARQUET, null,
HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS);
table.refresh();
String secondMetadataLocation = ((BaseTable) table).operations().current().metadataFileLocation();
Assert.assertNotEquals(firstMetadataLocation, secondMetadataLocation);
shell.executeStatement("ALTER TABLE " + tableIdentifier.name() + " SET TBLPROPERTIES('metadata_location'='" +
firstMetadataLocation + "')");
// during alter operation a new metadata file is created but reflecting the old metadata state
List<Object[]> rows = shell.executeStatement("SELECT * FROM " + tableIdentifier.name());
List<Record> records = HiveIcebergTestUtils.valueForRow(table.schema(), rows);
HiveIcebergTestUtils.validateData(HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, records, 0);
// add another batch of data to the table to make sure data manipulation is working
testTables.appendIcebergTable(shell.getHiveConf(), table, FileFormat.PARQUET, null,
HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS);
rows = shell.executeStatement("SELECT * FROM " + tableIdentifier.name());
records = HiveIcebergTestUtils.valueForRow(table.schema(), rows);
HiveIcebergTestUtils.validateData(
Stream.concat(HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS.stream(),
HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS.stream()).collect(Collectors.toList()), records, 0);
}

@Test
public void testAlterTableWithMetadataLocationFromAnotherTable() throws IOException {
TableIdentifier sourceIdentifier = TableIdentifier.of("default", "source");
Table sourceTable =
testTables.createTable(shell, sourceIdentifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA,
PartitionSpec.unpartitioned(), FileFormat.PARQUET, Collections.emptyList(), 1,
ImmutableMap.<String, String>builder().put(InputFormatConfig.EXTERNAL_TABLE_PURGE, "FALSE").build());
testTables.appendIcebergTable(shell.getHiveConf(), sourceTable, FileFormat.PARQUET, null,
HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS);
String metadataLocation = ((BaseTable) sourceTable).operations().current().metadataFileLocation();
shell.executeStatement("DROP TABLE " + sourceIdentifier.name());
TableIdentifier targetIdentifier = TableIdentifier.of("default", "target");
testTables.createTable(shell, targetIdentifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA,
PartitionSpec.unpartitioned(), FileFormat.PARQUET, Collections.emptyList(), 1, Collections.emptyMap());
AssertHelpers.assertThrows("should throw exception", IllegalArgumentException.class,
"Cannot change iceberg table",
() -> {
shell.executeStatement("ALTER TABLE " + targetIdentifier.name() + " SET TBLPROPERTIES('metadata_location'='" +
metadataLocation + "')");
});
}

@Test
public void testAlterTableToIcebergAndMetadataLocation() throws IOException {
String tableName = "tbl";
String createQuery = "CREATE EXTERNAL TABLE " + tableName + " (a int) STORED AS PARQUET " +
testTables.locationForCreateTableSQL(TableIdentifier.of("default", tableName)) +
testTables.propertiesForCreateTableSQL(ImmutableMap.of());
shell.executeStatement(createQuery);
AssertHelpers.assertThrows("should throw exception", IllegalArgumentException.class,
"Cannot perform table migration to Iceberg and setting the snapshot location in one step.",
() -> {
shell.executeStatement("ALTER TABLE " + tableName + " SET TBLPROPERTIES(" +
"'storage_handler'='org.apache.iceberg.mr.hive.HiveIcebergStorageHandler','metadata_location'='asdf')");
});
}

/**
* Checks that the new schema has newintcol and newstring col columns on both HMS and Iceberg sides
* @throws Exception - any test error

0 comments on commit faa3f62

Please sign in to comment.