Skip to content

HIVE-27208: Iceberg: Add support for rename table. #4185

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Apr 11, 2023
Merged

HIVE-27208: Iceberg: Add support for rename table. #4185

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9d81483
HIVE-27208: Iceberg: Add support for rename table.
ayushtkn Mar 30, 2023
ea42e4f
Add v1 test.
ayushtkn Apr 3, 2023
98bb218
Address review comments.
ayushtkn Apr 5, 2023
ff6890a
Correct exception message.
ayushtkn Apr 10, 2023
dda330a
Address Review Comments.
ayushtkn Apr 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/Catalogs.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,19 @@ public static Table registerTable(Configuration conf, Properties props, String m
return new HadoopTables(conf).create(schema, spec, map, location);
}

public static void renameTable(Configuration conf, Properties props, TableIdentifier to) {
String catalogName = props.getProperty(InputFormatConfig.CATALOG_NAME);

Optional<Catalog> catalog = loadCatalog(conf, catalogName);
if (catalog.isPresent()) {
String name = props.getProperty(NAME);
Preconditions.checkNotNull(name, "Table identifier not set");
catalog.get().renameTable(TableIdentifier.parse(name), to);
} else {
throw new RuntimeException("Rename from " + props.getProperty(NAME) + " to " + to + " failed");
}
}

static Optional<Catalog> loadCatalog(Configuration conf, String catalogName) {
String catalogType = getCatalogType(conf, catalogName);
if (NO_CATALOG_TYPE.equalsIgnoreCase(catalogType)) {
Expand Down
10 changes: 9 additions & 1 deletion iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ public class HiveIcebergMetaHook implements HiveMetaHook {
static final EnumSet<AlterTableType> SUPPORTED_ALTER_OPS = EnumSet.of(
AlterTableType.ADDCOLS, AlterTableType.REPLACE_COLUMNS, AlterTableType.RENAME_COLUMN,
AlterTableType.ADDPROPS, AlterTableType.DROPPROPS, AlterTableType.SETPARTITIONSPEC,
AlterTableType.UPDATE_COLUMNS, AlterTableType.SETPARTITIONSPEC, AlterTableType.EXECUTE);
AlterTableType.UPDATE_COLUMNS, AlterTableType.RENAME, AlterTableType.EXECUTE);
private static final List<String> MIGRATION_ALLOWED_SOURCE_FORMATS = ImmutableList.of(
FileFormat.PARQUET.name().toLowerCase(),
FileFormat.ORC.name().toLowerCase(),
Expand Down Expand Up @@ -319,6 +319,10 @@ public void preAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable, E
throws MetaException {
catalogProperties = getCatalogProperties(hmsTable);
setupAlterOperationType(hmsTable, context);
if (AlterTableType.RENAME.equals(currentAlterTableOp)) {
catalogProperties.put(Catalogs.NAME, TableIdentifier.of(context.getProperties().get(OLD_DB_NAME),
context.getProperties().get(OLD_TABLE_NAME)).toString());
}
if (commitLock == null) {
commitLock = new HiveCommitLock(conf, new CachedClientPool(conf, Maps.fromProperties(catalogProperties)),
catalogProperties.getProperty(Catalogs.NAME), hmsTable.getDbName(), hmsTable.getTableName());
Expand Down Expand Up @@ -549,6 +553,10 @@ public void commitAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable
case SETPARTITIONSPEC:
IcebergTableUtil.updateSpec(conf, icebergTable);
break;
case RENAME:
Catalogs.renameTable(conf, catalogProperties, TableIdentifier.of(hmsTable.getDbName(),
hmsTable.getTableName()));
break;
}
}
}
Expand Down
65 changes: 65 additions & 0 deletions iceberg/iceberg-handler/src/test/queries/positive/iceberg_rename.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@

-- create a v1 table
create table icev1 (id int, name string) Stored by Iceberg;

-- insert some values
insert into icev1 values (1, 'SSD'),(2, 'RAM');

select * from icev1 order by id;

-- do the rename
explain alter table icev1 rename to icev1renamed;

alter table icev1 rename to icev1renamed;

select * from icev1renamed order by id;

-- create an unpartitioned v2 table
create table iceorgin (id int, name string) Stored by Iceberg TBLPROPERTIES ('format-version'='2');

-- insert some values
insert into iceorgin values (1, 'ABC'),(2, 'CBS'),(3, null),(4, 'POPI'),(5, 'AQWR'),(6, 'POIU'),
(9, null),(8,'POIKL'),(10, 'YUIO');

-- do some deletes
delete from iceorgin where id>9 OR id=8;

select * from iceorgin order by id;

-- do the rename

alter table iceorgin rename to icerenamed;

select * from icerenamed order by id;

-- create a partitioned v2 table
create table iceorginpart (id int) partitioned by (part string) Stored by Iceberg TBLPROPERTIES ('format-version'='2');

insert into iceorginpart values (1, 'ABC'),(2, 'CBS'),(3,'CBS'),(4, 'ABC'),(5, 'AQWR'),(6, 'ABC'),
(9, 'AQWR'),(8,'ABC'),(10, 'YUIO');

-- do some deletes
delete from iceorginpart where id<3 OR id=7;

select * from iceorginpart order by id;

explain alter table iceorginpart rename to icerenamedpart;

alter table iceorginpart rename to icerenamedpart;

select * from icerenamedpart order by id;

-- create a new unpartitioned table with old name
create table iceorgin (id int, name string) Stored by Iceberg TBLPROPERTIES ('format-version'='2');

insert into iceorgin values (100, 'ABCDWC');

select * from iceorgin order by id;

-- create a new partitioned table with old name

create table iceorginpart (id int) partitioned by (part string) Stored by Iceberg TBLPROPERTIES ('format-version'='2');

insert into iceorginpart values (22, 'DER'),(2, 'KLM');

select * from iceorginpart order by id;
265 changes: 265 additions & 0 deletions iceberg/iceberg-handler/src/test/results/positive/iceberg_rename.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,265 @@
PREHOOK: query: create table icev1 (id int, name string) Stored by Iceberg
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@icev1
POSTHOOK: query: create table icev1 (id int, name string) Stored by Iceberg
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@icev1
PREHOOK: query: insert into icev1 values (1, 'SSD'),(2, 'RAM')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@icev1
POSTHOOK: query: insert into icev1 values (1, 'SSD'),(2, 'RAM')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@icev1
PREHOOK: query: select * from icev1 order by id
PREHOOK: type: QUERY
PREHOOK: Input: default@icev1
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select * from icev1 order by id
POSTHOOK: type: QUERY
POSTHOOK: Input: default@icev1
POSTHOOK: Output: hdfs://### HDFS PATH ###
1 SSD
2 RAM
PREHOOK: query: explain alter table icev1 rename to icev1renamed
PREHOOK: type: ALTERTABLE_RENAME
PREHOOK: Input: default@icev1
PREHOOK: Output: database:default
PREHOOK: Output: default@icev1
PREHOOK: Output: default@icev1renamed
POSTHOOK: query: explain alter table icev1 rename to icev1renamed
POSTHOOK: type: ALTERTABLE_RENAME
POSTHOOK: Input: default@icev1
POSTHOOK: Output: database:default
POSTHOOK: Output: default@icev1
POSTHOOK: Output: default@icev1renamed
Stage-0
Rename Table{"table name:":"default.icev1","new table name:":"default.icev1renamed"}

PREHOOK: query: alter table icev1 rename to icev1renamed
PREHOOK: type: ALTERTABLE_RENAME
PREHOOK: Input: default@icev1
PREHOOK: Output: database:default
PREHOOK: Output: default@icev1
PREHOOK: Output: default@icev1renamed
POSTHOOK: query: alter table icev1 rename to icev1renamed
POSTHOOK: type: ALTERTABLE_RENAME
POSTHOOK: Input: default@icev1
POSTHOOK: Output: database:default
POSTHOOK: Output: default@icev1
POSTHOOK: Output: default@icev1renamed
PREHOOK: query: select * from icev1renamed order by id
PREHOOK: type: QUERY
PREHOOK: Input: default@icev1renamed
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select * from icev1renamed order by id
POSTHOOK: type: QUERY
POSTHOOK: Input: default@icev1renamed
POSTHOOK: Output: hdfs://### HDFS PATH ###
1 SSD
2 RAM
PREHOOK: query: create table iceorgin (id int, name string) Stored by Iceberg TBLPROPERTIES ('format-version'='2')
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@iceorgin
POSTHOOK: query: create table iceorgin (id int, name string) Stored by Iceberg TBLPROPERTIES ('format-version'='2')
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@iceorgin
PREHOOK: query: insert into iceorgin values (1, 'ABC'),(2, 'CBS'),(3, null),(4, 'POPI'),(5, 'AQWR'),(6, 'POIU'),
(9, null),(8,'POIKL'),(10, 'YUIO')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@iceorgin
POSTHOOK: query: insert into iceorgin values (1, 'ABC'),(2, 'CBS'),(3, null),(4, 'POPI'),(5, 'AQWR'),(6, 'POIU'),
(9, null),(8,'POIKL'),(10, 'YUIO')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@iceorgin
PREHOOK: query: delete from iceorgin where id>9 OR id=8
PREHOOK: type: QUERY
PREHOOK: Input: default@iceorgin
PREHOOK: Output: default@iceorgin
POSTHOOK: query: delete from iceorgin where id>9 OR id=8
POSTHOOK: type: QUERY
POSTHOOK: Input: default@iceorgin
POSTHOOK: Output: default@iceorgin
PREHOOK: query: select * from iceorgin order by id
PREHOOK: type: QUERY
PREHOOK: Input: default@iceorgin
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select * from iceorgin order by id
POSTHOOK: type: QUERY
POSTHOOK: Input: default@iceorgin
POSTHOOK: Output: hdfs://### HDFS PATH ###
1 ABC
2 CBS
3 NULL
4 POPI
5 AQWR
6 POIU
9 NULL
PREHOOK: query: alter table iceorgin rename to icerenamed
PREHOOK: type: ALTERTABLE_RENAME
PREHOOK: Input: default@iceorgin
PREHOOK: Output: database:default
PREHOOK: Output: default@iceorgin
PREHOOK: Output: default@icerenamed
POSTHOOK: query: alter table iceorgin rename to icerenamed
POSTHOOK: type: ALTERTABLE_RENAME
POSTHOOK: Input: default@iceorgin
POSTHOOK: Output: database:default
POSTHOOK: Output: default@iceorgin
POSTHOOK: Output: default@icerenamed
PREHOOK: query: select * from icerenamed order by id
PREHOOK: type: QUERY
PREHOOK: Input: default@icerenamed
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select * from icerenamed order by id
POSTHOOK: type: QUERY
POSTHOOK: Input: default@icerenamed
POSTHOOK: Output: hdfs://### HDFS PATH ###
1 ABC
2 CBS
3 NULL
4 POPI
5 AQWR
6 POIU
9 NULL
PREHOOK: query: create table iceorginpart (id int) partitioned by (part string) Stored by Iceberg TBLPROPERTIES ('format-version'='2')
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@iceorginpart
POSTHOOK: query: create table iceorginpart (id int) partitioned by (part string) Stored by Iceberg TBLPROPERTIES ('format-version'='2')
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@iceorginpart
PREHOOK: query: insert into iceorginpart values (1, 'ABC'),(2, 'CBS'),(3,'CBS'),(4, 'ABC'),(5, 'AQWR'),(6, 'ABC'),
(9, 'AQWR'),(8,'ABC'),(10, 'YUIO')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@iceorginpart
POSTHOOK: query: insert into iceorginpart values (1, 'ABC'),(2, 'CBS'),(3,'CBS'),(4, 'ABC'),(5, 'AQWR'),(6, 'ABC'),
(9, 'AQWR'),(8,'ABC'),(10, 'YUIO')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@iceorginpart
PREHOOK: query: delete from iceorginpart where id<3 OR id=7
PREHOOK: type: QUERY
PREHOOK: Input: default@iceorginpart
PREHOOK: Output: default@iceorginpart
POSTHOOK: query: delete from iceorginpart where id<3 OR id=7
POSTHOOK: type: QUERY
POSTHOOK: Input: default@iceorginpart
POSTHOOK: Output: default@iceorginpart
PREHOOK: query: select * from iceorginpart order by id
PREHOOK: type: QUERY
PREHOOK: Input: default@iceorginpart
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select * from iceorginpart order by id
POSTHOOK: type: QUERY
POSTHOOK: Input: default@iceorginpart
POSTHOOK: Output: hdfs://### HDFS PATH ###
3 CBS
4 ABC
5 AQWR
6 ABC
8 ABC
9 AQWR
10 YUIO
PREHOOK: query: explain alter table iceorginpart rename to icerenamedpart
PREHOOK: type: ALTERTABLE_RENAME
PREHOOK: Input: default@iceorginpart
PREHOOK: Output: database:default
PREHOOK: Output: default@iceorginpart
PREHOOK: Output: default@icerenamedpart
POSTHOOK: query: explain alter table iceorginpart rename to icerenamedpart
POSTHOOK: type: ALTERTABLE_RENAME
POSTHOOK: Input: default@iceorginpart
POSTHOOK: Output: database:default
POSTHOOK: Output: default@iceorginpart
POSTHOOK: Output: default@icerenamedpart
Stage-0
Rename Table{"table name:":"default.iceorginpart","new table name:":"default.icerenamedpart"}

PREHOOK: query: alter table iceorginpart rename to icerenamedpart
PREHOOK: type: ALTERTABLE_RENAME
PREHOOK: Input: default@iceorginpart
PREHOOK: Output: database:default
PREHOOK: Output: default@iceorginpart
PREHOOK: Output: default@icerenamedpart
POSTHOOK: query: alter table iceorginpart rename to icerenamedpart
POSTHOOK: type: ALTERTABLE_RENAME
POSTHOOK: Input: default@iceorginpart
POSTHOOK: Output: database:default
POSTHOOK: Output: default@iceorginpart
POSTHOOK: Output: default@icerenamedpart
PREHOOK: query: select * from icerenamedpart order by id
PREHOOK: type: QUERY
PREHOOK: Input: default@icerenamedpart
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select * from icerenamedpart order by id
POSTHOOK: type: QUERY
POSTHOOK: Input: default@icerenamedpart
POSTHOOK: Output: hdfs://### HDFS PATH ###
3 CBS
4 ABC
5 AQWR
6 ABC
8 ABC
9 AQWR
10 YUIO
PREHOOK: query: create table iceorgin (id int, name string) Stored by Iceberg TBLPROPERTIES ('format-version'='2')
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@iceorgin
POSTHOOK: query: create table iceorgin (id int, name string) Stored by Iceberg TBLPROPERTIES ('format-version'='2')
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@iceorgin
PREHOOK: query: insert into iceorgin values (100, 'ABCDWC')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@iceorgin
POSTHOOK: query: insert into iceorgin values (100, 'ABCDWC')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@iceorgin
PREHOOK: query: select * from iceorgin order by id
PREHOOK: type: QUERY
PREHOOK: Input: default@iceorgin
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select * from iceorgin order by id
POSTHOOK: type: QUERY
POSTHOOK: Input: default@iceorgin
POSTHOOK: Output: hdfs://### HDFS PATH ###
100 ABCDWC
PREHOOK: query: create table iceorginpart (id int) partitioned by (part string) Stored by Iceberg TBLPROPERTIES ('format-version'='2')
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@iceorginpart
POSTHOOK: query: create table iceorginpart (id int) partitioned by (part string) Stored by Iceberg TBLPROPERTIES ('format-version'='2')
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@iceorginpart
PREHOOK: query: insert into iceorginpart values (22, 'DER'),(2, 'KLM')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@iceorginpart
POSTHOOK: query: insert into iceorginpart values (22, 'DER'),(2, 'KLM')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@iceorginpart
PREHOOK: query: select * from iceorginpart order by id
PREHOOK: type: QUERY
PREHOOK: Input: default@iceorginpart
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select * from iceorginpart order by id
POSTHOOK: type: QUERY
POSTHOOK: Input: default@iceorginpart
POSTHOOK: Output: hdfs://### HDFS PATH ###
2 KLM
22 DER
Loading