From 133a29b20f8569bbf97c3e4c35740cdc95eb40fd Mon Sep 17 00:00:00 2001 From: Butao Zhang Date: Tue, 11 Jul 2023 23:38:13 +0800 Subject: [PATCH] HIVE-27235: Iceberg: CREATE TAG SQL implementation (Butao Zhang, reviewed by Denys Kuzmenko) Closes #4372 --- .../iceberg/mr/hive/HiveIcebergMetaHook.java | 3 +- .../mr/hive/HiveIcebergStorageHandler.java | 26 ++-- .../iceberg/mr/hive/IcebergBranchExec.java | 6 +- .../iceberg/mr/hive/IcebergTagExec.java | 55 +++++++ .../mr/hive/TestHiveIcebergTagOperation.java | 135 ++++++++++++++++++ .../alter_table_create_tag_negative.q | 3 + .../queries/positive/alter_table_create_tag.q | 19 +++ .../alter_table_create_branch_negative.q.out | 2 +- .../alter_table_create_tag_negative.q.out | 12 ++ .../positive/alter_table_create_branch.q.out | 16 +-- .../positive/alter_table_create_tag.q.out | 85 +++++++++++ .../hadoop/hive/ql/parse/AlterClauseParser.g | 20 ++- .../hadoop/hive/ql/parse/HiveLexerParent.g | 1 + .../apache/hadoop/hive/ql/parse/HiveParser.g | 1 + .../hadoop/hive/ql/parse/IdentifiersParser.g | 1 + .../hive/ql/ddl/table/AlterTableType.java | 1 + .../AlterTableCreateSnapshotRefAnalyzer.java} | 39 +++-- .../AlterTableCreateSnapshotRefDesc.java} | 26 ++-- ...AlterTableCreateSnapshotRefOperation.java} | 9 +- .../AlterTableCreateBranchAnalyzer.java | 35 +++++ .../create/AlterTableCreateTagAnalyzer.java | 35 +++++ .../apache/hadoop/hive/ql/metadata/Hive.java | 6 +- .../hive/ql/metadata/HiveStorageHandler.java | 6 +- ...ec.java => AlterTableSnapshotRefSpec.java} | 41 +++--- .../hadoop/hive/ql/plan/HiveOperation.java | 1 + .../plugin/HiveOperationType.java | 1 + .../plugin/sqlstd/Operation2Privilege.java | 2 + 27 files changed, 491 insertions(+), 96 deletions(-) create mode 100644 iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTagExec.java create mode 100644 iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergTagOperation.java create mode 100644 iceberg/iceberg-handler/src/test/queries/negative/alter_table_create_tag_negative.q create mode 100644 iceberg/iceberg-handler/src/test/queries/positive/alter_table_create_tag.q create mode 100644 iceberg/iceberg-handler/src/test/results/negative/alter_table_create_tag_negative.q.out create mode 100644 iceberg/iceberg-handler/src/test/results/positive/alter_table_create_tag.q.out rename ql/src/java/org/apache/hadoop/hive/ql/ddl/table/{branch/create/AlterTableCreateBranchAnalyzer.java => snapshotref/AlterTableCreateSnapshotRefAnalyzer.java} (73%) rename ql/src/java/org/apache/hadoop/hive/ql/ddl/table/{branch/create/AlterTableCreateBranchDesc.java => snapshotref/AlterTableCreateSnapshotRefDesc.java} (55%) rename ql/src/java/org/apache/hadoop/hive/ql/ddl/table/{branch/create/AlterTableCreateBranchOperation.java => snapshotref/AlterTableCreateSnapshotRefOperation.java} (75%) create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/branch/create/AlterTableCreateBranchAnalyzer.java create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/tag/create/AlterTableCreateTagAnalyzer.java rename ql/src/java/org/apache/hadoop/hive/ql/parse/{AlterTableBranchSpec.java => AlterTableSnapshotRefSpec.java} (69%) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index 605f07728d33..a470486ba74d 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -120,7 +120,8 @@ public class HiveIcebergMetaHook implements HiveMetaHook { static final EnumSet SUPPORTED_ALTER_OPS = EnumSet.of( AlterTableType.ADDCOLS, AlterTableType.REPLACE_COLUMNS, AlterTableType.RENAME_COLUMN, AlterTableType.ADDPROPS, AlterTableType.DROPPROPS, AlterTableType.SETPARTITIONSPEC, - AlterTableType.UPDATE_COLUMNS, AlterTableType.RENAME, AlterTableType.EXECUTE, AlterTableType.CREATE_BRANCH); + AlterTableType.UPDATE_COLUMNS, AlterTableType.RENAME, AlterTableType.EXECUTE, AlterTableType.CREATE_BRANCH, + AlterTableType.CREATE_TAG); private static final List MIGRATION_ALLOWED_SOURCE_FORMATS = ImmutableList.of( FileFormat.PARQUET.name().toLowerCase(), FileFormat.ORC.name().toLowerCase(), diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 761482a342d1..30a7ca82a8d7 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -81,8 +81,8 @@ import org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler; import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; -import org.apache.hadoop.hive.ql.parse.AlterTableBranchSpec; import org.apache.hadoop.hive.ql.parse.AlterTableExecuteSpec; +import org.apache.hadoop.hive.ql.parse.AlterTableSnapshotRefSpec; import org.apache.hadoop.hive.ql.parse.PartitionTransform; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.StorageFormat; @@ -789,25 +789,29 @@ private static ExecutorService getDeleteExecutorService(String completeName, int } @Override - public void alterTableBranchOperation(org.apache.hadoop.hive.ql.metadata.Table hmsTable, - AlterTableBranchSpec alterBranchSpec) { + public void alterTableSnapshotRefOperation(org.apache.hadoop.hive.ql.metadata.Table hmsTable, + AlterTableSnapshotRefSpec alterTableSnapshotRefSpec) { TableDesc tableDesc = Utilities.getTableDesc(hmsTable); Table icebergTable = IcebergTableUtil.getTable(conf, tableDesc.getProperties()); Optional.ofNullable(icebergTable.currentSnapshot()).orElseThrow(() -> - new UnsupportedOperationException(String.format("Cannot alter branch on iceberg table" + - " %s.%s which has no snapshot", hmsTable.getDbName(), hmsTable.getTableName()))); + new UnsupportedOperationException(String.format("Cannot alter %s on iceberg table %s.%s which has no snapshot", + alterTableSnapshotRefSpec.getOperationType().getName(), hmsTable.getDbName(), hmsTable.getTableName()))); - switch (alterBranchSpec.getOperationType()) { + switch (alterTableSnapshotRefSpec.getOperationType()) { case CREATE_BRANCH: - AlterTableBranchSpec.CreateBranchSpec createBranchSpec = - (AlterTableBranchSpec.CreateBranchSpec) alterBranchSpec.getOperationParams(); + AlterTableSnapshotRefSpec.CreateSnapshotRefSpec createBranchSpec = + (AlterTableSnapshotRefSpec.CreateSnapshotRefSpec) alterTableSnapshotRefSpec.getOperationParams(); IcebergBranchExec.createBranch(icebergTable, createBranchSpec); break; + case CREATE_TAG: + AlterTableSnapshotRefSpec.CreateSnapshotRefSpec createTagSpec = + (AlterTableSnapshotRefSpec.CreateSnapshotRefSpec) alterTableSnapshotRefSpec.getOperationParams(); + IcebergTagExec.createTag(icebergTable, createTagSpec); + break; default: - throw new UnsupportedOperationException( - String.format("Operation type %s is not supported", alterBranchSpec.getOperationType().name())); + throw new UnsupportedOperationException(String.format( + "Operation type %s is not supported", alterTableSnapshotRefSpec.getOperationType().getName())); } - } @Override diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergBranchExec.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergBranchExec.java index ef910b14b4b3..0115b303580f 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergBranchExec.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergBranchExec.java @@ -19,7 +19,7 @@ package org.apache.iceberg.mr.hive; -import org.apache.hadoop.hive.ql.parse.AlterTableBranchSpec; +import org.apache.hadoop.hive.ql.parse.AlterTableSnapshotRefSpec; import org.apache.iceberg.ManageSnapshots; import org.apache.iceberg.Table; import org.apache.iceberg.util.SnapshotUtil; @@ -38,8 +38,8 @@ private IcebergBranchExec() { * @param table the iceberg table * @param createBranchSpec Get the basic parameters needed to create a branch */ - public static void createBranch(Table table, AlterTableBranchSpec.CreateBranchSpec createBranchSpec) { - String branchName = createBranchSpec.getBranchName(); + public static void createBranch(Table table, AlterTableSnapshotRefSpec.CreateSnapshotRefSpec createBranchSpec) { + String branchName = createBranchSpec.getRefName(); Long snapshotId = null; if (createBranchSpec.getSnapshotId() != null) { snapshotId = createBranchSpec.getSnapshotId(); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTagExec.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTagExec.java new file mode 100644 index 000000000000..705d152bb340 --- /dev/null +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTagExec.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.hive; + +import org.apache.hadoop.hive.ql.parse.AlterTableSnapshotRefSpec; +import org.apache.iceberg.ManageSnapshots; +import org.apache.iceberg.Table; +import org.apache.iceberg.util.SnapshotUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class IcebergTagExec { + + private static final Logger LOG = LoggerFactory.getLogger(IcebergTagExec.class); + + private IcebergTagExec() { + } + + public static void createTag(Table table, AlterTableSnapshotRefSpec.CreateSnapshotRefSpec createTagSpec) { + String tagName = createTagSpec.getRefName(); + Long snapshotId = null; + if (createTagSpec.getSnapshotId() != null) { + snapshotId = createTagSpec.getSnapshotId(); + } else if (createTagSpec.getAsOfTime() != null) { + snapshotId = SnapshotUtil.snapshotIdAsOfTime(table, createTagSpec.getAsOfTime()); + } else { + snapshotId = table.currentSnapshot().snapshotId(); + } + LOG.info("Creating tag {} on iceberg table {} with snapshotId {}", tagName, table.name(), snapshotId); + ManageSnapshots manageSnapshots = table.manageSnapshots(); + manageSnapshots.createTag(tagName, snapshotId); + if (createTagSpec.getMaxRefAgeMs() != null) { + manageSnapshots.setMaxRefAgeMs(tagName, createTagSpec.getMaxRefAgeMs()); + } + + manageSnapshots.commit(); + } +} diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergTagOperation.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergTagOperation.java new file mode 100644 index 000000000000..23a7be72f1f5 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergTagOperation.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.hive; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.junit.Assert; +import org.junit.Test; + +import static org.apache.iceberg.mr.hive.HiveIcebergTestUtils.timestampAfterSnapshot; + +public class TestHiveIcebergTagOperation extends HiveIcebergStorageHandlerWithEngineBase { + @Test + public void testCreateTagWithDefaultConfig() throws InterruptedException, IOException { + Table table = + testTables.createTableWithVersions(shell, "customers", HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + fileFormat, HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, 2); + + String tagName = "test_tag_1"; + shell.executeStatement(String.format("ALTER TABLE customers CREATE TAG %s", tagName)); + table.refresh(); + SnapshotRef ref = table.refs().get(tagName); + Assert.assertEquals(table.currentSnapshot().snapshotId(), ref.snapshotId()); + Assert.assertNull(ref.maxRefAgeMs()); + + // creating a tag which is already exists will fail + try { + shell.executeStatement(String.format("ALTER TABLE customers CREATE TAG %s", tagName)); + } catch (Throwable e) { + while (e.getCause() != null) { + e = e.getCause(); + } + Assert.assertTrue(e.getMessage().contains("Ref test_tag_1 already exists")); + } + } + + @Test + public void testCreateTagWithSnapshotId() throws InterruptedException, IOException { + Table table = + testTables.createTableWithVersions(shell, "customers", HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + fileFormat, HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, 2); + + String tagName = "test_tag_1"; + Long snapshotId = table.history().get(0).snapshotId(); + shell.executeStatement(String.format("ALTER TABLE customers CREATE TAG %s FOR SYSTEM_VERSION AS OF %d", + tagName, snapshotId)); + table.refresh(); + SnapshotRef ref = table.refs().get(tagName); + Assert.assertEquals(snapshotId.longValue(), ref.snapshotId()); + Assert.assertNull(ref.maxRefAgeMs()); + } + + @Test + public void testCreateTagWithTimeStamp() throws InterruptedException, IOException { + Table table = + testTables.createTableWithVersions(shell, "customers", HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + fileFormat, HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, 2); + + String tagName = "test_tag_1"; + Long snapshotId = table.history().get(0).snapshotId(); + + shell.executeStatement(String.format("ALTER TABLE customers CREATE TAG %s FOR SYSTEM_TIME AS OF '%s'", + tagName, timestampAfterSnapshot(table, 0))); + table.refresh(); + SnapshotRef ref = table.refs().get(tagName); + Assert.assertEquals(snapshotId.longValue(), ref.snapshotId()); + } + + @Test + public void testCreateTagWithMaxRefAge() throws InterruptedException, IOException { + Table table = + testTables.createTableWithVersions(shell, "customers", HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + fileFormat, HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, 2); + + String tagName = "test_tag_1"; + long maxRefAge = 5L; + shell.executeStatement(String.format("ALTER TABLE customers CREATE TAG %s RETAIN %d DAYS", tagName, maxRefAge)); + table.refresh(); + SnapshotRef ref = table.refs().get(tagName); + Assert.assertEquals(table.currentSnapshot().snapshotId(), ref.snapshotId()); + Assert.assertEquals(TimeUnit.DAYS.toMillis(maxRefAge), ref.maxRefAgeMs().longValue()); + } + + @Test + public void testCreateTagWithAllCustomConfig() throws IOException, InterruptedException { + Table table = + testTables.createTableWithVersions(shell, "customers", HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + fileFormat, HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, 2); + + String tagName = "test_tag_1"; + Long snapshotId = table.history().get(0).snapshotId(); + long maxRefAge = 5L; + shell.executeStatement(String.format("ALTER TABLE customers CREATE TAG %s FOR SYSTEM_VERSION AS OF %d RETAIN" + + " %d DAYS", + tagName, snapshotId, maxRefAge)); + table.refresh(); + SnapshotRef ref = table.refs().get(tagName); + Assert.assertEquals(snapshotId.longValue(), ref.snapshotId()); + Assert.assertEquals(TimeUnit.DAYS.toMillis(maxRefAge), ref.maxRefAgeMs().longValue()); + } + + @Test + public void testCreateTagWithNonIcebergTable() { + shell.executeStatement("create table nonice_tbl (id int, name string)"); + + String tagName = "test_tag_1"; + try { + shell.executeStatement(String.format("ALTER TABLE nonice_tbl CREATE TAG %s", tagName)); + } catch (Throwable e) { + while (e.getCause() != null) { + e = e.getCause(); + } + Assert.assertTrue(e.getMessage().contains("Not an iceberg table")); + } + } +} diff --git a/iceberg/iceberg-handler/src/test/queries/negative/alter_table_create_tag_negative.q b/iceberg/iceberg-handler/src/test/queries/negative/alter_table_create_tag_negative.q new file mode 100644 index 000000000000..81ccaf31bf7c --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/negative/alter_table_create_tag_negative.q @@ -0,0 +1,3 @@ +create table ice_tbl (id int, name string) Stored by Iceberg; + +alter table ice_tbl create tag test_branch_1; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/alter_table_create_tag.q b/iceberg/iceberg-handler/src/test/queries/positive/alter_table_create_tag.q new file mode 100644 index 000000000000..dbbde623d84b --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/alter_table_create_tag.q @@ -0,0 +1,19 @@ +-- SORT_QUERY_RESULTS +set hive.explain.user=false; + +create table iceTbl (id int, name string) Stored by Iceberg; + +-- creating tag requires table to have current snapshot. here insert some values to generate current snapshot +insert into iceTbl values(1, 'jack'); + +-- create tag with default values based on the current snapshotId +explain alter table iceTbl create tag test_tag_1; +alter table iceTbl create tag test_tag_1; +select name, max_reference_age_in_ms from default.iceTbl.refs where type='TAG'; + +-- create a tag which could be retained 5 days based on the current snapshotId +insert into iceTbl values(2, 'bob'); +explain alter table iceTbl create tag test_tag_2 retain 5 days; +alter table iceTbl create tag test_tag_2 retain 5 days; +select name, max_reference_age_in_ms from default.iceTbl.refs where type='TAG'; + diff --git a/iceberg/iceberg-handler/src/test/results/negative/alter_table_create_branch_negative.q.out b/iceberg/iceberg-handler/src/test/results/negative/alter_table_create_branch_negative.q.out index 4d5e2812e4b3..75c7f0936c6e 100644 --- a/iceberg/iceberg-handler/src/test/results/negative/alter_table_create_branch_negative.q.out +++ b/iceberg/iceberg-handler/src/test/results/negative/alter_table_create_branch_negative.q.out @@ -9,4 +9,4 @@ POSTHOOK: Output: default@ice_tbl PREHOOK: query: alter table ice_tbl create branch test_branch_1 PREHOOK: type: ALTERTABLE_CREATEBRANCH PREHOOK: Input: default@ice_tbl -FAILED: Execution Error, return code 40000 from org.apache.hadoop.hive.ql.ddl.DDLTask. java.lang.UnsupportedOperationException: Cannot alter branch on iceberg table default.ice_tbl which has no snapshot +FAILED: Execution Error, return code 40000 from org.apache.hadoop.hive.ql.ddl.DDLTask. java.lang.UnsupportedOperationException: Cannot alter create branch on iceberg table default.ice_tbl which has no snapshot diff --git a/iceberg/iceberg-handler/src/test/results/negative/alter_table_create_tag_negative.q.out b/iceberg/iceberg-handler/src/test/results/negative/alter_table_create_tag_negative.q.out new file mode 100644 index 000000000000..7d4d91b5745f --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/negative/alter_table_create_tag_negative.q.out @@ -0,0 +1,12 @@ +PREHOOK: query: create table ice_tbl (id int, name string) Stored by Iceberg +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_tbl +POSTHOOK: query: create table ice_tbl (id int, name string) Stored by Iceberg +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_tbl +PREHOOK: query: alter table ice_tbl create tag test_branch_1 +PREHOOK: type: ALTERTABLE_CREATETAG +PREHOOK: Input: default@ice_tbl +FAILED: Execution Error, return code 40000 from org.apache.hadoop.hive.ql.ddl.DDLTask. java.lang.UnsupportedOperationException: Cannot alter create tag on iceberg table default.ice_tbl which has no snapshot diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_table_create_branch.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_table_create_branch.q.out index 190f190ec893..8cc5c2a0c479 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/alter_table_create_branch.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/alter_table_create_branch.q.out @@ -25,9 +25,9 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 - CreateBranch operation + CreateSnapshotRef Operation table name: default.iceTbl - spec: AlterTableBranchSpec{operationType=CREATE_BRANCH, operationParams=CreateBranchSpec{branchName=test_branch_1, snapshotId=null, asOfTime=null, maxRefAgeMs=null, minSnapshotsToKeep=null, maxSnapshotAgeMs=null}} + spec: AlterTableSnapshotRefSpec{operationType=CREATE_BRANCH, operationParams=CreateSnapshotRefSpec{branchName=test_branch_1}} PREHOOK: query: alter table iceTbl create branch test_branch_1 PREHOOK: type: ALTERTABLE_CREATEBRANCH @@ -63,9 +63,9 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 - CreateBranch operation + CreateSnapshotRef Operation table name: default.iceTbl - spec: AlterTableBranchSpec{operationType=CREATE_BRANCH, operationParams=CreateBranchSpec{branchName=test_branch_2, snapshotId=null, asOfTime=null, maxRefAgeMs=432000000, minSnapshotsToKeep=null, maxSnapshotAgeMs=null}} + spec: AlterTableSnapshotRefSpec{operationType=CREATE_BRANCH, operationParams=CreateSnapshotRefSpec{branchName=test_branch_2, maxRefAgeMs=432000000}} PREHOOK: query: alter table iceTbl create branch test_branch_2 retain 5 days PREHOOK: type: ALTERTABLE_CREATEBRANCH @@ -102,9 +102,9 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 - CreateBranch operation + CreateSnapshotRef Operation table name: default.iceTbl - spec: AlterTableBranchSpec{operationType=CREATE_BRANCH, operationParams=CreateBranchSpec{branchName=test_branch_3, snapshotId=null, asOfTime=null, maxRefAgeMs=null, minSnapshotsToKeep=5, maxSnapshotAgeMs=null}} + spec: AlterTableSnapshotRefSpec{operationType=CREATE_BRANCH, operationParams=CreateSnapshotRefSpec{branchName=test_branch_3, minSnapshotsToKeep=5}} PREHOOK: query: alter table iceTbl create branch test_branch_3 with snapshot retention 5 snapshots PREHOOK: type: ALTERTABLE_CREATEBRANCH @@ -142,9 +142,9 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 - CreateBranch operation + CreateSnapshotRef Operation table name: default.iceTbl - spec: AlterTableBranchSpec{operationType=CREATE_BRANCH, operationParams=CreateBranchSpec{branchName=test_branch_4, snapshotId=null, asOfTime=null, maxRefAgeMs=null, minSnapshotsToKeep=5, maxSnapshotAgeMs=432000000}} + spec: AlterTableSnapshotRefSpec{operationType=CREATE_BRANCH, operationParams=CreateSnapshotRefSpec{branchName=test_branch_4, minSnapshotsToKeep=5, maxSnapshotAgeMs=432000000}} PREHOOK: query: alter table iceTbl create branch test_branch_4 with snapshot retention 5 snapshots 5 days PREHOOK: type: ALTERTABLE_CREATEBRANCH diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_table_create_tag.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_table_create_tag.q.out new file mode 100644 index 000000000000..789cb3c9ecb2 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/alter_table_create_tag.q.out @@ -0,0 +1,85 @@ +PREHOOK: query: create table iceTbl (id int, name string) Stored by Iceberg +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@iceTbl +POSTHOOK: query: create table iceTbl (id int, name string) Stored by Iceberg +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@iceTbl +PREHOOK: query: insert into iceTbl values(1, 'jack') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@icetbl +POSTHOOK: query: insert into iceTbl values(1, 'jack') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@icetbl +PREHOOK: query: explain alter table iceTbl create tag test_tag_1 +PREHOOK: type: ALTERTABLE_CREATETAG +PREHOOK: Input: default@icetbl +POSTHOOK: query: explain alter table iceTbl create tag test_tag_1 +POSTHOOK: type: ALTERTABLE_CREATETAG +POSTHOOK: Input: default@icetbl +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + CreateSnapshotRef Operation + table name: default.iceTbl + spec: AlterTableSnapshotRefSpec{operationType=CREATE_TAG, operationParams=CreateSnapshotRefSpec{branchName=test_tag_1}} + +PREHOOK: query: alter table iceTbl create tag test_tag_1 +PREHOOK: type: ALTERTABLE_CREATETAG +PREHOOK: Input: default@icetbl +POSTHOOK: query: alter table iceTbl create tag test_tag_1 +POSTHOOK: type: ALTERTABLE_CREATETAG +POSTHOOK: Input: default@icetbl +PREHOOK: query: select name, max_reference_age_in_ms from default.iceTbl.refs where type='TAG' +PREHOOK: type: QUERY +PREHOOK: Input: default@icetbl +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select name, max_reference_age_in_ms from default.iceTbl.refs where type='TAG' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@icetbl +POSTHOOK: Output: hdfs://### HDFS PATH ### +test_tag_1 NULL +PREHOOK: query: insert into iceTbl values(2, 'bob') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@icetbl +POSTHOOK: query: insert into iceTbl values(2, 'bob') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@icetbl +PREHOOK: query: explain alter table iceTbl create tag test_tag_2 retain 5 days +PREHOOK: type: ALTERTABLE_CREATETAG +PREHOOK: Input: default@icetbl +POSTHOOK: query: explain alter table iceTbl create tag test_tag_2 retain 5 days +POSTHOOK: type: ALTERTABLE_CREATETAG +POSTHOOK: Input: default@icetbl +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + CreateSnapshotRef Operation + table name: default.iceTbl + spec: AlterTableSnapshotRefSpec{operationType=CREATE_TAG, operationParams=CreateSnapshotRefSpec{branchName=test_tag_2, maxRefAgeMs=432000000}} + +PREHOOK: query: alter table iceTbl create tag test_tag_2 retain 5 days +PREHOOK: type: ALTERTABLE_CREATETAG +PREHOOK: Input: default@icetbl +POSTHOOK: query: alter table iceTbl create tag test_tag_2 retain 5 days +POSTHOOK: type: ALTERTABLE_CREATETAG +POSTHOOK: Input: default@icetbl +PREHOOK: query: select name, max_reference_age_in_ms from default.iceTbl.refs where type='TAG' +PREHOOK: type: QUERY +PREHOOK: Input: default@icetbl +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select name, max_reference_age_in_ms from default.iceTbl.refs where type='TAG' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@icetbl +POSTHOOK: Output: hdfs://### HDFS PATH ### +test_tag_1 NULL +test_tag_2 432000000 diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g index f785b583be9b..3dc8c4a3b0ba 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g @@ -75,6 +75,7 @@ alterTableStatementSuffix | alterStatementSuffixSetPartSpec | alterStatementSuffixExecute | alterStatementSuffixCreateBranch + | alterStatementSuffixCreateTag | alterStatementSuffixConvert ; @@ -481,12 +482,12 @@ alterStatementSuffixExecute alterStatementSuffixCreateBranch @init { gParent.pushMsg("alter table create branch", state); } @after { gParent.popMsg(state); } - : KW_CREATE KW_BRANCH branchName=identifier snapshotIdOfBranch? branchRetain? retentionOfSnapshots? - -> ^(TOK_ALTERTABLE_CREATE_BRANCH $branchName snapshotIdOfBranch? branchRetain? retentionOfSnapshots?) + : KW_CREATE KW_BRANCH branchName=identifier snapshotIdOfRef? refRetain? retentionOfSnapshots? + -> ^(TOK_ALTERTABLE_CREATE_BRANCH $branchName snapshotIdOfRef? refRetain? retentionOfSnapshots?) ; -snapshotIdOfBranch -@init { gParent.pushMsg("alter table create branch as of version", state); } +snapshotIdOfRef +@init { gParent.pushMsg("alter table create branch/tag as of version", state); } @after { gParent.popMsg(state); } : KW_FOR KW_SYSTEM_VERSION KW_AS KW_OF snapshotId=Number -> ^(TOK_AS_OF_VERSION $snapshotId) @@ -495,8 +496,8 @@ snapshotIdOfBranch -> ^(TOK_AS_OF_TIME $asOfTime) ; -branchRetain -@init { gParent.pushMsg("alter table create branch RETAIN", state); } +refRetain +@init { gParent.pushMsg("alter table create branch/tag RETAIN", state); } @after { gParent.popMsg(state); } : KW_RETAIN maxRefAge=Number timeUnit=timeUnitQualifiers -> ^(TOK_RETAIN $maxRefAge $timeUnit) @@ -509,6 +510,13 @@ retentionOfSnapshots -> ^(TOK_WITH_SNAPSHOT_RETENTION $minSnapshotsToKeep ($maxSnapshotAge $timeUnit)?) ; +alterStatementSuffixCreateTag +@init { gParent.pushMsg("alter table create tag", state); } +@after { gParent.popMsg(state); } + : KW_CREATE KW_TAG tagName=identifier snapshotIdOfRef? refRetain? + -> ^(TOK_ALTERTABLE_CREATE_TAG $tagName snapshotIdOfRef? refRetain?) + ; + fileFormat @init { gParent.pushMsg("file format specification", state); } @after { gParent.popMsg(state); } diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g index c7cbc73222c5..4b5326bbdf90 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g @@ -396,6 +396,7 @@ KW_BRANCH: 'BRANCH'; KW_SNAPSHOTS: 'SNAPSHOTS'; KW_RETAIN: 'RETAIN'; KW_RETENTION: 'RETENTION'; +KW_TAG: 'TAG'; // Operators // NOTE: if you add a new function/operator, add it to sysFuncNames so that describe function _FUNC_ will work. diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 0ca781526419..406e9a73d2fc 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -220,6 +220,7 @@ TOK_ALTERTABLE_OWNER; TOK_ALTERTABLE_SETPARTSPEC; TOK_ALTERTABLE_EXECUTE; TOK_ALTERTABLE_CREATE_BRANCH; +TOK_ALTERTABLE_CREATE_TAG; TOK_RETAIN; TOK_WITH_SNAPSHOT_RETENTION; TOK_ALTERTABLE_CONVERT; diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g index e6e0ae58ca5d..880baf1baf71 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g @@ -985,6 +985,7 @@ nonReserved | KW_EXPIRE_SNAPSHOTS | KW_SET_CURRENT_SNAPSHOT | KW_BRANCH | KW_SNAPSHOTS | KW_RETAIN | KW_RETENTION + | KW_TAG ; //The following SQL2011 reserved keywords are used as function name only, but not as identifiers. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java index 7d6164d6e07b..17a28bd0d9be 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java @@ -41,6 +41,7 @@ public enum AlterTableType { SETPARTITIONSPEC("set partition spec"), EXECUTE("execute"), CREATE_BRANCH("create branch"), + CREATE_TAG("create tag"), // constraint ADD_CONSTRAINT("add constraint"), DROP_CONSTRAINT("drop constraint"), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/branch/create/AlterTableCreateBranchAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/AlterTableCreateSnapshotRefAnalyzer.java similarity index 73% rename from ql/src/java/org/apache/hadoop/hive/ql/ddl/table/branch/create/AlterTableCreateBranchAnalyzer.java rename to ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/AlterTableCreateSnapshotRefAnalyzer.java index a742d39b84fb..7d02d8a7d9f0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/branch/create/AlterTableCreateBranchAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/AlterTableCreateSnapshotRefAnalyzer.java @@ -16,38 +16,35 @@ * limitations under the License. */ -package org.apache.hadoop.hive.ql.ddl.table.branch.create; +package org.apache.hadoop.hive.ql.ddl.table.snapshotref; import java.time.ZoneId; import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; - import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.common.type.TimestampTZ; import org.apache.hadoop.hive.common.type.TimestampTZUtil; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.QueryState; -import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory; import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.ddl.DDLWork; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableAnalyzer; +import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; import org.apache.hadoop.hive.ql.ddl.table.AlterTableType; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ASTNode; -import org.apache.hadoop.hive.ql.parse.AlterTableBranchSpec; +import org.apache.hadoop.hive.ql.parse.AlterTableSnapshotRefSpec; import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.session.SessionState; -import static org.apache.hadoop.hive.ql.parse.AlterTableBranchSpec.AlterBranchOperationType.CREATE_BRANCH; - -@DDLSemanticAnalyzerFactory.DDLType(types = HiveParser.TOK_ALTERTABLE_CREATE_BRANCH) -public class AlterTableCreateBranchAnalyzer extends AbstractAlterTableAnalyzer { +public abstract class AlterTableCreateSnapshotRefAnalyzer extends AbstractAlterTableAnalyzer { + protected AlterTableType alterTableType; - public AlterTableCreateBranchAnalyzer(QueryState queryState) throws SemanticException { + public AlterTableCreateSnapshotRefAnalyzer(QueryState queryState) throws SemanticException { super(queryState); } @@ -55,11 +52,11 @@ public AlterTableCreateBranchAnalyzer(QueryState queryState) throws SemanticExce protected void analyzeCommand(TableName tableName, Map partitionSpec, ASTNode command) throws SemanticException { Table table = getTable(tableName); - validateAlterTableType(table, AlterTableType.CREATE_BRANCH, false); DDLUtils.validateTableIsIceberg(table); inputs.add(new ReadEntity(table)); + validateAlterTableType(table, alterTableType, false); - String branchName = command.getChild(0).getText(); + String refName = command.getChild(0).getText(); Long snapshotId = null; Long asOfTime = null; Long maxRefAgeMs = null; @@ -80,8 +77,8 @@ protected void analyzeCommand(TableName tableName, Map partition case HiveParser.TOK_RETAIN: String maxRefAge = childNode.getChild(0).getText(); String timeUnitOfBranchRetain = childNode.getChild(1).getText(); - maxRefAgeMs = TimeUnit.valueOf(timeUnitOfBranchRetain.toUpperCase(Locale.ENGLISH)) - .toMillis(Long.parseLong(maxRefAge)); + maxRefAgeMs = + TimeUnit.valueOf(timeUnitOfBranchRetain.toUpperCase(Locale.ENGLISH)).toMillis(Long.parseLong(maxRefAge)); break; case HiveParser.TOK_WITH_SNAPSHOT_RETENTION: minSnapshotsToKeep = Integer.valueOf(childNode.getChild(0).getText()); @@ -93,15 +90,17 @@ protected void analyzeCommand(TableName tableName, Map partition } break; default: - throw new SemanticException("Unrecognized token in ALTER CREATE BRANCH statement"); + throw new SemanticException("Unrecognized token in ALTER " + alterTableType.getName() + " statement"); } } - AlterTableBranchSpec.CreateBranchSpec - createBranchspec = new AlterTableBranchSpec.CreateBranchSpec(branchName, snapshotId, asOfTime, - maxRefAgeMs, minSnapshotsToKeep, maxSnapshotAgeMs); - AlterTableBranchSpec alterTableBranchSpec = new AlterTableBranchSpec(CREATE_BRANCH, createBranchspec); - AlterTableCreateBranchDesc desc = new AlterTableCreateBranchDesc(tableName, alterTableBranchSpec); - rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc))); + AlterTableSnapshotRefSpec.CreateSnapshotRefSpec createSnapshotRefSpec = + new AlterTableSnapshotRefSpec.CreateSnapshotRefSpec(refName, snapshotId, asOfTime, + maxRefAgeMs, minSnapshotsToKeep, maxSnapshotAgeMs); + AlterTableSnapshotRefSpec alterTableSnapshotRefSpec + = new AlterTableSnapshotRefSpec(alterTableType, createSnapshotRefSpec); + AbstractAlterTableDesc alterTableDesc = + new AlterTableCreateSnapshotRefDesc(alterTableType, tableName, alterTableSnapshotRefSpec); + rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterTableDesc))); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/branch/create/AlterTableCreateBranchDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/AlterTableCreateSnapshotRefDesc.java similarity index 55% rename from ql/src/java/org/apache/hadoop/hive/ql/ddl/table/branch/create/AlterTableCreateBranchDesc.java rename to ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/AlterTableCreateSnapshotRefDesc.java index 43ef199b30cb..e13154845418 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/branch/create/AlterTableCreateBranchDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/AlterTableCreateSnapshotRefDesc.java @@ -16,34 +16,34 @@ * limitations under the License. */ -package org.apache.hadoop.hive.ql.ddl.table.branch.create; +package org.apache.hadoop.hive.ql.ddl.table.snapshotref; import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; import org.apache.hadoop.hive.ql.ddl.table.AlterTableType; -import org.apache.hadoop.hive.ql.parse.AlterTableBranchSpec; +import org.apache.hadoop.hive.ql.parse.AlterTableSnapshotRefSpec; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.Explain; -import org.apache.hadoop.hive.ql.plan.Explain.Level; -@Explain(displayName = "CreateBranch operation", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) -public class AlterTableCreateBranchDesc extends AbstractAlterTableDesc { +@Explain(displayName = "CreateSnapshotRef Operation", explainLevels = { Explain.Level.USER, Explain.Level.DEFAULT, + Explain.Level.EXTENDED }) +public class AlterTableCreateSnapshotRefDesc extends AbstractAlterTableDesc { private static final long serialVersionUID = 1L; - private final AlterTableBranchSpec alterTableBranchSpec; + protected AlterTableSnapshotRefSpec alterTableSnapshotRefSpec; - public AlterTableCreateBranchDesc(TableName tableName, AlterTableBranchSpec alterTableBranchSpec) + public AlterTableCreateSnapshotRefDesc(AlterTableType alterTableType, TableName tableName, AlterTableSnapshotRefSpec alterTableSnapshotRefSpec) throws SemanticException { - super(AlterTableType.CREATE_BRANCH, tableName, null, null, false, false, null); - this.alterTableBranchSpec = alterTableBranchSpec; + super(alterTableType, tableName, null, null, false, false, null); + this.alterTableSnapshotRefSpec = alterTableSnapshotRefSpec; } - public AlterTableBranchSpec getAlterTableBranchSpec() { - return alterTableBranchSpec; + public AlterTableSnapshotRefSpec getAlterTableSnapshotRefSpec() { + return alterTableSnapshotRefSpec; } - @Explain(displayName = "spec", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "spec", explainLevels = { Explain.Level.USER, Explain.Level.DEFAULT, Explain.Level.EXTENDED }) public String getExplainOutput() { - return alterTableBranchSpec.toString(); + return alterTableSnapshotRefSpec.toString(); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/branch/create/AlterTableCreateBranchOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/AlterTableCreateSnapshotRefOperation.java similarity index 75% rename from ql/src/java/org/apache/hadoop/hive/ql/ddl/table/branch/create/AlterTableCreateBranchOperation.java rename to ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/AlterTableCreateSnapshotRefOperation.java index 46fc6f37d459..a36c400ef451 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/branch/create/AlterTableCreateBranchOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/AlterTableCreateSnapshotRefOperation.java @@ -16,22 +16,21 @@ * limitations under the License. */ -package org.apache.hadoop.hive.ql.ddl.table.branch.create; +package org.apache.hadoop.hive.ql.ddl.table.snapshotref; import org.apache.hadoop.hive.ql.ddl.DDLOperation; import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; import org.apache.hadoop.hive.ql.metadata.Table; -public class AlterTableCreateBranchOperation extends DDLOperation { - - public AlterTableCreateBranchOperation(DDLOperationContext context, AlterTableCreateBranchDesc desc) { +public class AlterTableCreateSnapshotRefOperation extends DDLOperation { + public AlterTableCreateSnapshotRefOperation(DDLOperationContext context, AlterTableCreateSnapshotRefDesc desc) { super(context, desc); } @Override public int execute() throws Exception { Table table = context.getDb().getTable(desc.getFullTableName()); - context.getDb().alterTableBranchOperation(table, desc.getAlterTableBranchSpec()); + context.getDb().alterTableSnapshotRefOperation(table, desc.getAlterTableSnapshotRefSpec()); return 0; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/branch/create/AlterTableCreateBranchAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/branch/create/AlterTableCreateBranchAnalyzer.java new file mode 100644 index 000000000000..c49eea5d6f31 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/branch/create/AlterTableCreateBranchAnalyzer.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.snapshotref.branch.create; + +import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory; +import org.apache.hadoop.hive.ql.ddl.table.snapshotref.AlterTableCreateSnapshotRefAnalyzer; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableType; +import org.apache.hadoop.hive.ql.parse.HiveParser; +import org.apache.hadoop.hive.ql.parse.SemanticException; + +@DDLSemanticAnalyzerFactory.DDLType(types = HiveParser.TOK_ALTERTABLE_CREATE_BRANCH) +public class AlterTableCreateBranchAnalyzer extends AlterTableCreateSnapshotRefAnalyzer { + + public AlterTableCreateBranchAnalyzer(QueryState queryState) throws SemanticException { + super(queryState); + alterTableType = AlterTableType.CREATE_BRANCH; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/tag/create/AlterTableCreateTagAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/tag/create/AlterTableCreateTagAnalyzer.java new file mode 100644 index 000000000000..3a66760e4dbc --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/snapshotref/tag/create/AlterTableCreateTagAnalyzer.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.snapshotref.tag.create; + +import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory; +import org.apache.hadoop.hive.ql.ddl.table.snapshotref.AlterTableCreateSnapshotRefAnalyzer; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableType; +import org.apache.hadoop.hive.ql.parse.HiveParser; +import org.apache.hadoop.hive.ql.parse.SemanticException; + +@DDLSemanticAnalyzerFactory.DDLType(types = HiveParser.TOK_ALTERTABLE_CREATE_TAG) +public class AlterTableCreateTagAnalyzer extends AlterTableCreateSnapshotRefAnalyzer { + + public AlterTableCreateTagAnalyzer(QueryState queryState) throws SemanticException { + super(queryState); + alterTableType = AlterTableType.CREATE_TAG; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 5c91bc1abc61..e013db7db266 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -222,7 +222,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.HiveMaterializedViewUtils; import org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPrunerUtils; import org.apache.hadoop.hive.ql.parse.ASTNode; -import org.apache.hadoop.hive.ql.parse.AlterTableBranchSpec; +import org.apache.hadoop.hive.ql.parse.AlterTableSnapshotRefSpec; import org.apache.hadoop.hive.ql.parse.AlterTableExecuteSpec; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -6723,10 +6723,10 @@ public void alterTableExecuteOperation(Table table, AlterTableExecuteSpec execut } } - public void alterTableBranchOperation(Table table, AlterTableBranchSpec createBranchSpec) throws HiveException { + public void alterTableSnapshotRefOperation(Table table, AlterTableSnapshotRefSpec alterTableSnapshotRefSpec) throws HiveException { try { HiveStorageHandler storageHandler = createStorageHandler(table.getTTable()); - storageHandler.alterTableBranchOperation(table, createBranchSpec); + storageHandler.alterTableSnapshotRefOperation(table, alterTableSnapshotRefSpec); } catch (Exception e) { throw new HiveException(e); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java index 9ff2d3645c09..c04e68ba3bc4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java @@ -44,7 +44,7 @@ import org.apache.hadoop.hive.ql.ddl.table.create.like.CreateTableLikeDesc; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.io.StorageFormatDescriptor; -import org.apache.hadoop.hive.ql.parse.AlterTableBranchSpec; +import org.apache.hadoop.hive.ql.parse.AlterTableSnapshotRefSpec; import org.apache.hadoop.hive.ql.parse.AlterTableExecuteSpec; import org.apache.hadoop.hive.ql.parse.StorageFormat.StorageHandlerTypes; import org.apache.hadoop.hive.ql.parse.TransformSpec; @@ -636,8 +636,8 @@ default void validateSinkDesc(FileSinkDesc sinkDesc) throws SemanticException { default void executeOperation(org.apache.hadoop.hive.ql.metadata.Table table, AlterTableExecuteSpec executeSpec) { } - default void alterTableBranchOperation(org.apache.hadoop.hive.ql.metadata.Table table, - AlterTableBranchSpec alterBranchSpec) { + default void alterTableSnapshotRefOperation(org.apache.hadoop.hive.ql.metadata.Table table, + AlterTableSnapshotRefSpec alterTableSnapshotRefSpec) { } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTableBranchSpec.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTableSnapshotRefSpec.java similarity index 69% rename from ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTableBranchSpec.java rename to ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTableSnapshotRefSpec.java index a842218cf5eb..e02e6646c233 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTableBranchSpec.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTableSnapshotRefSpec.java @@ -19,22 +19,19 @@ package org.apache.hadoop.hive.ql.parse; import com.google.common.base.MoreObjects; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableType; -public class AlterTableBranchSpec { +public class AlterTableSnapshotRefSpec { - public enum AlterBranchOperationType { - CREATE_BRANCH - } - - private final AlterBranchOperationType operationType; + private final AlterTableType operationType; private final T operationParams; - public AlterTableBranchSpec(AlterBranchOperationType type, T value) { + public AlterTableSnapshotRefSpec(AlterTableType type, T value) { this.operationType = type; this.operationParams = value; } - public AlterBranchOperationType getOperationType() { + public AlterTableType getOperationType() { return operationType; } @@ -48,17 +45,17 @@ public String toString() { .add("operationParams", operationParams).toString(); } - public static class CreateBranchSpec { + public static class CreateSnapshotRefSpec { - private final String branchName; - private final Long snapshotId; - private final Long asOfTime; - private final Long maxRefAgeMs; - private final Integer minSnapshotsToKeep; - private final Long maxSnapshotAgeMs; + private String refName; + private Long snapshotId; + private Long asOfTime; + private Long maxRefAgeMs; + private Integer minSnapshotsToKeep; + private Long maxSnapshotAgeMs; - public String getBranchName() { - return branchName; + public String getRefName() { + return refName; } public Long getSnapshotId() { @@ -81,9 +78,9 @@ public Long getMaxSnapshotAgeMs() { return maxSnapshotAgeMs; } - public CreateBranchSpec(String branchName, Long snapShotId, Long asOfTime, Long maxRefAgeMs, - Integer minSnapshotsToKeep, Long maxSnapshotAgeMs) { - this.branchName = branchName; + public CreateSnapshotRefSpec(String refName, Long snapShotId, Long asOfTime, Long maxRefAgeMs, + Integer minSnapshotsToKeep, Long maxSnapshotAgeMs) { + this.refName = refName; this.snapshotId = snapShotId; this.asOfTime = asOfTime; this.maxRefAgeMs = maxRefAgeMs; @@ -92,9 +89,9 @@ public CreateBranchSpec(String branchName, Long snapShotId, Long asOfTime, Long } public String toString() { - return MoreObjects.toStringHelper(this).add("branchName", branchName).add("snapshotId", snapshotId) + return MoreObjects.toStringHelper(this).add("branchName", refName).add("snapshotId", snapshotId) .add("asOfTime", asOfTime).add("maxRefAgeMs", maxRefAgeMs).add("minSnapshotsToKeep", minSnapshotsToKeep) - .add("maxSnapshotAgeMs", maxSnapshotAgeMs).toString(); + .add("maxSnapshotAgeMs", maxSnapshotAgeMs).omitNullValues().toString(); } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java index d83bba55da36..4385d10ca5e7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java @@ -78,6 +78,7 @@ public enum HiveOperation { ALTERTABLE_SETPARTSPEC("ALTERTABLE_SETPARTSPEC", HiveParser.TOK_ALTERTABLE_SETPARTSPEC, null, null), ALTERTABLE_EXECUTE("ALTERTABLE_EXECUTE", HiveParser.TOK_ALTERTABLE_EXECUTE, null, null), ALTERTABLE_CREATEBRANCH("ALTERTABLE_CREATEBRANCH", HiveParser.TOK_ALTERTABLE_CREATE_BRANCH, null, null), + ALTERTABLE_CREATETAG("ALTERTABLE_CREATETAG", HiveParser.TOK_ALTERTABLE_CREATE_TAG, null, null), ALTERTABLE_CONVERT("ALTERTABLE_CONVERT", HiveParser.TOK_ALTERTABLE_CONVERT, null, null), ALTERTABLE_SERIALIZER("ALTERTABLE_SERIALIZER", HiveParser.TOK_ALTERTABLE_SERIALIZER, new Privilege[]{Privilege.ALTER_METADATA}, null), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java index a2279d9737ff..909ac7f53186 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java @@ -140,6 +140,7 @@ public enum HiveOperationType { ALTERTABLE_COMPACT, ALTERTABLE_UPDATECOLUMNS, ALTERTABLE_CREATEBRANCH, + ALTERTABLE_CREATETAG, SHOW_COMPACTIONS, SHOW_TRANSACTIONS, ABORT_TRANSACTIONS, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java index 75abbf754e8b..48b7257dba59 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java @@ -242,6 +242,8 @@ public HivePrivilegeObjectType getObjectType() { PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR)); op2Priv.put(HiveOperationType.ALTERTABLE_CREATEBRANCH, PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR)); + op2Priv.put(HiveOperationType.ALTERTABLE_CREATETAG, + PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR)); // require view ownership for alter/drop view op2Priv.put(HiveOperationType.ALTERVIEW_PROPERTIES,