From 6f480cd40b8ddf7383a48de878b96bffbdecea3b Mon Sep 17 00:00:00 2001 From: Krisztian Kasa Date: Wed, 15 Feb 2023 08:58:36 +0100 Subject: [PATCH] HIVE-27073: Apply SerDe properties when creating materialized view (Krisztian Kasa, reviewed by Laszlo Vegh, Steve Carlin) --- .../test/queries/positive/mv_iceberg_partitioned_orc.q | 2 ++ .../queries/positive/mv_iceberg_partitioned_orc2.q | 2 ++ .../results/positive/mv_iceberg_partitioned_orc.q.out | 10 ++++++---- .../results/positive/mv_iceberg_partitioned_orc2.q.out | 10 ++++++---- .../ql/ddl/view/create/CreateMaterializedViewDesc.java | 6 ++++++ 5 files changed, 22 insertions(+), 8 deletions(-) diff --git a/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_partitioned_orc.q b/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_partitioned_orc.q index 88f03ba83e1e..44a5561f926b 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_partitioned_orc.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_partitioned_orc.q @@ -2,6 +2,8 @@ --! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/ -- Mask random snapshot id --! qt:replace:/(\s+current-snapshot-id\s+)\d+(\s*)/$1#SnapshotId#/ +-- Mask the totalSize value as it can change at file format library update +--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/ -- SORT_QUERY_RESULTS drop materialized view if exists mat1; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_partitioned_orc2.q b/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_partitioned_orc2.q index d0be6ea68fe3..3cb8c04f1d0b 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_partitioned_orc2.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_partitioned_orc2.q @@ -1,6 +1,8 @@ -- MV data is stored by partitioned iceberg with partition spec --! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/ --! qt:replace:/(\s+current-snapshot-id\s+)\d+(\s*)/$1#SnapshotId#/ +-- Mask the totalSize value as it can change at file format library update +--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/ -- SORT_QUERY_RESULTS drop materialized view if exists mat1; diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out index 5666d10d68da..8cda12100b4c 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out @@ -64,16 +64,17 @@ Table Parameters: current-snapshot-id #SnapshotId# engine.hive.enabled true format-version 1 - iceberg.orc.files.only false + iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 2 numRows 2 previous_metadata_location hdfs://### HDFS PATH ### storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG - totalSize 1282 + totalSize #Masked# #### A masked pattern was here #### uuid #Masked# + write.format.default orc # Storage Information SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe @@ -143,17 +144,18 @@ Table Parameters: current-snapshot-id #SnapshotId# engine.hive.enabled true format-version 2 - iceberg.orc.files.only false + iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 2 numRows 2 previous_metadata_location hdfs://### HDFS PATH ### storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG - totalSize 1282 + totalSize #Masked# #### A masked pattern was here #### uuid #Masked# write.delete.mode merge-on-read + write.format.default orc write.merge.mode merge-on-read write.update.mode merge-on-read diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out index 169c00375ad6..6a31b96ef77f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out @@ -65,16 +65,17 @@ Table Parameters: current-snapshot-id #SnapshotId# engine.hive.enabled true format-version 1 - iceberg.orc.files.only false + iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 2 numRows 2 previous_metadata_location hdfs://### HDFS PATH ### storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG - totalSize 1282 + totalSize #Masked# #### A masked pattern was here #### uuid #Masked# + write.format.default orc # Storage Information SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe @@ -145,17 +146,18 @@ Table Parameters: current-snapshot-id #SnapshotId# engine.hive.enabled true format-version 2 - iceberg.orc.files.only false + iceberg.orc.files.only true metadata_location hdfs://### HDFS PATH ### numFiles 2 numRows 2 previous_metadata_location hdfs://### HDFS PATH ### storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG - totalSize 1282 + totalSize #Masked# #### A masked pattern was here #### uuid #Masked# write.delete.mode merge-on-read + write.format.default orc write.merge.mode merge-on-read write.update.mode merge-on-read diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateMaterializedViewDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateMaterializedViewDesc.java index 631a8868f10f..452e861cefed 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateMaterializedViewDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateMaterializedViewDesc.java @@ -349,6 +349,12 @@ public Table toTable(HiveConf conf) throws HiveException { org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE, getStorageHandler()); } + if (getSerdeProps() != null) { + for (Map.Entry entry : getSerdeProps().entrySet()) { + tbl.setSerdeParam(entry.getKey(), entry.getValue()); + } + } + HiveStorageHandler storageHandler = tbl.getStorageHandler(); setColumnsAndStorePartitionTransformSpecOfTable(getSchema(), getPartCols(), conf, tbl);