From 44a48a11996bc7b33fefc8fa91166a870c0f99c9 Mon Sep 17 00:00:00 2001 From: Ramit Gupta Date: Wed, 8 Apr 2026 23:50:56 +0530 Subject: [PATCH 01/10] HIVE-29413: Avoid code duplication by updating getPartCols method for iceberg tables --- .../column/show/ShowColumnsOperation.java | 5 +- .../table/info/desc/DescTableOperation.java | 2 +- .../formatter/TextDescTableFormatter.java | 4 +- .../ddl/table/partition/PartitionUtils.java | 1 - .../hive/ql/metadata/DummyPartition.java | 4 +- .../hadoop/hive/ql/metadata/Partition.java | 3 +- .../apache/hadoop/hive/ql/metadata/Table.java | 62 ++++++++++++++----- .../ql/optimizer/ColumnPrunerProcFactory.java | 3 +- .../ql/parse/AcidExportSemanticAnalyzer.java | 3 +- .../parse/ColumnStatsAutoGatherContext.java | 3 +- .../ql/parse/ColumnStatsSemanticAnalyzer.java | 3 +- .../hive/ql/parse/ImportSemanticAnalyzer.java | 2 - .../hive/ql/parse/MergeSemanticAnalyzer.java | 3 +- .../hadoop/hive/ql/parse/ParseUtils.java | 3 +- .../rewrite/CopyOnWriteMergeRewriter.java | 2 +- .../hive/ql/parse/rewrite/MergeRewriter.java | 2 +- .../ql/parse/rewrite/SplitMergeRewriter.java | 2 +- 17 files changed, 65 insertions(+), 42 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java index 289479b7ee79..6a2abd4ddfc2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java @@ -66,10 +66,7 @@ private List getColumnsByPattern() throws HiveException { private List getCols() throws HiveException { Table table = context.getDb().getTable(desc.getTableName()); - List allColumns = new ArrayList<>(); - allColumns.addAll(table.getCols()); - allColumns.addAll(table.getPartCols()); - return allColumns; + return new ArrayList<>(table.getAllCols()); } private Matcher getMatcher() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java index 9086ad90c677..b662008e95ed 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java @@ -127,7 +127,7 @@ private Deserializer getDeserializer(Table table) throws SQLException { private void getColumnsNoColumnPath(Table table, Partition partition, List cols) throws HiveException { cols.addAll(partition == null || table.getTableType() == TableType.VIRTUAL_VIEW ? table.getCols() : partition.getCols()); - if (!desc.isFormatted()) { + if (!desc.isFormatted() && !table.hasNonNativePartitionSupport()) { cols.addAll(table.getPartCols()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java index 75f39291cd07..c128948cd31b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java @@ -174,9 +174,7 @@ private void addPartitionData(DataOutputStream out, HiveConf conf, String column List partitionColumns = null; // TODO (HIVE-29413): Refactor to a generic getPartCols() implementation if (table.isPartitioned()) { - partitionColumns = table.hasNonNativePartitionSupport() ? - table.getStorageHandler().getPartitionKeys(table) : - table.getPartCols(); + partitionColumns = table.getPartCols(); } if (CollectionUtils.isNotEmpty(partitionColumns) && conf.getBoolVar(ConfVars.HIVE_DISPLAY_PARTITION_COLUMNS_SEPARATELY)) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java index db7a5dfcd3d0..5882e4616506 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java @@ -23,7 +23,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.Map.Entry; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java index c188eb09fdcf..45087c901e3f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java @@ -91,9 +91,7 @@ public List getValues() { values = new ArrayList<>(); // TODO (HIVE-29413): Refactor to a generic getPartCols() implementation - for (FieldSchema fs : table.hasNonNativePartitionSupport() - ? table.getStorageHandler().getPartitionKeys(table) - : table.getPartCols()) { + for (FieldSchema fs : table.getPartCols()) { String val = partSpec.get(fs.getName()); values.add(val); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java index 736e6e8c9f1a..330f37d1ef1c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java @@ -173,7 +173,8 @@ protected void initialize(Table table, // set default if location is not set and this is a physical // table partition (not a view partition) if (table.getDataLocation() != null) { - Path partPath = new Path(table.getDataLocation(), Warehouse.makePartName(table.getPartCols(), tPartition.getValues())); + Path partPath = new Path(table.getDataLocation(), + Warehouse.makePartName(table.getPartCols(), tPartition.getValues())); tPartition.getSd().setLocation(partPath.toString()); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index f857e7d505f1..7d0c00d61d45 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -57,6 +57,7 @@ import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.HiveMetaHook; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; @@ -118,6 +119,8 @@ public class Table implements Serializable { private transient StorageHandlerInfo storageHandlerInfo; private transient MaterializedViewMetadata materializedViewMetadata; + private List cachedPartCols; + private TableSpec tableSpec; private boolean materializedTable; @@ -193,6 +196,9 @@ public Table makeCopy() { newTab.setMetaTable(this.getMetaTable()); newTab.setSnapshotRef(this.getSnapshotRef()); + if (this.cachedPartCols != null) { + newTab.cachedPartCols = new ArrayList<>(this.cachedPartCols); + } return newTab; } @@ -214,6 +220,7 @@ public org.apache.hadoop.hive.metastore.api.Table getTTable() { */ public void setTTable(org.apache.hadoop.hive.metastore.api.Table tTable) { this.tTable = tTable; + clearCachedPartCols(); } /** @@ -248,7 +255,7 @@ public void setTTable(org.apache.hadoop.hive.metastore.api.Table tTable) { org.apache.hadoop.hive.metastore.api.Table t = new org.apache.hadoop.hive.metastore.api.Table(); { t.setSd(sd); - t.setPartitionKeys(new ArrayList()); + t.setPartitionKeys(null); t.setParameters(new HashMap()); t.setTableType(TableType.MANAGED_TABLE.toString()); t.setDbName(databaseName); @@ -595,14 +602,35 @@ public boolean equals(Object obj) { } public List getPartCols() { - List partKeys = tTable.getPartitionKeys(); - if (partKeys == null) { - partKeys = new ArrayList<>(); - tTable.setPartitionKeys(partKeys); + if (cachedPartCols != null) { + return cachedPartCols; + } + List partKeys; + if (isTableTypeSet() && hasNonNativePartitionSupport()) { + partKeys = getStorageHandler().getPartitionKeys(this); + } else { + partKeys = tTable.getPartitionKeys(); + if (partKeys == null) { + partKeys = new ArrayList<>(); + tTable.setPartitionKeys(partKeys); + } } + cachedPartCols = partKeys; return partKeys; } + private void clearCachedPartCols() { + cachedPartCols = null; + } + + private boolean isTableTypeSet() { + if (tTable.getParameters() == null) { + return false; + } + String tableType = tTable.getParameters().get(HiveMetaHook.TABLE_TYPE); + return tableType != null; + } + public FieldSchema getPartColByName(String colName) { return getPartCols().stream() .filter(key -> key.getName().toLowerCase().equals(colName)) @@ -610,9 +638,7 @@ public FieldSchema getPartColByName(String colName) { } public List getPartColNames() { - List partCols = hasNonNativePartitionSupport() ? - getStorageHandler().getPartitionKeys(this) : getPartCols(); - return partCols.stream().map(FieldSchema::getName) + return getPartCols().stream().map(FieldSchema::getName) .collect(Collectors.toList()); } @@ -761,14 +787,22 @@ private List getColsInternal(boolean forMs) { * @return List<FieldSchema> */ public List getAllCols() { - ArrayList f_list = new ArrayList(); - f_list.addAll(getCols()); - f_list.addAll(getPartCols()); - return f_list; + List allCols = new ArrayList<>(getCols()); + Set colNames = new HashSet<>(); + for (FieldSchema col : allCols) { + colNames.add(col.getName()); + } + for (FieldSchema col : getPartCols()) { + if (!colNames.contains(col.getName())) { + allCols.add(col); + } + } + return allCols; } public void setPartCols(List partCols) { tTable.setPartitionKeys(partCols); + clearCachedPartCols(); } public String getCatName() { @@ -812,7 +846,7 @@ public void setOutputFormatClass(String name) throws HiveException { } public boolean isPartitioned() { - return hasNonNativePartitionSupport() ? getStorageHandler().isPartitioned(this) : + return hasNonNativePartitionSupport() ? getStorageHandler().isPartitioned(this) : CollectionUtils.isNotEmpty(getPartCols()); } @@ -1153,7 +1187,7 @@ public static void validateColumns(List columns, List } colNames.add(colName); } - if (partCols != null) { + if (partCols != null && !icebergTable) { // there is no overlap between columns and partitioning columns for (FieldSchema partCol: partCols) { String colName = normalize(partCol.getName()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java index 3d3e4ce7663f..0e914843e2e1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java @@ -807,8 +807,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, for (FieldNode col : cols) { int index = originalOutputColumnNames.indexOf(col.getFieldName()); Table tab = cppCtx.getParseContext().getViewProjectToTableSchema().get(op); - List fullFieldList = new ArrayList(tab.getCols()); - fullFieldList.addAll(tab.getPartCols()); + List fullFieldList = new ArrayList<>(tab.getAllCols()); cppCtx.getParseContext().getColumnAccessInfo() .add(tab.getCompleteName(), fullFieldList.get(index).getName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java index 06912a1b3226..05f3b85f271f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java @@ -175,7 +175,8 @@ private void analyzeAcidExport(ASTNode ast, Table exportTable, ASTNode tokRefOrN //now generate insert statement //insert into newTableName select * from ts StringBuilder rewrittenQueryStr = generateExportQuery( - newTable.getPartCols(), tokRefOrNameExportTable, (ASTNode) tokRefOrNameExportTable.parent, newTableName); + newTable.getPartCols(), + tokRefOrNameExportTable, (ASTNode) tokRefOrNameExportTable.parent, newTableName); ReparseResult rr = ParseUtils.parseRewrittenQuery(ctx, rewrittenQueryStr); Context rewrittenCtx = rr.rewrittenCtx; rewrittenCtx.setIsUpdateDeleteMerge(false); //it's set in parseRewrittenQuery() diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java index 9109f9cb6086..d8c7d9527ec3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java @@ -83,7 +83,8 @@ public ColumnStatsAutoGatherContext(SemanticAnalyzer sa, HiveConf conf, this.isInsertInto = isInsertInto; this.origCtx = ctx; columns = tbl.getCols(); - partitionColumns = tbl.getPartCols(); + // current behaviour intact until we have getCols() giving only non-partition columns for non native tables as well + partitionColumns = tbl.hasNonNativePartitionSupport() ? new ArrayList<>() : tbl.getPartCols(); } public List getLoadFileWork() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java index e23e54aa5230..9bf6269334e7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java @@ -207,8 +207,7 @@ private static CharSequence genPartitionClause(Table tbl, List pa private static String getColTypeOf(Table tbl, String partKey) { - for (FieldSchema fs : tbl.hasNonNativePartitionSupport() ? - tbl.getStorageHandler().getPartitionKeys(tbl) : tbl.getPartitionKeys()) { + for (FieldSchema fs : tbl.getPartCols()) { if (partKey.equalsIgnoreCase(fs.getName())) { return fs.getType().toLowerCase(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java index 4d4956fbec13..dcf197a2c201 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java @@ -37,7 +37,6 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.ReplChangeManager; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; -import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.ql.QueryState; @@ -89,7 +88,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.TreeMap; /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java index 882840ffef5a..ac0a7e8f12be 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java @@ -29,7 +29,6 @@ import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.rewrite.MergeStatement; import org.apache.hadoop.hive.ql.parse.rewrite.RewriterFactory; -import org.apache.hadoop.hive.ql.plan.HiveOperation; import java.util.ArrayList; import java.util.HashMap; @@ -230,7 +229,7 @@ private MergeStatement.UpdateClause handleUpdate(ASTNode whenMatchedUpdateClause String deleteExtraPredicate) throws SemanticException { assert whenMatchedUpdateClause.getType() == HiveParser.TOK_MATCHED; assert getWhenClauseOperation(whenMatchedUpdateClause).getType() == HiveParser.TOK_UPDATE; - Map newValuesMap = new HashMap<>(targetTable.getCols().size() + targetTable.getPartCols().size()); + Map newValuesMap = new HashMap<>(targetTable.getAllCols().size()); ASTNode setClause = (ASTNode)getWhenClauseOperation(whenMatchedUpdateClause).getChild(0); //columns being updated -> update expressions; "setRCols" (last param) is null because we use actual expressions //before re-parsing, i.e. they are known to SemanticAnalyzer logic diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java index 9964b9369065..5fa22d9a2f82 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java @@ -581,8 +581,7 @@ public static Map> getFullPartitionSpecs( CommonTree ast, Table table, Configuration conf, boolean canGroupExprs) throws SemanticException { String defaultPartitionName = HiveConf.getVar(conf, HiveConf.ConfVars.DEFAULT_PARTITION_NAME); Map colTypes = new HashMap<>(); - List partitionKeys = table.hasNonNativePartitionSupport() ? - table.getStorageHandler().getPartitionKeys(table) : table.getPartitionKeys(); + List partitionKeys = table.getPartCols(); for (FieldSchema fs : partitionKeys) { colTypes.put(fs.getName().toLowerCase(), fs.getType()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteMergeRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteMergeRewriter.java index b72f2496d938..b7335473da85 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteMergeRewriter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteMergeRewriter.java @@ -202,7 +202,7 @@ public void appendWhenMatchedUpdateClause(MergeStatement.UpdateClause updateClau sqlGenerator.append(hintStr); hintStr = null; } - List values = new ArrayList<>(targetTable.getCols().size() + targetTable.getPartCols().size()); + List values = new ArrayList<>(targetTable.getAllCols().size()); values.addAll(sqlGenerator.getDeleteValues(Context.Operation.MERGE)); addValues(targetTable, targetAlias, updateClause.getNewValuesMap(), values); addValuesForRowLineageForCopyOnMerge(isRowLineageSupported, values, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java index 3ec2e580f046..c436e85a4eb6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java @@ -224,7 +224,7 @@ public void appendWhenMatchedUpdateClause(MergeStatement.UpdateClause updateClau sqlGenerator.append(" -- update clause").append("\n"); List valuesAndAcidSortKeys = new ArrayList<>( - targetTable.getCols().size() + targetTable.getPartCols().size() + 1); + targetTable.getAllCols().size() + 1); valuesAndAcidSortKeys.addAll(sqlGenerator.getSortKeys(Operation.MERGE)); addValues(targetTable, targetAlias, updateClause.getNewValuesMap(), valuesAndAcidSortKeys); sqlGenerator.appendInsertBranch(hintStr, valuesAndAcidSortKeys); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitMergeRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitMergeRewriter.java index 84fcf186f6b7..06edaca90f0f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitMergeRewriter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitMergeRewriter.java @@ -58,7 +58,7 @@ public void appendWhenMatchedUpdateClause(MergeStatement.UpdateClause updateClau String onClauseAsString = mergeStatement.getOnClauseAsText(); sqlGenerator.append(" -- update clause (insert part)\n"); - List values = new ArrayList<>(targetTable.getCols().size() + targetTable.getPartCols().size()); + List values = new ArrayList<>(targetTable.getAllCols().size()); addValues(targetTable, targetAlias, updateClause.getNewValuesMap(), values); addRowLineageColumnsForWhenMatchedUpdateClause(isRowLineageSupported, values, targetAlias, conf); sqlGenerator.appendInsertBranch(hintStr, values); From 827e0b270d270790a8588e4bfddb7ec419816b52 Mon Sep 17 00:00:00 2001 From: Ramit Gupta Date: Thu, 9 Apr 2026 15:33:03 +0530 Subject: [PATCH 02/10] commit-2 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java | 2 +- .../java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index 7d0c00d61d45..3879d8d78040 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -255,7 +255,7 @@ public void setTTable(org.apache.hadoop.hive.metastore.api.Table tTable) { org.apache.hadoop.hive.metastore.api.Table t = new org.apache.hadoop.hive.metastore.api.Table(); { t.setSd(sd); - t.setPartitionKeys(null); + t.setPartitionKeys(new ArrayList()); t.setParameters(new HashMap()); t.setTableType(TableType.MANAGED_TABLE.toString()); t.setDbName(databaseName); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 00cc988eb0a8..c69a25f76e12 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -2998,6 +2998,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc final NotNullConstraint nnc = tabMetaData.getNotNullConstraint(); final PrimaryKeyInfo pkc = tabMetaData.getPrimaryKeyInfo(); + Set alreadyAdded = new HashSet<>(); for (StructField structField : fields) { colName = structField.getFieldName(); colInfo = new ColumnInfo( @@ -3006,6 +3007,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc isNullable(colName, nnc, pkc), tableAlias, false); colInfo.setSkewedCol(isSkewedCol(tableAlias, qb, colName)); rr.put(tableAlias, colName, colInfo); + alreadyAdded.add(colName); cInfoLst.add(colInfo); } // TODO: Fix this @@ -3015,6 +3017,9 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc // 3.2 Add column info corresponding to partition columns for (FieldSchema part_col : tabMetaData.getPartCols()) { colName = part_col.getName(); + if (alreadyAdded.contains(colName)) { + continue; + } colInfo = new ColumnInfo(colName, TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), isNullable(colName, nnc, pkc), tableAlias, true); From 86105a9c3a768ff2b0083ec4ec07032649fcccdf Mon Sep 17 00:00:00 2001 From: Ramit Gupta Date: Fri, 10 Apr 2026 01:14:09 +0530 Subject: [PATCH 03/10] corrected bucket-map-join test --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java | 6 +++--- .../org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index 3879d8d78040..89934346f047 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -632,9 +632,9 @@ private boolean isTableTypeSet() { } public FieldSchema getPartColByName(String colName) { - return getPartCols().stream() - .filter(key -> key.getName().toLowerCase().equals(colName)) - .findFirst().orElse(null); + return hasNonNativePartitionSupport() ? null : getPartCols().stream() + .filter(key -> key.getName().toLowerCase().equals(colName)) + .findFirst().orElse(null); } public List getPartColNames() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index f8ec90287202..4dae0653e0ee 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -12003,6 +12003,8 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { // Determine row schema for TSOP. // Include column names from SerDe, the partition and virtual columns. rwsch = new RowResolver(); + Set partCols = tab.hasNonNativePartitionSupport() ? + Sets.newHashSet(tab.getPartColNames()) : Collections.emptySet(); try { // Including parameters passed in the query if (properties != null) { @@ -12020,8 +12022,6 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { deserializer.handleJobLevelConfiguration(conf); List fields = rowObjectInspector .getAllStructFieldRefs(); - Set partCols = tab.hasNonNativePartitionSupport() ? - Sets.newHashSet(tab.getPartColNames()) : Collections.emptySet(); for (int i = 0; i < fields.size(); i++) { /** * if the column is a skewed column, use ColumnInfo accordingly @@ -12041,6 +12041,9 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { // Hack!! - refactor once the metadata APIs with types are ready // Finally add the partitioning columns for (FieldSchema part_col : tab.getPartCols()) { + if(partCols.contains(part_col.getName())){ + break; + } LOG.trace("Adding partition col: " + part_col); rwsch.put(alias, part_col.getName(), new ColumnInfo(part_col.getName(), TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), alias, true)); From f29e91e1d1ac763eddcd0a53d812d8003f88bb04 Mon Sep 17 00:00:00 2001 From: Ramit Gupta Date: Fri, 10 Apr 2026 11:34:14 +0530 Subject: [PATCH 04/10] corrected update statements --- .../hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java | 2 +- .../hadoop/hive/ql/parse/rewrite/MergeRewriter.java | 6 ++++-- .../hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java | 2 +- .../ql/parse/rewrite/sql/MultiInsertSqlGenerator.java | 8 +++++++- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java index 101f6b1fc3d8..8e52f63c6611 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java @@ -120,7 +120,7 @@ protected void checkValidSetClauseTarget(ASTNode colName, Table targetTable) thr // Make sure this isn't one of the partitioning columns, that's not supported. for (FieldSchema fschema : targetTable.getPartCols()) { - if (fschema.getName().equalsIgnoreCase(columnName)) { + if (fschema.getName().equalsIgnoreCase(columnName) && !targetTable.hasNonNativePartitionSupport()) { throw new SemanticException(ErrorMsg.UPDATE_CANNOT_UPDATE_PART_VALUE.getMsg()); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java index c436e85a4eb6..c9ba50d110f8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java @@ -250,8 +250,10 @@ protected void addValues(Table targetTable, String targetAlias, Map values.add( - formatter.apply(fieldSchema.getName()))); + if (!targetTable.hasNonNativePartitionSupport()) { + targetTable.getPartCols().forEach(fieldSchema -> values.add( + formatter.apply(fieldSchema.getName()))); + } } protected String getRhsExpValue(String newValue, String alias) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java index d14ddc7eb485..838d1d8a09a6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java @@ -98,7 +98,7 @@ public ParseUtils.ReparseResult rewrite(Context context, UpdateStatement updateB insertValues.add(sqlGenerator.qualify(identifier)); } - if (updateBlock.getTargetTable().getPartCols() != null) { + if (!updateBlock.getTargetTable().hasNonNativePartitionSupport()) { updateBlock.getTargetTable().getPartCols().forEach( fieldSchema -> insertValues.add(sqlGenerator.qualify(HiveUtils.unparseIdentifier(fieldSchema.getName(), conf)))); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java index 7587daf13055..6576dd28d2ec 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java @@ -111,6 +111,9 @@ public void appendPartitionColsOfTarget() { */ public void appendPartitionCols(Table table) { // If the table is partitioned we have to put the partition() clause in + if (table.hasNonNativePartitionSupport()) { + return; + } List partCols = table.getPartCols(); if (partCols == null || partCols.isEmpty()) { return; @@ -148,7 +151,10 @@ public void removeLastChar() { } public void appendPartColsOfTargetTableWithComma(String alias) { - if (targetTable.getPartCols() == null || targetTable.getPartCols().isEmpty()) { + if (targetTable.hasNonNativePartitionSupport()) { + return; + } + if (targetTable.getPartCols().isEmpty()) { return; } queryStr.append(','); From 0b3311bd799fd458e53040394dd4c06c5ad7c99c Mon Sep 17 00:00:00 2001 From: Ramit Gupta Date: Sat, 11 Apr 2026 20:13:07 +0530 Subject: [PATCH 05/10] corrected load, partition evolution tests --- .../org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java | 3 +++ .../java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java | 4 ++-- .../apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java | 5 +++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java index 17a964a44583..ae7696c3e536 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java @@ -75,6 +75,9 @@ public static boolean isSchemaEvolutionEnabled(Table table, Configuration conf) } public static boolean isFullPartitionSpec(Table table, Map partitionSpec) { + if (table.hasNonNativePartitionSupport()) { + return true; + } for (FieldSchema partitionCol : table.getPartCols()) { if (partitionSpec.get(partitionCol.getName()) == null) { return false; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index c69a25f76e12..2176dd133d16 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -3017,8 +3017,8 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc // 3.2 Add column info corresponding to partition columns for (FieldSchema part_col : tabMetaData.getPartCols()) { colName = part_col.getName(); - if (alreadyAdded.contains(colName)) { - continue; + if (tabMetaData.hasNonNativePartitionSupport()) { + break; } colInfo = new ColumnInfo(colName, TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index eb4a73f1e5e9..6aa5b08fd5f4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -23,6 +23,7 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -511,7 +512,7 @@ private void reparseAndSuperAnalyze(Table table, URI fromURI) throws SemanticExc // Partition spec was already validated by caller when create TableSpec object. // So, need not validate inpPartSpec here. - List parts = table.getPartCols(); + List parts = table.hasNonNativePartitionSupport() ? Collections.emptyList() : table.getPartCols(); if (tableTree.getChildCount() >= 2) { ASTNode partSpecNode = (ASTNode) tableTree.getChild(1); inpPartSpec = new HashMap<>(partSpecNode.getChildCount()); @@ -561,7 +562,7 @@ private void reparseAndSuperAnalyze(Table table, URI fromURI) throws SemanticExc } rewrittenQueryStr.append(getFullTableNameForSQL((ASTNode)(tableTree.getChild(0)))); - addPartitionColsToInsert(table.getPartCols(), inpPartSpec, rewrittenQueryStr); + addPartitionColsToInsert(parts, inpPartSpec, rewrittenQueryStr); rewrittenQueryStr.append(" select * from "); rewrittenQueryStr.append(tempTblName); From bcd763af894803c1b7c354ee90431493a26d11f6 Mon Sep 17 00:00:00 2001 From: Ramit Gupta Date: Sun, 12 Apr 2026 01:54:59 +0530 Subject: [PATCH 06/10] refractored # Conflicts: # ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java --- .../mapreduce/TestHCatMultiOutputFormat.java | 2 +- .../hive/ql/ddl/table/AlterTableUtils.java | 3 -- .../create/like/CreateTableLikeOperation.java | 2 +- .../table/info/desc/DescTableOperation.java | 2 +- .../formatter/TextDescTableFormatter.java | 2 +- .../JsonShowTableStatusFormatter.java | 2 +- .../TextShowTableStatusFormatter.java | 2 +- .../ddl/table/partition/PartitionUtils.java | 2 +- .../AlterTableExchangePartitionAnalyzer.java | 4 +- .../partition/show/ShowPartitionAnalyzer.java | 6 +-- .../archive/AlterTableArchiveOperation.java | 2 +- .../archive/AlterTableArchiveUtils.java | 2 +- .../archive/AlterTableUnarchiveOperation.java | 2 +- .../create/AbstractCreateViewAnalyzer.java | 2 +- .../hadoop/hive/ql/exec/ArchiveUtils.java | 6 +-- .../hadoop/hive/ql/exec/DDLPlanUtils.java | 5 +-- .../apache/hadoop/hive/ql/exec/MoveTask.java | 2 +- .../apache/hadoop/hive/ql/exec/Utilities.java | 2 +- .../hive/ql/exec/repl/ReplLoadTask.java | 2 +- .../hive/ql/metadata/DummyPartition.java | 2 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 16 +++---- .../HiveMaterializedViewsRegistry.java | 3 +- .../hadoop/hive/ql/metadata/Partition.java | 8 ++-- .../apache/hadoop/hive/ql/metadata/Table.java | 44 +++++++++---------- .../hive/ql/optimizer/GenMapRedUtils.java | 2 +- .../ql/optimizer/ppr/PartitionPruner.java | 2 +- .../ql/parse/AcidExportSemanticAnalyzer.java | 2 +- .../hive/ql/parse/BaseSemanticAnalyzer.java | 2 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 5 --- .../parse/ColumnStatsAutoGatherContext.java | 2 +- .../ql/parse/ColumnStatsSemanticAnalyzer.java | 4 +- .../hive/ql/parse/ImportSemanticAnalyzer.java | 2 +- .../hive/ql/parse/LoadSemanticAnalyzer.java | 3 +- .../hive/ql/parse/MergeSemanticAnalyzer.java | 2 +- .../hadoop/hive/ql/parse/ParseUtils.java | 4 +- .../ql/parse/RewriteSemanticAnalyzer.java | 2 +- .../hive/ql/parse/SemanticAnalyzer.java | 11 ++--- .../hive/ql/parse/rewrite/MergeRewriter.java | 8 ++-- .../ql/parse/rewrite/SplitUpdateRewriter.java | 2 +- .../rewrite/sql/MultiInsertSqlGenerator.java | 10 +---- .../NativeAcidMultiInsertSqlGenerator.java | 6 +-- .../hive/ql/stats/ColStatsProcessor.java | 2 +- 42 files changed, 85 insertions(+), 111 deletions(-) diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java index d87158b23fae..05e5495d2656 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java @@ -374,7 +374,7 @@ private List getTableData(String table, String database) throws Exceptio Hive hive = Hive.get(conf); org.apache.hadoop.hive.ql.metadata.Table tbl = hive.getTable(database, table); FetchWork work; - if (!tbl.getPartCols().isEmpty()) { + if (!tbl.getEffectivePartCols().isEmpty()) { List partitions = hive.getPartitions(tbl); List partDesc = new ArrayList(); List partLocs = new ArrayList(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java index ae7696c3e536..17a964a44583 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableUtils.java @@ -75,9 +75,6 @@ public static boolean isSchemaEvolutionEnabled(Table table, Configuration conf) } public static boolean isFullPartitionSpec(Table table, Map partitionSpec) { - if (table.hasNonNativePartitionSupport()) { - return true; - } for (FieldSchema partitionCol : table.getPartCols()) { if (partitionSpec.get(partitionCol.getName()) == null) { return false; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java index 770724b90abf..e10d4bdb00ce 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java @@ -100,7 +100,7 @@ private Table createViewLikeTable(Table oldTable) throws HiveException { setUserSpecifiedLocation(table); table.setFields(oldTable.getCols()); - table.setPartCols(oldTable.getPartCols()); + table.setPartCols(oldTable.getEffectivePartCols()); if (desc.getDefaultSerdeProps() != null) { for (Map.Entry e : desc.getDefaultSerdeProps().entrySet()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java index b662008e95ed..9086ad90c677 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java @@ -127,7 +127,7 @@ private Deserializer getDeserializer(Table table) throws SQLException { private void getColumnsNoColumnPath(Table table, Partition partition, List cols) throws HiveException { cols.addAll(partition == null || table.getTableType() == TableType.VIRTUAL_VIEW ? table.getCols() : partition.getCols()); - if (!desc.isFormatted() && !table.hasNonNativePartitionSupport()) { + if (!desc.isFormatted()) { cols.addAll(table.getPartCols()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java index c128948cd31b..2ecce54e088d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java @@ -174,7 +174,7 @@ private void addPartitionData(DataOutputStream out, HiveConf conf, String column List partitionColumns = null; // TODO (HIVE-29413): Refactor to a generic getPartCols() implementation if (table.isPartitioned()) { - partitionColumns = table.getPartCols(); + partitionColumns = table.getEffectivePartCols(); } if (CollectionUtils.isNotEmpty(partitionColumns) && conf.getBoolVar(ConfVars.HIVE_DISPLAY_PARTITION_COLUMNS_SEPARATELY)) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/JsonShowTableStatusFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/JsonShowTableStatusFormatter.java index 073db26e756c..405417916990 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/JsonShowTableStatusFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/JsonShowTableStatusFormatter.java @@ -69,7 +69,7 @@ private Map makeOneTableStatus(Table table, Hive db, HiveConf co builder.put("partitioned", table.isPartitioned()); if (table.isPartitioned()) { - builder.put("partitionColumns", JsonDescTableFormatter.createColumnsInfo(table.getPartCols(), + builder.put("partitionColumns", JsonDescTableFormatter.createColumnsInfo(table.getEffectivePartCols(), Collections.emptyList())); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/TextShowTableStatusFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/TextShowTableStatusFormatter.java index 552dc310465b..33205bebcbea 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/TextShowTableStatusFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/TextShowTableStatusFormatter.java @@ -73,7 +73,7 @@ private void writeStorageInfo(DataOutputStream out, Partition partition, Table t private void writeColumnsInfo(DataOutputStream out, Table table) throws IOException, UnsupportedEncodingException { String columns = MetaStoreUtils.getDDLFromFieldSchema("columns", table.getCols()); String partitionColumns = table.isPartitioned() ? - MetaStoreUtils.getDDLFromFieldSchema("partition_columns", table.getPartCols()) : ""; + MetaStoreUtils.getDDLFromFieldSchema("partition_columns", table.getEffectivePartCols()) : ""; out.write(Utilities.newLineCode); out.write(("columns:" + columns).getBytes(StandardCharsets.UTF_8)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java index 5882e4616506..0011b25df358 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java @@ -149,7 +149,7 @@ public static List getPartitionsWithSpecs(Hive db, Table table, GetPa } private static String tablePartitionColNames(Table table) { - List partCols = table.getPartCols(); + List partCols = table.getEffectivePartCols(); return String.join("/", partCols.toString()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionAnalyzer.java index 6485627c7e6e..91c8da2f74ba 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionAnalyzer.java @@ -84,7 +84,7 @@ protected void analyzeCommand(TableName tableName, Map partition if (AcidUtils.isTransactionalTable(sourceTable) || AcidUtils.isTransactionalTable(destTable)) { throw new SemanticException(ErrorMsg.EXCHANGE_PARTITION_NOT_ALLOWED_WITH_TRANSACTIONAL_TABLES.getMsg()); } - List sourceProjectFilters = MetaStoreUtils.getPvals(sourceTable.getPartCols(), partitionSpecs); + List sourceProjectFilters = MetaStoreUtils.getPvals(sourceTable.getEffectivePartCols(), partitionSpecs); // check if source partition exists GetPartitionsFilterSpec sourcePartitionsFilterSpec = new GetPartitionsFilterSpec(); @@ -106,7 +106,7 @@ protected void analyzeCommand(TableName tableName, Map partition throw new SemanticException(ErrorMsg.PARTITION_VALUE_NOT_CONTINUOUS.getMsg(partitionSpecs.toString())); } - List destProjectFilters = MetaStoreUtils.getPvals(destTable.getPartCols(), partitionSpecs); + List destProjectFilters = MetaStoreUtils.getPvals(destTable.getEffectivePartCols(), partitionSpecs); // check if dest partition exists GetPartitionsFilterSpec getDestPartitionsFilterSpec = new GetPartitionsFilterSpec(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java index c0bffcebdb23..4a7aea0490f3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java @@ -102,7 +102,7 @@ ExprNodeDesc getShowPartitionsFilter(Table table, ASTNode command) throws Semant if (astChild.getType() == HiveParser.TOK_WHERE) { RowResolver rwsch = new RowResolver(); Map colTypes = new HashMap(); - for (FieldSchema fs : table.getPartCols()) { + for (FieldSchema fs : table.getEffectivePartCols()) { rwsch.put(table.getTableName(), fs.getName(), new ColumnInfo(fs.getName(), TypeInfoFactory.stringTypeInfo, null, true)); colTypes.put(fs.getName().toLowerCase(), fs.getType()); @@ -202,8 +202,8 @@ private String getShowPartitionsOrder(Table table, ASTNode command) throws Seman if (astChild.getType() == HiveParser.TOK_ORDERBY) { Map poses = new HashMap(); RowResolver rwsch = new RowResolver(); - for (int i = 0; i < table.getPartCols().size(); i++) { - FieldSchema fs = table.getPartCols().get(i); + for (int i = 0; i < table.getEffectivePartCols().size(); i++) { + FieldSchema fs = table.getEffectivePartCols().get(i); rwsch.put(table.getTableName(), fs.getName(), new ColumnInfo(fs.getName(), TypeInfoFactory.getPrimitiveTypeInfo(fs.getType()), null, true)); poses.put(fs.getName().toLowerCase(), i); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/archive/AlterTableArchiveOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/archive/AlterTableArchiveOperation.java index e218e590a24e..67f5b7946760 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/archive/AlterTableArchiveOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/archive/AlterTableArchiveOperation.java @@ -129,7 +129,7 @@ private Path getOriginalDir(Table table, PartSpecInfo partitionSpecInfo, List partitionColumns) throws SemanticException { - if (oldView.getPartCols().isEmpty() || oldView.getPartCols().equals(partitionColumns)) { + if (oldView.getEffectivePartCols().isEmpty() || oldView.getEffectivePartCols().equals(partitionColumns)) { return; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java index ebe8f2f52775..f25cadd40073 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java @@ -74,7 +74,7 @@ static public PartSpecInfo create(Table tbl, Map partSpec) // ARCHIVE PARTITION(hr='13') won't List prefixFields = new ArrayList(); List prefixValues = new ArrayList(); - List partCols = tbl.getPartCols(); + List partCols = tbl.getEffectivePartCols(); Iterator itrPsKeys = partSpec.keySet().iterator(); for (FieldSchema fs : partCols) { if (!itrPsKeys.hasNext()) { @@ -222,7 +222,7 @@ public static int getArchivingLevel(Partition p) throws HiveException { * @throws HiveException */ public static String getPartialName(Partition p, int level) throws HiveException { - List fields = p.getTable().getPartCols().subList(0, level); + List fields = p.getTable().getEffectivePartCols().subList(0, level); List values = p.getValues().subList(0, level); try { return Warehouse.makePartName(fields, values); @@ -273,7 +273,7 @@ public static String conflictingArchiveNameOrNull(Hive db, Table tbl, Map spec = new HashMap(partSpec); List reversedKeys = new ArrayList(); - for (FieldSchema fs : tbl.getPartCols()) { + for (FieldSchema fs : tbl.getEffectivePartCols()) { if (spec.containsKey(fs.getName())) { reversedKeys.add(fs.getName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java index a5bc66733f46..3fe753de5e08 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java @@ -38,7 +38,6 @@ import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; -import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.SkewedInfo; @@ -294,7 +293,7 @@ public String getPartitionActualName(Partition pt) { */ private Map getPartitionColumnToPrimitiveCategory(Partition pt) { Map resultMap = new HashMap<>(); - for (FieldSchema schema: pt.getTable().getPartCols()) { + for (FieldSchema schema: pt.getTable().getEffectivePartCols()) { resultMap.put( schema.getName(), ((PrimitiveTypeInfo) TypeInfoUtils.getTypeInfoFromTypeString(schema.getType())).getPrimitiveCategory() @@ -976,7 +975,7 @@ private String getComment(Table table) { } private String getPartitionsForView(Table table) { - List partitionKeys = table.getPartCols(); + List partitionKeys = table.getEffectivePartCols(); if (partitionKeys.isEmpty()) { return ""; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java index 3eca5531f127..a5ec3a023092 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java @@ -633,7 +633,7 @@ public void logMessage(LoadTableDesc tbd) { private DataContainer handleStaticParts(Hive db, Table table, LoadTableDesc tbd, TaskInformation ti) throws HiveException, IOException, InvalidOperationException { - List partVals = MetaStoreUtils.getPvals(table.getPartCols(), tbd.getPartitionSpec()); + List partVals = MetaStoreUtils.getPvals(table.getEffectivePartCols(), tbd.getPartitionSpec()); db.validatePartitionNameCharacters(partVals); if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("loadPartition called from " + tbd.getSourcePath() diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index a29e532b113b..9cd67fcb9619 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -4321,7 +4321,7 @@ public static void setPartitionColumnNames(Configuration conf, TableScanOperator if (metadata == null) { return; } - List partCols = metadata.getPartCols(); + List partCols = metadata.getEffectivePartCols(); if (partCols != null && !partCols.isEmpty()) { conf.set(serdeConstants.LIST_PARTITION_COLUMNS, MetaStoreUtils.getColumnNamesFromFieldSchema(partCols)); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java index 9287fd75e766..a7829d7b78bf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java @@ -548,7 +548,7 @@ public static Task createViewTask(MetaData metaData, String dbNameToLoadIn, H } CreateViewDesc desc = new CreateViewDesc(dbDotView, table.getCols(), null, table.getParameters(), - table.getPartColNames(), false, false, viewOriginalText, viewExpandedText, table.getPartCols()); + table.getPartColNames(), false, false, viewOriginalText, viewExpandedText, table.getEffectivePartCols()); desc.setReplicationSpec(metaData.getReplicationSpec()); desc.setOwnerName(table.getOwner()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java index 45087c901e3f..7af16770c059 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java @@ -91,7 +91,7 @@ public List getValues() { values = new ArrayList<>(); // TODO (HIVE-29413): Refactor to a generic getPartCols() implementation - for (FieldSchema fs : table.getPartCols()) { + for (FieldSchema fs : table.getEffectivePartCols()) { String val = partSpec.get(fs.getName()); values.add(val); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index cd33896807bc..0c583264816b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -869,7 +869,7 @@ public void createTable(String tableName, List columns, List par FieldSchema part = new FieldSchema(); part.setName(partCol); part.setType(STRING_TYPE_NAME); // default partition key - tbl.getPartCols().add(part); + tbl.getEffectivePartCols().add(part); } } tbl.setSerializationLib(LazySimpleSerDe.class.getName()); @@ -1246,8 +1246,8 @@ public void renamePartition(Table tbl, Map oldPartSpec, Partitio throws HiveException { try { Map newPartSpec = newPart.getSpec(); - if (oldPartSpec.keySet().size() != tbl.getPartCols().size() - || newPartSpec.keySet().size() != tbl.getPartCols().size()) { + if (oldPartSpec.keySet().size() != tbl.getEffectivePartCols().size() + || newPartSpec.keySet().size() != tbl.getEffectivePartCols().size()) { throw new HiveException("Unable to rename partition to the same name: number of partition cols don't match. "); } if (!oldPartSpec.keySet().equals(newPartSpec.keySet())){ @@ -1255,7 +1255,7 @@ public void renamePartition(Table tbl, Map oldPartSpec, Partitio } List pvals = new ArrayList(); - for (FieldSchema field : tbl.getPartCols()) { + for (FieldSchema field : tbl.getEffectivePartCols()) { String val = oldPartSpec.get(field.getName()); if (val == null || val.length() == 0) { throw new HiveException("get partition: Value for key " @@ -3832,7 +3832,7 @@ public Partition getPartition(Table tbl, Map partSpec, boolean forceCreate, String partPath, boolean inheritTableSpecs) throws HiveException { tbl.validatePartColumnNames(partSpec, true); List pvals = new ArrayList(); - for (FieldSchema field : tbl.getPartCols()) { + for (FieldSchema field : tbl.getEffectivePartCols()) { String val = partSpec.get(field.getName()); // enable dynamic partitioning if ((val == null && !HiveConf.getBoolVar(conf, HiveConf.ConfVars.DYNAMIC_PARTITIONING)) @@ -4221,7 +4221,7 @@ public List getPartitionNames(Table tbl, Map partSpec, s if (tbl.hasNonNativePartitionSupport()) { return tbl.getStorageHandler().getPartitionNames(tbl, partSpec); } - List pvals = MetaStoreUtils.getPvals(tbl.getPartCols(), partSpec); + List pvals = MetaStoreUtils.getPvals(tbl.getEffectivePartCols(), partSpec); return getPartitionNamesByPartitionVals(tbl, pvals, max); } @@ -4463,7 +4463,7 @@ private List getPartitionsWithAuth(Table tbl, Map par throw new HiveException(ErrorMsg.TABLE_NOT_PARTITIONED, tbl.getTableName()); } - List partialPvals = MetaStoreUtils.getPvals(tbl.getPartCols(), partialPartSpec); + List partialPvals = MetaStoreUtils.getPvals(tbl.getEffectivePartCols(), partialPartSpec); List partitions = null; try { @@ -4772,7 +4772,7 @@ static List convertFromPartSpec(Iterator iterator, Tab || partitionWithoutSD.getRelativePath().isEmpty()) { if (tbl.getDataLocation() != null) { Path partPath = new Path(tbl.getDataLocation(), - Warehouse.makePartName(tbl.getPartCols(), + Warehouse.makePartName(tbl.getEffectivePartCols(), partitionWithoutSD.getValues())); partitionLocation = partPath.toString(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java index 0cf02a95e392..b763c379bdc7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java @@ -20,7 +20,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -430,7 +429,7 @@ private static RelNode createMaterializedViewScan(HiveConf conf, Table viewTable // 1.2 Add column info corresponding to partition columns ArrayList partitionColumns = new ArrayList(); - for (FieldSchema part_col : viewTable.getPartCols()) { + for (FieldSchema part_col : viewTable.getEffectivePartCols()) { colName = part_col.getName(); colInfo = new ColumnInfo(colName, TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), null, true); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java index 330f37d1ef1c..d284e0231a03 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java @@ -125,7 +125,7 @@ public Partition(Table tbl, Map partSpec, Path location) throws public static org.apache.hadoop.hive.metastore.api.Partition createMetaPartitionObject( Table tbl, Map partSpec, Path location) throws HiveException { List pvals = new ArrayList(); - for (FieldSchema field : tbl.getPartCols()) { + for (FieldSchema field : tbl.getEffectivePartCols()) { String val = partSpec.get(field.getName()); if (val == null || val.isEmpty()) { throw new HiveException("partition spec is invalid; field " @@ -174,7 +174,7 @@ protected void initialize(Table table, // table partition (not a view partition) if (table.getDataLocation() != null) { Path partPath = new Path(table.getDataLocation(), - Warehouse.makePartName(table.getPartCols(), tPartition.getValues())); + Warehouse.makePartName(table.getEffectivePartCols(), tPartition.getValues())); tPartition.getSd().setLocation(partPath.toString()); } } @@ -201,7 +201,7 @@ protected void initialize(Table table, public String getName() { try { - return Warehouse.makePartName(table.getPartCols(), tPartition.getValues()); + return Warehouse.makePartName(table.getEffectivePartCols(), tPartition.getValues()); } catch (MetaException e) { throw new RuntimeException(e); } @@ -544,7 +544,7 @@ public void setLocation(String location) { public void setValues(Map partSpec) throws HiveException { List pvals = new ArrayList(); - for (FieldSchema field : table.getPartCols()) { + for (FieldSchema field : table.getEffectivePartCols()) { String val = partSpec.get(field.getName()); if (val == null) { throw new HiveException( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index 89934346f047..809bab85af12 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -602,21 +602,24 @@ public boolean equals(Object obj) { } public List getPartCols() { + List partKeys = tTable.getPartitionKeys(); + if (partKeys == null) { + partKeys = new ArrayList<>(); + tTable.setPartitionKeys(partKeys); + } + return partKeys; + } + + public List getEffectivePartCols() { if (cachedPartCols != null) { return cachedPartCols; } - List partKeys; if (isTableTypeSet() && hasNonNativePartitionSupport()) { - partKeys = getStorageHandler().getPartitionKeys(this); + cachedPartCols = getStorageHandler().getPartitionKeys(this); } else { - partKeys = tTable.getPartitionKeys(); - if (partKeys == null) { - partKeys = new ArrayList<>(); - tTable.setPartitionKeys(partKeys); - } + cachedPartCols = getPartCols(); } - cachedPartCols = partKeys; - return partKeys; + return cachedPartCols; } private void clearCachedPartCols() { @@ -632,13 +635,13 @@ private boolean isTableTypeSet() { } public FieldSchema getPartColByName(String colName) { - return hasNonNativePartitionSupport() ? null : getPartCols().stream() + return getPartCols().stream() .filter(key -> key.getName().toLowerCase().equals(colName)) .findFirst().orElse(null); } public List getPartColNames() { - return getPartCols().stream().map(FieldSchema::getName) + return getEffectivePartCols().stream().map(FieldSchema::getName) .collect(Collectors.toList()); } @@ -787,16 +790,9 @@ private List getColsInternal(boolean forMs) { * @return List<FieldSchema> */ public List getAllCols() { - List allCols = new ArrayList<>(getCols()); - Set colNames = new HashSet<>(); - for (FieldSchema col : allCols) { - colNames.add(col.getName()); - } - for (FieldSchema col : getPartCols()) { - if (!colNames.contains(col.getName())) { - allCols.add(col); - } - } + ArrayList allCols = new ArrayList<>(); + allCols.addAll(getCols()); + allCols.addAll(getPartCols()); return allCols; } @@ -847,7 +843,7 @@ public void setOutputFormatClass(String name) throws HiveException { public boolean isPartitioned() { return hasNonNativePartitionSupport() ? getStorageHandler().isPartitioned(this) : - CollectionUtils.isNotEmpty(getPartCols()); + CollectionUtils.isNotEmpty(getEffectivePartCols()); } public void setFields(List fields) { @@ -1045,7 +1041,7 @@ public boolean isMaterializedView() { public LinkedHashMap createSpec( org.apache.hadoop.hive.metastore.api.Partition tp) { - List fsl = getPartCols(); + List fsl = getEffectivePartCols(); List tpl = tp.getValues(); LinkedHashMap spec = new LinkedHashMap(fsl.size()); for (int i = 0; i < fsl.size(); i++) { @@ -1187,7 +1183,7 @@ public static void validateColumns(List columns, List } colNames.add(colName); } - if (partCols != null && !icebergTable) { + if (partCols != null) { // there is no overlap between columns and partitioning columns for (FieldSchema partCol: partCols) { String colName = normalize(partCol.getName()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index bd1be003a512..c560783f070f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -2149,7 +2149,7 @@ static void usePartitionColumns(Properties properties, Table table, List if (properties.containsKey(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS)) { usePartitionColumns(properties, partColNames); } else { - List partCols = table.getPartCols(); + List partCols = table.getEffectivePartCols(); String partNames = partCols.stream().map(FieldSchema::getName).collect(Collectors.joining("/")); String partTypes = partCols.stream().map(FieldSchema::getType).collect(Collectors.joining(":")); properties.setProperty( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java index b1bc9eaf0a75..428f1f8cf537 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java @@ -525,7 +525,7 @@ static private boolean pruneBySequentialScan(Table tab, List partitio } private static List extractPartColTypes(Table tab) { - List pCols = tab.getPartCols(); + List pCols = tab.getEffectivePartCols(); List partColTypeInfos = new ArrayList<>(pCols.size()); for (FieldSchema pCol : pCols) { partColTypeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(pCol.getType())); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java index 05f3b85f271f..c42f39fa6309 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java @@ -175,7 +175,7 @@ private void analyzeAcidExport(ASTNode ast, Table exportTable, ASTNode tokRefOrN //now generate insert statement //insert into newTableName select * from ts StringBuilder rewrittenQueryStr = generateExportQuery( - newTable.getPartCols(), + newTable.getEffectivePartCols(), tokRefOrNameExportTable, (ASTNode) tokRefOrNameExportTable.parent, newTableName); ReparseResult rr = ParseUtils.parseRewrittenQuery(ctx, rewrittenQueryStr); Context rewrittenCtx = rr.rewrittenCtx; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index f616049d8591..211462775911 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -1202,7 +1202,7 @@ public TableSpec(Table tableHandle, List partitions) { if (partitions != null && !partitions.isEmpty()) { this.specType = SpecType.STATIC_PARTITION; this.partitions = partitions; - List partCols = this.tableHandle.getPartCols(); + List partCols = this.tableHandle.getEffectivePartCols(); this.partSpec = new LinkedHashMap<>(); for (FieldSchema partCol : partCols) { partSpec.put(partCol.getName(), null); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 2176dd133d16..00cc988eb0a8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -2998,7 +2998,6 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc final NotNullConstraint nnc = tabMetaData.getNotNullConstraint(); final PrimaryKeyInfo pkc = tabMetaData.getPrimaryKeyInfo(); - Set alreadyAdded = new HashSet<>(); for (StructField structField : fields) { colName = structField.getFieldName(); colInfo = new ColumnInfo( @@ -3007,7 +3006,6 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc isNullable(colName, nnc, pkc), tableAlias, false); colInfo.setSkewedCol(isSkewedCol(tableAlias, qb, colName)); rr.put(tableAlias, colName, colInfo); - alreadyAdded.add(colName); cInfoLst.add(colInfo); } // TODO: Fix this @@ -3017,9 +3015,6 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc // 3.2 Add column info corresponding to partition columns for (FieldSchema part_col : tabMetaData.getPartCols()) { colName = part_col.getName(); - if (tabMetaData.hasNonNativePartitionSupport()) { - break; - } colInfo = new ColumnInfo(colName, TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), isNullable(colName, nnc, pkc), tableAlias, true); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java index d8c7d9527ec3..b3e25950985e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java @@ -84,7 +84,7 @@ public ColumnStatsAutoGatherContext(SemanticAnalyzer sa, HiveConf conf, this.origCtx = ctx; columns = tbl.getCols(); // current behaviour intact until we have getCols() giving only non-partition columns for non native tables as well - partitionColumns = tbl.hasNonNativePartitionSupport() ? new ArrayList<>() : tbl.getPartCols(); + partitionColumns = tbl.getPartCols(); } public List getLoadFileWork() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java index 9bf6269334e7..5c9f898bea13 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java @@ -207,7 +207,7 @@ private static CharSequence genPartitionClause(Table tbl, List pa private static String getColTypeOf(Table tbl, String partKey) { - for (FieldSchema fs : tbl.getPartCols()) { + for (FieldSchema fs : tbl.getEffectivePartCols()) { if (partKey.equalsIgnoreCase(fs.getName())) { return fs.getType().toLowerCase(); } @@ -309,7 +309,7 @@ private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, if (isPartitionStats) { if (partTransformSpec == null) { - for (FieldSchema fs : tbl.getPartCols()) { + for (FieldSchema fs : tbl.getEffectivePartCols()) { String identifier = unparseIdentifier(fs.getName(), conf); rewrittenQueryBuilder.append(", ").append(identifier); columnNamesBuilder.append(", ").append(identifier); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java index dcf197a2c201..e671ad24d7b9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java @@ -876,7 +876,7 @@ private static void checkTable(Table table, ImportTableDesc tableDesc, } { // check partitioning column order and types - List existingTablePartCols = table.getPartCols(); + List existingTablePartCols = table.getEffectivePartCols(); List importedTablePartCols = tableDesc.getPartCols(); if (!EximUtil.schemaCompare(importedTablePartCols, existingTablePartCols)) { throw new SemanticException( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index 6aa5b08fd5f4..9093570706f3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -23,7 +23,6 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -512,7 +511,7 @@ private void reparseAndSuperAnalyze(Table table, URI fromURI) throws SemanticExc // Partition spec was already validated by caller when create TableSpec object. // So, need not validate inpPartSpec here. - List parts = table.hasNonNativePartitionSupport() ? Collections.emptyList() : table.getPartCols(); + List parts = table.getPartCols(); if (tableTree.getChildCount() >= 2) { ASTNode partSpecNode = (ASTNode) tableTree.getChild(1); inpPartSpec = new HashMap<>(partSpecNode.getChildCount()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java index ac0a7e8f12be..353ad9e2b574 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java @@ -431,7 +431,7 @@ private static final class OnClauseAnalyzer { HiveConf conf, String onClauseAsString) { this.onClause = onClause; allTargetTableColumns.addAll(targetTable.getCols()); - allTargetTableColumns.addAll(targetTable.getPartCols()); + allTargetTableColumns.addAll(targetTable.getEffectivePartCols()); this.targetTableNameInSourceQuery = unescapeIdentifier(targetTableNameInSourceQuery); this.conf = conf; this.onClauseAsString = onClauseAsString; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java index 5fa22d9a2f82..27bde7acfe2d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java @@ -581,7 +581,7 @@ public static Map> getFullPartitionSpecs( CommonTree ast, Table table, Configuration conf, boolean canGroupExprs) throws SemanticException { String defaultPartitionName = HiveConf.getVar(conf, HiveConf.ConfVars.DEFAULT_PARTITION_NAME); Map colTypes = new HashMap<>(); - List partitionKeys = table.getPartCols(); + List partitionKeys = table.getEffectivePartCols(); for (FieldSchema fs : partitionKeys) { colTypes.put(fs.getName().toLowerCase(), fs.getType()); } @@ -691,7 +691,7 @@ public static Map> getFullPartitionSpecs( */ private static int calculatePartPrefix(Table tbl, Set partSpecKeys) { int partPrefixToDrop = 0; - for (FieldSchema fs : tbl.getPartCols()) { + for (FieldSchema fs : tbl.getEffectivePartCols()) { if (!partSpecKeys.contains(fs.getName())) { break; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java index 8e52f63c6611..101f6b1fc3d8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java @@ -120,7 +120,7 @@ protected void checkValidSetClauseTarget(ASTNode colName, Table targetTable) thr // Make sure this isn't one of the partitioning columns, that's not supported. for (FieldSchema fschema : targetTable.getPartCols()) { - if (fschema.getName().equalsIgnoreCase(columnName) && !targetTable.hasNonNativePartitionSupport()) { + if (fschema.getName().equalsIgnoreCase(columnName)) { throw new SemanticException(ErrorMsg.UPDATE_CANNOT_UPDATE_PART_VALUE.getMsg()); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 4dae0653e0ee..915ff21bd912 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -2210,7 +2210,7 @@ private void handleInsertStatementSpecPhase1(ASTNode ast, QBParseInfo qbp, Phase } } else { // partition spec is not specified but column schema can have partitions specified - for(FieldSchema f : targetTable.getPartCols()) { + for(FieldSchema f : targetTable.getEffectivePartCols()) { //parser only allows foo(a,b), not foo(foo.a, foo.b) targetColumns.remove(f.getName()); } @@ -12003,8 +12003,6 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { // Determine row schema for TSOP. // Include column names from SerDe, the partition and virtual columns. rwsch = new RowResolver(); - Set partCols = tab.hasNonNativePartitionSupport() ? - Sets.newHashSet(tab.getPartColNames()) : Collections.emptySet(); try { // Including parameters passed in the query if (properties != null) { @@ -12022,6 +12020,8 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { deserializer.handleJobLevelConfiguration(conf); List fields = rowObjectInspector .getAllStructFieldRefs(); + Set partCols = tab.hasNonNativePartitionSupport() ? + Sets.newHashSet(tab.getPartColNames()) : Collections.emptySet(); for (int i = 0; i < fields.size(); i++) { /** * if the column is a skewed column, use ColumnInfo accordingly @@ -12041,9 +12041,6 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { // Hack!! - refactor once the metadata APIs with types are ready // Finally add the partitioning columns for (FieldSchema part_col : tab.getPartCols()) { - if(partCols.contains(part_col.getName())){ - break; - } LOG.trace("Adding partition col: " + part_col); rwsch.put(alias, part_col.getName(), new ColumnInfo(part_col.getName(), TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), alias, true)); @@ -12309,7 +12306,7 @@ private void setupStats(TableScanDesc tsDesc, QBParseInfo qbp, Table tab, String if (tab.isPartitioned() && !tab.hasNonNativePartitionSupport()) { List cols = new ArrayList(); if (qbp.getAnalyzeRewrite() != null) { - List partitionCols = tab.getPartCols(); + List partitionCols = tab.getEffectivePartCols(); for (FieldSchema fs : partitionCols) { cols.add(fs.getName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java index c9ba50d110f8..cb759a11d080 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java @@ -249,11 +249,9 @@ protected void addValues(Table targetTable, String targetAlias, Map values.add( - formatter.apply(fieldSchema.getName()))); - } + + targetTable.getPartCols().forEach(fieldSchema -> values.add( + formatter.apply(fieldSchema.getName()))); } protected String getRhsExpValue(String newValue, String alias) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java index 838d1d8a09a6..d14ddc7eb485 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java @@ -98,7 +98,7 @@ public ParseUtils.ReparseResult rewrite(Context context, UpdateStatement updateB insertValues.add(sqlGenerator.qualify(identifier)); } - if (!updateBlock.getTargetTable().hasNonNativePartitionSupport()) { + if (updateBlock.getTargetTable().getPartCols() != null) { updateBlock.getTargetTable().getPartCols().forEach( fieldSchema -> insertValues.add(sqlGenerator.qualify(HiveUtils.unparseIdentifier(fieldSchema.getName(), conf)))); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java index 6576dd28d2ec..928564012985 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java @@ -111,9 +111,6 @@ public void appendPartitionColsOfTarget() { */ public void appendPartitionCols(Table table) { // If the table is partitioned we have to put the partition() clause in - if (table.hasNonNativePartitionSupport()) { - return; - } List partCols = table.getPartCols(); if (partCols == null || partCols.isEmpty()) { return; @@ -151,14 +148,11 @@ public void removeLastChar() { } public void appendPartColsOfTargetTableWithComma(String alias) { - if (targetTable.hasNonNativePartitionSupport()) { - return; - } - if (targetTable.getPartCols().isEmpty()) { + if (targetTable.getPartCols() == null || targetTable.getPartCols().isEmpty()) { return; } queryStr.append(','); - appendCols(targetTable.getPartCols(), alias, null, FieldSchema::getName); + appendCols(targetTable.getEffectivePartCols(), alias, null, FieldSchema::getName); } public void appendAllColsOfTargetTable(String prefix) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java index 87e426800442..94cda746396b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java @@ -36,7 +36,7 @@ public NativeAcidMultiInsertSqlGenerator(Table table, String targetTableFullName @Override public void appendAcidSelectColumns(Operation operation) { queryStr.append("ROW__ID,"); - for (FieldSchema fieldSchema : targetTable.getPartCols()) { + for (FieldSchema fieldSchema : targetTable.getEffectivePartCols()) { String identifier = HiveUtils.unparseIdentifier(fieldSchema.getName(), this.conf); queryStr.append(identifier); queryStr.append(","); @@ -45,9 +45,9 @@ public void appendAcidSelectColumns(Operation operation) { @Override public List getDeleteValues(Operation operation) { - List deleteValues = new ArrayList<>(1 + targetTable.getPartCols().size()); + List deleteValues = new ArrayList<>(1 + targetTable.getEffectivePartCols().size()); deleteValues.add(qualify("ROW__ID")); - for (FieldSchema fieldSchema : targetTable.getPartCols()) { + for (FieldSchema fieldSchema : targetTable.getEffectivePartCols()) { deleteValues.add(qualify(HiveUtils.unparseIdentifier(fieldSchema.getName(), conf))); } return deleteValues; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java index be62d94019ed..cb257ee216f2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java @@ -156,7 +156,7 @@ private boolean constructColumnStatsFromPackedRows(Table tbl, List Date: Sun, 12 Apr 2026 21:24:51 +0530 Subject: [PATCH 07/10] addressed sonar issues --- .../llap/io/api/impl/LlapInputFormat.java | 3 +- .../hadoop/hive/llap/ProactiveEviction.java | 32 ++++----- .../column/show/ShowColumnsOperation.java | 8 +-- .../formatter/TextDescTableFormatter.java | 34 ++++------ .../hadoop/hive/ql/exec/ArchiveUtils.java | 3 +- .../ql/exec/vector/VectorizedRowBatchCtx.java | 3 +- .../hive/ql/metadata/DummyPartition.java | 1 - .../HiveMaterializedViewsRegistry.java | 6 +- .../hadoop/hive/ql/metadata/Partition.java | 21 +++--- .../apache/hadoop/hive/ql/metadata/Table.java | 67 +++++++++---------- .../ql/optimizer/physical/Vectorizer.java | 2 +- .../ql/optimizer/ppr/PartExprEvalUtils.java | 9 ++- .../parse/ColumnStatsAutoGatherContext.java | 1 - .../hive/ql/parse/MergeSemanticAnalyzer.java | 30 ++++----- .../hadoop/hive/ql/parse/ParseUtils.java | 26 +++---- .../hive/ql/parse/SemanticAnalyzer.java | 8 +-- .../hadoop/hive/ql/plan/PartitionDesc.java | 8 +-- .../hadoop/hive/common/io/CacheTag.java | 2 +- 18 files changed, 112 insertions(+), 152 deletions(-) diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java index e74885e57a3d..67e59155b67c 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java @@ -31,7 +31,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; @@ -232,7 +231,7 @@ static VectorizedRowBatchCtx createFakeVrbCtx(MapWork mapWork) throws HiveExcept if (paths.hasNext()) { PartitionDesc partDesc = mapWork.getPathToPartitionInfo().get(paths.next()); if (partDesc != null) { - LinkedHashMap partSpec = partDesc.getPartSpec(); + Map partSpec = partDesc.getPartSpec(); if (partSpec != null && !partSpec.isEmpty()) { partitionColumnCount = partSpec.size(); } diff --git a/ql/src/java/org/apache/hadoop/hive/llap/ProactiveEviction.java b/ql/src/java/org/apache/hadoop/hive/llap/ProactiveEviction.java index 120949fc9949..f4a008257bb8 100644 --- a/ql/src/java/org/apache/hadoop/hive/llap/ProactiveEviction.java +++ b/ql/src/java/org/apache/hadoop/hive/llap/ProactiveEviction.java @@ -158,13 +158,13 @@ public static final class Request { // Holds a hierarchical structure of DBs, tables and partitions such as: // { testdb : { testtab0 : [], testtab1 : [ {pk0 : p0v0, pk1 : p0v1}, {pk0 : p1v0, pk1 : p1v1} ] }, testdb2 : {} } - private final Map>>> entities; + private final Map>>> entities; - private Request(Map>>> entities) { + private Request(Map>>> entities) { this.entities = entities; } - public Map>>> getEntities() { + public Map>>> getEntities() { return entities; } @@ -191,21 +191,21 @@ public List toProtoRequests() List protoRequests = new LinkedList<>(); - for (Map.Entry>>> dbEntry : entities.entrySet()) { + for (Map.Entry>>> dbEntry : entities.entrySet()) { String dbName = dbEntry.getKey(); - Map>> tables = dbEntry.getValue(); + Map>> tables = dbEntry.getValue(); LlapDaemonProtocolProtos.EvictEntityRequestProto.Builder requestBuilder = LlapDaemonProtocolProtos.EvictEntityRequestProto.newBuilder(); LlapDaemonProtocolProtos.TableProto.Builder tableBuilder = null; requestBuilder.setDbName(dbName.toLowerCase()); - for (Map.Entry>> tableEntry : tables.entrySet()) { + for (Map.Entry>> tableEntry : tables.entrySet()) { String tableName = tableEntry.getKey(); tableBuilder = LlapDaemonProtocolProtos.TableProto.newBuilder(); tableBuilder.setTableName(tableName.toLowerCase()); - Set> partitions = tableEntry.getValue(); + Set> partitions = tableEntry.getValue(); Set partitionKeys = null; for (Map partitionSpec : partitions) { @@ -245,7 +245,7 @@ public boolean isTagMatch(CacheTag cacheTag) { return false; } - Map>> tables = entities.get(db); + Map>> tables = entities.get(db); // If true, must be a drop DB event and this cacheTag matches. if (tables.isEmpty()) { @@ -261,7 +261,7 @@ public boolean isTagMatch(CacheTag cacheTag) { for (String tableAndDbName : tables.keySet()) { if (tableAndDbName.equals(tagTableName.getNotEmptyDbTable())) { - Set> partDescs = tables.get(tableAndDbName); + Set> partDescs = tables.get(tableAndDbName); // If true, must be a drop table event, and this cacheTag matches. if (partDescs == null) { @@ -292,7 +292,7 @@ public String toString() { */ public static final class Builder { - private final Map>>> entities; + private final Map>>> entities; private Builder() { this.entities = new HashMap<>(); @@ -302,7 +302,7 @@ public static Builder create() { return new Builder(); } - public Builder addPartitionOfATable(String db, String tableName, LinkedHashMap partSpec) { + public Builder addPartitionOfATable(String db, String tableName, Map partSpec) { ensureDb(db); ensureTable(db, tableName); entities.get(db).get(tableName).add(partSpec); @@ -325,7 +325,7 @@ public Request build() { } private void ensureDb(String dbName) { - Map>> tables = entities.get(dbName); + Map>> tables = entities.get(dbName); if (tables == null) { tables = new HashMap<>(); entities.put(dbName, tables); @@ -334,9 +334,9 @@ private void ensureDb(String dbName) { private void ensureTable(String dbName, String tableName) { ensureDb(dbName); - Map>> tables = entities.get(dbName); + Map>> tables = entities.get(dbName); - Set> partitions = tables.get(tableName); + Set> partitions = tables.get(tableName); if (partitions == null) { partitions = new HashSet<>(); tables.put(tableName, partitions); @@ -352,7 +352,7 @@ public Builder fromProtoRequest(LlapDaemonProtocolProtos.EvictEntityRequestProto entities.clear(); String dbName = protoRequest.getDbName().toLowerCase(); - Map>> entitiesInDb = new HashMap<>(); + Map>> entitiesInDb = new HashMap<>(); List tables = protoRequest.getTableList(); if (tables != null && !tables.isEmpty()) { @@ -364,7 +364,7 @@ public Builder fromProtoRequest(LlapDaemonProtocolProtos.EvictEntityRequestProto entitiesInDb.put(dbAndTableName, null); continue; } - Set> partitions = new HashSet<>(); + Set> partitions = new HashSet<>(); LinkedHashMap partDesc = new LinkedHashMap<>(); for (int valIx = 0; valIx < table.getPartValCount(); ++valIx) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java index 6a2abd4ddfc2..87d115df72ed 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java @@ -91,13 +91,7 @@ private List filterColumns(List columns, Matcher match } if (desc.isSorted()) { - result.sort( - new Comparator() { - @Override - public int compare(FieldSchema f1, FieldSchema f2) { - return f1.getName().compareTo(f2.getName()); - } - }); + result.sort(Comparator.comparing(FieldSchema::getName)); } return result; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java index 2ecce54e088d..651b40c6f37d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java @@ -59,7 +59,6 @@ import java.io.DataOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; @@ -70,7 +69,6 @@ import java.util.Set; import java.util.TreeMap; import java.util.Map.Entry; -import java.util.stream.Collectors; import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.TABLE_IS_CTAS; import static org.apache.hadoop.hive.ql.ddl.ShowUtils.ALIGNMENT; @@ -171,11 +169,7 @@ private void addPartitionData(DataOutputStream out, HiveConf conf, String column boolean isFormatted, boolean isOutputPadded) throws IOException { String partitionData = ""; if (columnPath == null) { - List partitionColumns = null; - // TODO (HIVE-29413): Refactor to a generic getPartCols() implementation - if (table.isPartitioned()) { - partitionColumns = table.getEffectivePartCols(); - } + List partitionColumns = table.isPartitioned() ? table.getEffectivePartCols() : null; if (CollectionUtils.isNotEmpty(partitionColumns) && conf.getBoolVar(ConfVars.HIVE_DISPLAY_PARTITION_COLUMNS_SEPARATELY)) { TextMetaDataTable metaDataTable = new TextMetaDataTable(); @@ -202,13 +196,9 @@ private void addPartitionData(DataOutputStream out, HiveConf conf, String column } private void addFormattedTableData(DataOutputStream out, Table table, Partition partition, boolean isOutputPadded) - throws IOException, UnsupportedEncodingException { - String formattedTableInfo = null; - if (partition != null) { - formattedTableInfo = getPartitionInformation(table, partition); - } else { - formattedTableInfo = getTableInformation(table, isOutputPadded); - } + throws IOException { + String formattedTableInfo = (partition != null) ? getPartitionInformation(table, partition) : + getTableInformation(table, isOutputPadded); if (table.getTableConstraintsInfo().isTableConstraintsInfoNotEmpty()) { formattedTableInfo += getConstraintsInformation(table); @@ -335,7 +325,7 @@ private void getStorageDescriptorInfo(StringBuilder tableInfo, Table table, Stor List skewedCoumnNames = storageDesc.getSkewedInfo().getSkewedColNames().stream() .sorted() - .collect(Collectors.toList()); + .toList(); formatOutput("Skewed Columns:", skewedCoumnNames.toString(), tableInfo); } @@ -343,16 +333,16 @@ private void getStorageDescriptorInfo(StringBuilder tableInfo, Table table, Stor List> skewedColumnValues = storageDesc.getSkewedInfo().getSkewedColValues().stream() .sorted(new VectorComparator()) - .collect(Collectors.toList()); + .toList(); formatOutput("Skewed Values:", skewedColumnValues.toString(), tableInfo); } - Map, String> skewedColMap = new TreeMap<>(new VectorComparator()); + Map, String> skewedColMap = new TreeMap<>(new VectorComparator<>()); skewedColMap.putAll(storageDesc.getSkewedInfo().getSkewedColValueLocationMaps()); if (MapUtils.isNotEmpty(skewedColMap)) { formatOutput("Skewed Value to Path:", skewedColMap.toString(), tableInfo); Map, String> truncatedSkewedColMap = - new TreeMap, String>(new VectorComparator()); + new TreeMap<>(new VectorComparator<>()); // walk through existing map to truncate path so that test won't mask it then we can verify location is right Set, String>> entries = skewedColMap.entrySet(); for (Entry, String> entry : entries) { @@ -401,7 +391,7 @@ private void getPartitionMetaDataInformation(StringBuilder tableInfo, Partition } } - private class VectorComparator> implements Comparator>{ + private static final class VectorComparator> implements Comparator> { @Override public int compare(List listA, List listB) { for (int i = 0; i < listA.size() && i < listB.size(); i++) { @@ -436,7 +426,7 @@ private void displayAllParameters(Map params, StringBuilder tabl private void displayAllParameters(Map params, StringBuilder tableInfo, boolean escapeUnicode, boolean isOutputPadded) { - List keys = new ArrayList(params.keySet()); + List keys = new ArrayList<>(params.keySet()); Collections.sort(keys); for (String key : keys) { String value = params.get(key); @@ -624,7 +614,7 @@ private void addExtendedTableData(DataOutputStream out, Table table, Partition p } private void addExtendedConstraintData(DataOutputStream out, Table table) - throws IOException, UnsupportedEncodingException { + throws IOException { if (table.getTableConstraintsInfo().isTableConstraintsInfoNotEmpty()) { out.write(("Constraints").getBytes(StandardCharsets.UTF_8)); out.write(Utilities.tabCode); @@ -656,7 +646,7 @@ private void addExtendedConstraintData(DataOutputStream out, Table table) } private void addExtendedStorageData(DataOutputStream out, Table table) - throws IOException, UnsupportedEncodingException { + throws IOException { if (table.getStorageHandlerInfo() != null) { out.write(("StorageHandlerInfo").getBytes(StandardCharsets.UTF_8)); out.write(Utilities.newLineCode); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java index f25cadd40073..df2a92202bcc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java @@ -24,7 +24,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -254,7 +253,7 @@ public static String getPartialName(Partition p, int level) throws HiveException * @throws HiveException */ public static String conflictingArchiveNameOrNull(Hive db, Table tbl, - LinkedHashMap partSpec) + Map partSpec) throws HiveException { List partKeys = tbl.getPartitionKeys(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java index a0906cfb0339..38c4dfb036b8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.util.Arrays; -import java.util.LinkedHashMap; import java.util.Map; import java.util.stream.IntStream; @@ -287,7 +286,7 @@ public static void getPartitionValues(VectorizedRowBatchCtx vrbCtx, public static void getPartitionValues(VectorizedRowBatchCtx vrbCtx, PartitionDesc partDesc, Object[] partitionValues) { - LinkedHashMap partSpec = partDesc.getPartSpec(); + Map partSpec = partDesc.getPartSpec(); for (int i = 0; i < vrbCtx.partitionColumnCount; i++) { Object objectValue; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java index 7af16770c059..782b4f6e5258 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java @@ -90,7 +90,6 @@ public List getValues() { Table table = this.getTable(); values = new ArrayList<>(); - // TODO (HIVE-29413): Refactor to a generic getPartCols() implementation for (FieldSchema fs : table.getEffectivePartCols()) { String val = partSpec.get(fs.getName()); values.add(val); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java index b763c379bdc7..2453a0c64358 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java @@ -429,10 +429,10 @@ private static RelNode createMaterializedViewScan(HiveConf conf, Table viewTable // 1.2 Add column info corresponding to partition columns ArrayList partitionColumns = new ArrayList(); - for (FieldSchema part_col : viewTable.getEffectivePartCols()) { - colName = part_col.getName(); + for (FieldSchema partCol : viewTable.getEffectivePartCols()) { + colName = partCol.getName(); colInfo = new ColumnInfo(colName, - TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), null, true); + TypeInfoFactory.getPrimitiveTypeInfo(partCol.getType()), null, true); rr.put(null, colName, colInfo); cInfoLst.add(colInfo); partitionColumns.add(colInfo); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java index d284e0231a03..2e24dcd0087b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java @@ -23,7 +23,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -124,7 +123,7 @@ public Partition(Table tbl, Map partSpec, Path location) throws public static org.apache.hadoop.hive.metastore.api.Partition createMetaPartitionObject( Table tbl, Map partSpec, Path location) throws HiveException { - List pvals = new ArrayList(); + List pvals = new ArrayList<>(); for (FieldSchema field : tbl.getEffectivePartCols()) { String val = partSpec.get(field.getName()); if (val == null || val.isEmpty()) { @@ -417,7 +416,7 @@ public Path[] getPath(Sample s) throws HiveException { } int scount = s.getSampleFraction(); - ArrayList ret = new ArrayList(); + List ret = new ArrayList<>(); if (bcount == scount) { ret.add(getBucketPath(s.getSampleNum() - 1)); @@ -429,7 +428,7 @@ public Path[] getPath(Sample s) throws HiveException { } // undersampling a bucket ret.add(getBucketPath((s.getSampleNum() - 1) % bcount)); - } else if (bcount > scount) { + } else { if ((bcount / scount) * scount != bcount) { throw new HiveException("Sample Count" + scount + " is not a divisor of bucket count " + bcount + " for table " @@ -440,11 +439,11 @@ public Path[] getPath(Sample s) throws HiveException { ret.add(getBucketPath(i * scount + (s.getSampleNum() - 1))); } } - return (ret.toArray(new Path[ret.size()])); + return (ret.toArray(new Path[0])); } } - public LinkedHashMap getSpec() { + public Map getSpec() { return table.createSpec(tPartition); } @@ -543,7 +542,7 @@ public void setLocation(String location) { */ public void setValues(Map partSpec) throws HiveException { - List pvals = new ArrayList(); + List pvals = new ArrayList<>(); for (FieldSchema field : table.getEffectivePartCols()) { String val = partSpec.get(field.getName()); if (val == null) { @@ -583,12 +582,11 @@ public List getSkewedColNames() { return tPartition.getSd().getSkewedInfo().getSkewedColNames(); } - public void setSkewedValueLocationMap(List valList, String dirName) - throws HiveException { + public void setSkewedValueLocationMap(List valList, String dirName) { Map, String> mappings = tPartition.getSd().getSkewedInfo() .getSkewedColValueLocationMaps(); if (null == mappings) { - mappings = new HashMap, String>(); + mappings = new HashMap<>(); tPartition.getSd().getSkewedInfo().setSkewedColValueLocationMaps(mappings); } @@ -613,8 +611,7 @@ public int hashCode() { @Override public boolean equals(Object obj) { - if (obj instanceof Partition) { - Partition o = (Partition) obj; + if (obj instanceof Partition o) { return Objects.equals(tPartition, o.tPartition); } return false; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index 809bab85af12..8587797162b7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.metadata; import java.io.IOException; +import java.io.Serial; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; @@ -31,7 +32,7 @@ import java.util.Objects; import java.util.Properties; import java.util.Set; -import java.util.stream.Collectors; + import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -47,6 +48,7 @@ import org.apache.hadoop.hive.metastore.HiveMetaStoreUtils; import org.apache.hadoop.hive.metastore.api.SourceTable; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.HiveMetaHook; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; @@ -57,7 +59,6 @@ import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; -import org.apache.hadoop.hive.metastore.HiveMetaHook; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; @@ -101,6 +102,7 @@ */ public class Table implements Serializable { + @Serial private static final long serialVersionUID = 1L; static final private Logger LOG = LoggerFactory.getLogger("hive.ql.metadata.Table"); @@ -110,6 +112,7 @@ public class Table implements Serializable { /** * These fields are all cached fields. The information comes from tTable. */ + private List cachedPartCols; private transient Deserializer deserializer; private Class outputFormatClass; private Class inputFormatClass; @@ -119,8 +122,6 @@ public class Table implements Serializable { private transient StorageHandlerInfo storageHandlerInfo; private transient MaterializedViewMetadata materializedViewMetadata; - private List cachedPartCols; - private TableSpec tableSpec; private boolean materializedTable; @@ -220,7 +221,6 @@ public org.apache.hadoop.hive.metastore.api.Table getTTable() { */ public void setTTable(org.apache.hadoop.hive.metastore.api.Table tTable) { this.tTable = tTable; - clearCachedPartCols(); } /** @@ -232,11 +232,11 @@ public void setTTable(org.apache.hadoop.hive.metastore.api.Table tTable) { { sd.setSerdeInfo(new SerDeInfo()); sd.setNumBuckets(-1); - sd.setBucketCols(new ArrayList()); - sd.setCols(new ArrayList()); - sd.setParameters(new HashMap()); - sd.setSortCols(new ArrayList()); - sd.getSerdeInfo().setParameters(new HashMap()); + sd.setBucketCols(new ArrayList<>()); + sd.setCols(new ArrayList<>()); + sd.setParameters(new HashMap<>()); + sd.setSortCols(new ArrayList<>()); + sd.getSerdeInfo().setParameters(new HashMap<>()); // We have to use MetadataTypedColumnsetSerDe because LazySimpleSerDe does // not support a table with no columns. sd.getSerdeInfo().setSerializationLib(MetadataTypedColumnsetSerDe.class.getName()); @@ -246,17 +246,17 @@ public void setTTable(org.apache.hadoop.hive.metastore.api.Table tTable) { sd.setInputFormat(SequenceFileInputFormat.class.getName()); sd.setOutputFormat(HiveSequenceFileOutputFormat.class.getName()); SkewedInfo skewInfo = new SkewedInfo(); - skewInfo.setSkewedColNames(new ArrayList()); - skewInfo.setSkewedColValues(new ArrayList>()); - skewInfo.setSkewedColValueLocationMaps(new HashMap, String>()); + skewInfo.setSkewedColNames(new ArrayList<>()); + skewInfo.setSkewedColValues(new ArrayList<>()); + skewInfo.setSkewedColValueLocationMaps(new HashMap<>()); sd.setSkewedInfo(skewInfo); } org.apache.hadoop.hive.metastore.api.Table t = new org.apache.hadoop.hive.metastore.api.Table(); { t.setSd(sd); - t.setPartitionKeys(new ArrayList()); - t.setParameters(new HashMap()); + t.setPartitionKeys(new ArrayList<>()); + t.setParameters(new HashMap<>()); t.setTableType(TableType.MANAGED_TABLE.toString()); t.setDbName(databaseName); t.setTableName(tableName); @@ -409,7 +409,7 @@ public void setStorageHandlerInfo(StorageHandlerInfo storageHandlerInfo) { this.storageHandlerInfo = storageHandlerInfo; } - final public Class getInputFormatClass() { + public final Class getInputFormatClass() { if (inputFormatClass == null) { try { String className = tTable.getSd().getInputFormat(); @@ -429,7 +429,7 @@ final public Class getInputFormatClass() { return inputFormatClass; } - final public Class getOutputFormatClass() { + public final Class getOutputFormatClass() { if (outputFormatClass == null) { try { String className = tTable.getSd().getOutputFormat(); @@ -463,7 +463,7 @@ public void setMaterializedTable(boolean materializedTable) { * Marker SemanticException, so that processing that allows for table validation failures * and appropriately handles them can recover from these types of SemanticExceptions */ - public class ValidationFailureSemanticException extends SemanticException{ + public static class ValidationFailureSemanticException extends SemanticException{ public ValidationFailureSemanticException(String s) { super(s); } @@ -533,9 +533,9 @@ public TableType getTableType() { return Enum.valueOf(TableType.class, tTable.getTableType()); } - public ArrayList getFields() { + public List getFields() { - ArrayList fields = new ArrayList(); + List fields = new ArrayList<>(); try { Deserializer decoder = getDeserializer(); @@ -610,6 +610,10 @@ public List getPartCols() { return partKeys; } + /** + * Returns partition columns, consulting the storage handler for non-native tables (e.g. Iceberg) + * where partition columns are not stored in the metastore. + */ public List getEffectivePartCols() { if (cachedPartCols != null) { return cachedPartCols; @@ -622,10 +626,6 @@ public List getEffectivePartCols() { return cachedPartCols; } - private void clearCachedPartCols() { - cachedPartCols = null; - } - private boolean isTableTypeSet() { if (tTable.getParameters() == null) { return false; @@ -641,8 +641,7 @@ public FieldSchema getPartColByName(String colName) { } public List getPartColNames() { - return getEffectivePartCols().stream().map(FieldSchema::getName) - .collect(Collectors.toList()); + return getEffectivePartCols().stream().map(FieldSchema::getName).toList(); } public boolean hasNonNativePartitionSupport() { @@ -700,7 +699,7 @@ public void setSkewedValueLocationMap(List valList, String dirName) { Map, String> mappings = tTable.getSd().getSkewedInfo() .getSkewedColValueLocationMaps(); if (null == mappings) { - mappings = new HashMap, String>(); + mappings = new HashMap<>(); tTable.getSd().getSkewedInfo().setSkewedColValueLocationMaps(mappings); } @@ -710,7 +709,7 @@ public void setSkewedValueLocationMap(List valList, String dirName) { public Map, String> getSkewedColValueLocationMaps() { return (tTable.getSd().getSkewedInfo() != null) ? tTable.getSd().getSkewedInfo() - .getSkewedColValueLocationMaps() : new HashMap, String>(); + .getSkewedColValueLocationMaps() : new HashMap<>(); } public void setSkewedColValues(List> skewedValues) { @@ -719,7 +718,7 @@ public void setSkewedColValues(List> skewedValues) { public List> getSkewedColValues(){ return (tTable.getSd().getSkewedInfo() != null) ? tTable.getSd().getSkewedInfo() - .getSkewedColValues() : new ArrayList>(); + .getSkewedColValues() : new ArrayList<>(); } public void setSkewedColNames(List skewedColNames) { @@ -728,7 +727,7 @@ public void setSkewedColNames(List skewedColNames) { public List getSkewedColNames() { return (tTable.getSd().getSkewedInfo() != null) ? tTable.getSd().getSkewedInfo() - .getSkewedColNames() : new ArrayList(); + .getSkewedColNames() : new ArrayList<>(); } public SkewedInfo getSkewedInfo() { @@ -790,15 +789,13 @@ private List getColsInternal(boolean forMs) { * @return List<FieldSchema> */ public List getAllCols() { - ArrayList allCols = new ArrayList<>(); - allCols.addAll(getCols()); + ArrayList allCols = new ArrayList<>(getCols()); allCols.addAll(getPartCols()); return allCols; } public void setPartCols(List partCols) { tTable.setPartitionKeys(partCols); - clearCachedPartCols(); } public String getCatName() { @@ -1038,12 +1035,12 @@ public boolean isMaterializedView() { * Use the information from this partition. * @return Partition name to value mapping. */ - public LinkedHashMap createSpec( + public Map createSpec( org.apache.hadoop.hive.metastore.api.Partition tp) { List fsl = getEffectivePartCols(); List tpl = tp.getValues(); - LinkedHashMap spec = new LinkedHashMap(fsl.size()); + Map spec = LinkedHashMap.newLinkedHashMap(fsl.size()); for (int i = 0; i < fsl.size(); i++) { FieldSchema fs = fsl.get(i); String value = tpl.get(i); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index f4b4c2ff3bad..82f81861a4dc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -1767,7 +1767,7 @@ private ImmutablePair validateInputFormatAndSchemaEvolution(Ma * allColumnNameList and allTypeInfoList variables -- into the data and partition columns. */ - LinkedHashMap partSpec = partDesc.getPartSpec(); + Map partSpec = partDesc.getPartSpec(); if (partSpec != null && partSpec.size() > 0) { partitionColumnCount = partSpec.size(); dataColumnCount = dataAndPartColumnCount - partitionColumnCount; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java index 91340b1b76ef..dd4478e0acc9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.optimizer.ppr; import java.util.ArrayList; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -51,7 +50,7 @@ public class PartExprEvalUtils { * @throws HiveException */ static public Object evalExprWithPart(ExprNodeDesc expr, Partition p) throws HiveException { - LinkedHashMap partSpec = p.getSpec(); + Map partSpec = p.getSpec(); Properties partProps = p.getSchema(); String[] partKeyTypes; @@ -59,8 +58,8 @@ static public Object evalExprWithPart(ExprNodeDesc expr, Partition p) throws Hiv if (!partSpec.keySet().containsAll(expr.getCols())) { return null; } - partKeyTypes = p.getTable().getStorageHandler().getPartitionKeys(p.getTable()).stream() - .map(FieldSchema::getType).toArray(String[]::new); + partKeyTypes = p.getTable().getEffectivePartCols().stream().map(FieldSchema::getType) + .toArray(String[]::new); } else { String pcolTypes = partProps.getProperty(hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES); partKeyTypes = pcolTypes.trim().split(":"); @@ -104,7 +103,7 @@ public static Pair prepareExpr( ExprNodeDesc expr, List partColumnNames, List partColumnTypeInfos) throws HiveException { // Create the row object - List partObjectInspectors = new ArrayList(); + List partObjectInspectors = new ArrayList<>(); for (int i = 0; i < partColumnNames.size(); i++) { partObjectInspectors.add(PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector( partColumnTypeInfos.get(i))); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java index b3e25950985e..9109f9cb6086 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java @@ -83,7 +83,6 @@ public ColumnStatsAutoGatherContext(SemanticAnalyzer sa, HiveConf conf, this.isInsertInto = isInsertInto; this.origCtx = ctx; columns = tbl.getCols(); - // current behaviour intact until we have getCols() giving only non-partition columns for non native tables as well partitionColumns = tbl.getPartCols(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java index 353ad9e2b574..6f32b0a9293a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java @@ -229,7 +229,7 @@ private MergeStatement.UpdateClause handleUpdate(ASTNode whenMatchedUpdateClause String deleteExtraPredicate) throws SemanticException { assert whenMatchedUpdateClause.getType() == HiveParser.TOK_MATCHED; assert getWhenClauseOperation(whenMatchedUpdateClause).getType() == HiveParser.TOK_UPDATE; - Map newValuesMap = new HashMap<>(targetTable.getAllCols().size()); + Map newValuesMap = HashMap.newHashMap(targetTable.getAllCols().size()); ASTNode setClause = (ASTNode)getWhenClauseOperation(whenMatchedUpdateClause).getChild(0); //columns being updated -> update expressions; "setRCols" (last param) is null because we use actual expressions //before re-parsing, i.e. they are known to SemanticAnalyzer logic @@ -302,7 +302,7 @@ private List findWhenClauses(ASTNode tree, int start) throws SemanticEx "Unexpected node type found: " + whenClause.getType() + addParseInfo(whenClause); whenClauses.add(whenClause); } - if (whenClauses.size() <= 0) { + if (whenClauses.isEmpty()) { //Futureproofing: the parser will actually not allow this throw new SemanticException("Must have at least 1 WHEN clause in MERGE statement"); } @@ -498,11 +498,7 @@ private void handleUnresolvedColumns() { private void addColumn2Table(String tableName, String columnName) { tableName = tableName.toLowerCase(); //normalize name for mapping tableNamesFound.add(tableName); - List cols = table2column.get(tableName); - if (cols == null) { - cols = new ArrayList<>(); - table2column.put(tableName, cols); - } + List cols = table2column.computeIfAbsent(tableName, k -> new ArrayList<>()); //we want to preserve 'columnName' as it was in original input query so that rewrite //looks as much as possible like original query cols.add(columnName); @@ -525,7 +521,7 @@ private String getPredicate() { } StringBuilder sb = new StringBuilder(); for (String col : targetCols) { - if (sb.length() > 0) { + if (!sb.isEmpty()) { sb.append(" AND "); } //but preserve table name in SQL @@ -604,17 +600,15 @@ protected String getMatchedText(ASTNode n) { } protected boolean isAliased(ASTNode n) { - switch (n.getType()) { - case HiveParser.TOK_TABREF: - return findTabRefIdxs(n)[0] != 0; - case HiveParser.TOK_TABNAME: - return false; - case HiveParser.TOK_SUBQUERY: + return switch (n.getType()) { + case HiveParser.TOK_TABREF -> findTabRefIdxs(n)[0] != 0; + case HiveParser.TOK_TABNAME -> false; + case HiveParser.TOK_SUBQUERY -> { assert n.getChildCount() > 1 : "Expected Derived Table to be aliased"; - return true; - default: - throw raiseWrongType("TOK_TABREF|TOK_TABNAME", n); - } + yield true; + } + default -> throw raiseWrongType("TOK_TABREF|TOK_TABNAME", n); + }; } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java index 27bde7acfe2d..0904e2c9b5c5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java @@ -123,15 +123,11 @@ public static ASTNode parse( * @return boolean */ public static boolean isJoinToken(ASTNode node) { - switch (node.getToken().getType()) { - case HiveParser.TOK_JOIN: - case HiveParser.TOK_LEFTOUTERJOIN: - case HiveParser.TOK_RIGHTOUTERJOIN: - case HiveParser.TOK_FULLOUTERJOIN: - return true; - default: - return false; - } + return switch (node.getToken().getType()) { + case HiveParser.TOK_JOIN, HiveParser.TOK_LEFTOUTERJOIN, HiveParser.TOK_RIGHTOUTERJOIN, + HiveParser.TOK_FULLOUTERJOIN -> true; + default -> false; + }; } /** @@ -163,12 +159,10 @@ public static List validateColumnNameUniqueness( // but it should not be a major bottleneck as the number of columns are // anyway not so big Iterator iterCols = fieldSchemas.iterator(); - List colNames = new ArrayList(); + List colNames = new ArrayList<>(); while (iterCols.hasNext()) { String colName = iterCols.next().getName(); - Iterator iter = colNames.iterator(); - while (iter.hasNext()) { - String oldColName = iter.next(); + for (String oldColName : colNames) { if (colName.equalsIgnoreCase(oldColName)) { throw new SemanticException(ErrorMsg.DUPLICATE_COLUMN_NAMES .getMsg(oldColName)); @@ -286,7 +280,7 @@ public static Pair containsTokenOfType(ASTNode root, Integer .. final Set tokensToMatch = new HashSet<>(Arrays.asList(tokens)); final String[] matched = {null}; - boolean check = ParseUtils.containsTokenOfType(root, new PTFUtils.Predicate() { + boolean check = ParseUtils.containsTokenOfType(root, new PTFUtils.Predicate<>() { @Override public boolean apply(ASTNode node) { if (tokensToMatch.contains(node.getType())) { @@ -302,7 +296,7 @@ public boolean apply(ASTNode node) { } public static boolean containsTokenOfType(ASTNode root, PTFUtils.Predicate predicate) { - Queue queue = new ArrayDeque(); + Queue queue = new ArrayDeque<>(); // BFS queue.add(root); @@ -535,7 +529,7 @@ public static String getKeywords(Set excludes) { if (excludes != null && excludes.contains(name)) { continue; } - if (sb.length() > 0) { + if (!sb.isEmpty()) { sb.append(","); } sb.append(name); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 915ff21bd912..e1ca15a76b5e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -12040,10 +12040,10 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { } // Hack!! - refactor once the metadata APIs with types are ready // Finally add the partitioning columns - for (FieldSchema part_col : tab.getPartCols()) { - LOG.trace("Adding partition col: " + part_col); - rwsch.put(alias, part_col.getName(), new ColumnInfo(part_col.getName(), - TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), alias, true)); + for (FieldSchema partCol : tab.getPartCols()) { + LOG.trace("Adding partition col: " + partCol); + rwsch.put(alias, partCol.getName(), new ColumnInfo(partCol.getName(), + TypeInfoFactory.getPrimitiveTypeInfo(partCol.getType()), alias, true)); } // put virtual columns into RowResolver. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java index 0dcfe72d7f5b..b5b4662a1491 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java @@ -57,7 +57,7 @@ public class PartitionDesc implements Serializable, Cloneable { private static final Interner> CLASS_INTERNER = Interners.newWeakInterner(); private TableDesc tableDesc; - private LinkedHashMap partSpec; + private Map partSpec; private Class inputFileFormatClass; private Class outputFileFormatClass; private Properties properties; @@ -73,7 +73,7 @@ public void setBaseFileName(String baseFileName) { public PartitionDesc() { } - public PartitionDesc(final TableDesc table, final LinkedHashMap partSpec) { + public PartitionDesc(final TableDesc table, final Map partSpec) { this.tableDesc = table; setPartSpec(partSpec); } @@ -138,11 +138,11 @@ public void setTableDesc(TableDesc tableDesc) { } @Explain(displayName = "partition values", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) - public LinkedHashMap getPartSpec() { + public Map getPartSpec() { return partSpec; } - public void setPartSpec(final LinkedHashMap partSpec) { + public void setPartSpec(final Map partSpec) { StringInternUtils.internValuesInMap(partSpec); this.partSpec = partSpec; } diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/CacheTag.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/CacheTag.java index 0f5d7b915168..f81f8e9ec816 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/common/io/CacheTag.java +++ b/storage-api/src/java/org/apache/hadoop/hive/common/io/CacheTag.java @@ -82,7 +82,7 @@ public static final CacheTag build(String tableName) { return new TableCacheTag(tableName); } - public static final CacheTag build(String tableName, LinkedHashMap partDescMap) { + public static final CacheTag build(String tableName, Map partDescMap) { if (StringUtils.isEmpty(tableName) || partDescMap == null || partDescMap.isEmpty()) { throw new IllegalArgumentException(); } From f64adc691a9d5b5dc6ec6a38affa087e85355aff Mon Sep 17 00:00:00 2001 From: Ramit Gupta Date: Sun, 26 Apr 2026 15:07:04 +0530 Subject: [PATCH 08/10] dummy --- .../mapreduce/TestHCatMultiOutputFormat.java | 2 +- .../create/like/CreateTableLikeOperation.java | 2 +- .../formatter/TextDescTableFormatter.java | 2 +- .../JsonShowTableStatusFormatter.java | 2 +- .../TextShowTableStatusFormatter.java | 2 +- .../ddl/table/partition/PartitionUtils.java | 2 +- .../AlterTableExchangePartitionAnalyzer.java | 4 +-- .../partition/show/ShowPartitionAnalyzer.java | 6 ++--- .../archive/AlterTableArchiveOperation.java | 2 +- .../archive/AlterTableArchiveUtils.java | 2 +- .../archive/AlterTableUnarchiveOperation.java | 2 +- .../create/AbstractCreateViewAnalyzer.java | 2 +- .../hadoop/hive/ql/exec/ArchiveUtils.java | 6 ++--- .../hadoop/hive/ql/exec/DDLPlanUtils.java | 4 +-- .../apache/hadoop/hive/ql/exec/MoveTask.java | 2 +- .../apache/hadoop/hive/ql/exec/Utilities.java | 2 +- .../hive/ql/exec/repl/ReplLoadTask.java | 2 +- .../hive/ql/metadata/DummyPartition.java | 2 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 16 ++++++------ .../HiveMaterializedViewsRegistry.java | 2 +- .../hadoop/hive/ql/metadata/Partition.java | 8 +++--- .../apache/hadoop/hive/ql/metadata/Table.java | 25 +++++++++++++------ .../hive/ql/optimizer/GenMapRedUtils.java | 2 +- .../ql/optimizer/ppr/PartExprEvalUtils.java | 2 +- .../ql/optimizer/ppr/PartitionPruner.java | 2 +- .../ql/parse/AcidExportSemanticAnalyzer.java | 2 +- .../hive/ql/parse/BaseSemanticAnalyzer.java | 2 +- .../ql/parse/ColumnStatsSemanticAnalyzer.java | 4 +-- .../hive/ql/parse/ImportSemanticAnalyzer.java | 2 +- .../hive/ql/parse/MergeSemanticAnalyzer.java | 2 +- .../hadoop/hive/ql/parse/ParseUtils.java | 4 +-- .../hive/ql/parse/SemanticAnalyzer.java | 4 +-- .../rewrite/sql/MultiInsertSqlGenerator.java | 2 +- .../NativeAcidMultiInsertSqlGenerator.java | 6 ++--- .../hive/ql/stats/ColStatsProcessor.java | 2 +- 35 files changed, 73 insertions(+), 62 deletions(-) diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java index 05e5495d2656..d87158b23fae 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java @@ -374,7 +374,7 @@ private List getTableData(String table, String database) throws Exceptio Hive hive = Hive.get(conf); org.apache.hadoop.hive.ql.metadata.Table tbl = hive.getTable(database, table); FetchWork work; - if (!tbl.getEffectivePartCols().isEmpty()) { + if (!tbl.getPartCols().isEmpty()) { List partitions = hive.getPartitions(tbl); List partDesc = new ArrayList(); List partLocs = new ArrayList(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java index e10d4bdb00ce..770724b90abf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java @@ -100,7 +100,7 @@ private Table createViewLikeTable(Table oldTable) throws HiveException { setUserSpecifiedLocation(table); table.setFields(oldTable.getCols()); - table.setPartCols(oldTable.getEffectivePartCols()); + table.setPartCols(oldTable.getPartCols()); if (desc.getDefaultSerdeProps() != null) { for (Map.Entry e : desc.getDefaultSerdeProps().entrySet()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java index 651b40c6f37d..dc6c3680ca1c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java @@ -169,7 +169,7 @@ private void addPartitionData(DataOutputStream out, HiveConf conf, String column boolean isFormatted, boolean isOutputPadded) throws IOException { String partitionData = ""; if (columnPath == null) { - List partitionColumns = table.isPartitioned() ? table.getEffectivePartCols() : null; + List partitionColumns = table.isPartitioned() ? table.getPartCols() : null; if (CollectionUtils.isNotEmpty(partitionColumns) && conf.getBoolVar(ConfVars.HIVE_DISPLAY_PARTITION_COLUMNS_SEPARATELY)) { TextMetaDataTable metaDataTable = new TextMetaDataTable(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/JsonShowTableStatusFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/JsonShowTableStatusFormatter.java index 405417916990..073db26e756c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/JsonShowTableStatusFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/JsonShowTableStatusFormatter.java @@ -69,7 +69,7 @@ private Map makeOneTableStatus(Table table, Hive db, HiveConf co builder.put("partitioned", table.isPartitioned()); if (table.isPartitioned()) { - builder.put("partitionColumns", JsonDescTableFormatter.createColumnsInfo(table.getEffectivePartCols(), + builder.put("partitionColumns", JsonDescTableFormatter.createColumnsInfo(table.getPartCols(), Collections.emptyList())); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/TextShowTableStatusFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/TextShowTableStatusFormatter.java index 33205bebcbea..552dc310465b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/TextShowTableStatusFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/status/formatter/TextShowTableStatusFormatter.java @@ -73,7 +73,7 @@ private void writeStorageInfo(DataOutputStream out, Partition partition, Table t private void writeColumnsInfo(DataOutputStream out, Table table) throws IOException, UnsupportedEncodingException { String columns = MetaStoreUtils.getDDLFromFieldSchema("columns", table.getCols()); String partitionColumns = table.isPartitioned() ? - MetaStoreUtils.getDDLFromFieldSchema("partition_columns", table.getEffectivePartCols()) : ""; + MetaStoreUtils.getDDLFromFieldSchema("partition_columns", table.getPartCols()) : ""; out.write(Utilities.newLineCode); out.write(("columns:" + columns).getBytes(StandardCharsets.UTF_8)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java index 0011b25df358..5882e4616506 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java @@ -149,7 +149,7 @@ public static List getPartitionsWithSpecs(Hive db, Table table, GetPa } private static String tablePartitionColNames(Table table) { - List partCols = table.getEffectivePartCols(); + List partCols = table.getPartCols(); return String.join("/", partCols.toString()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionAnalyzer.java index 91c8da2f74ba..6485627c7e6e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/exchange/AlterTableExchangePartitionAnalyzer.java @@ -84,7 +84,7 @@ protected void analyzeCommand(TableName tableName, Map partition if (AcidUtils.isTransactionalTable(sourceTable) || AcidUtils.isTransactionalTable(destTable)) { throw new SemanticException(ErrorMsg.EXCHANGE_PARTITION_NOT_ALLOWED_WITH_TRANSACTIONAL_TABLES.getMsg()); } - List sourceProjectFilters = MetaStoreUtils.getPvals(sourceTable.getEffectivePartCols(), partitionSpecs); + List sourceProjectFilters = MetaStoreUtils.getPvals(sourceTable.getPartCols(), partitionSpecs); // check if source partition exists GetPartitionsFilterSpec sourcePartitionsFilterSpec = new GetPartitionsFilterSpec(); @@ -106,7 +106,7 @@ protected void analyzeCommand(TableName tableName, Map partition throw new SemanticException(ErrorMsg.PARTITION_VALUE_NOT_CONTINUOUS.getMsg(partitionSpecs.toString())); } - List destProjectFilters = MetaStoreUtils.getPvals(destTable.getEffectivePartCols(), partitionSpecs); + List destProjectFilters = MetaStoreUtils.getPvals(destTable.getPartCols(), partitionSpecs); // check if dest partition exists GetPartitionsFilterSpec getDestPartitionsFilterSpec = new GetPartitionsFilterSpec(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java index 4a7aea0490f3..c0bffcebdb23 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java @@ -102,7 +102,7 @@ ExprNodeDesc getShowPartitionsFilter(Table table, ASTNode command) throws Semant if (astChild.getType() == HiveParser.TOK_WHERE) { RowResolver rwsch = new RowResolver(); Map colTypes = new HashMap(); - for (FieldSchema fs : table.getEffectivePartCols()) { + for (FieldSchema fs : table.getPartCols()) { rwsch.put(table.getTableName(), fs.getName(), new ColumnInfo(fs.getName(), TypeInfoFactory.stringTypeInfo, null, true)); colTypes.put(fs.getName().toLowerCase(), fs.getType()); @@ -202,8 +202,8 @@ private String getShowPartitionsOrder(Table table, ASTNode command) throws Seman if (astChild.getType() == HiveParser.TOK_ORDERBY) { Map poses = new HashMap(); RowResolver rwsch = new RowResolver(); - for (int i = 0; i < table.getEffectivePartCols().size(); i++) { - FieldSchema fs = table.getEffectivePartCols().get(i); + for (int i = 0; i < table.getPartCols().size(); i++) { + FieldSchema fs = table.getPartCols().get(i); rwsch.put(table.getTableName(), fs.getName(), new ColumnInfo(fs.getName(), TypeInfoFactory.getPrimitiveTypeInfo(fs.getType()), null, true)); poses.put(fs.getName().toLowerCase(), i); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/archive/AlterTableArchiveOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/archive/AlterTableArchiveOperation.java index 67f5b7946760..e218e590a24e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/archive/AlterTableArchiveOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/archive/AlterTableArchiveOperation.java @@ -129,7 +129,7 @@ private Path getOriginalDir(Table table, PartSpecInfo partitionSpecInfo, List partitionColumns) throws SemanticException { - if (oldView.getEffectivePartCols().isEmpty() || oldView.getEffectivePartCols().equals(partitionColumns)) { + if (oldView.getPartCols().isEmpty() || oldView.getPartCols().equals(partitionColumns)) { return; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java index df2a92202bcc..e333ed85f439 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java @@ -73,7 +73,7 @@ static public PartSpecInfo create(Table tbl, Map partSpec) // ARCHIVE PARTITION(hr='13') won't List prefixFields = new ArrayList(); List prefixValues = new ArrayList(); - List partCols = tbl.getEffectivePartCols(); + List partCols = tbl.getPartCols(); Iterator itrPsKeys = partSpec.keySet().iterator(); for (FieldSchema fs : partCols) { if (!itrPsKeys.hasNext()) { @@ -221,7 +221,7 @@ public static int getArchivingLevel(Partition p) throws HiveException { * @throws HiveException */ public static String getPartialName(Partition p, int level) throws HiveException { - List fields = p.getTable().getEffectivePartCols().subList(0, level); + List fields = p.getTable().getPartCols().subList(0, level); List values = p.getValues().subList(0, level); try { return Warehouse.makePartName(fields, values); @@ -272,7 +272,7 @@ public static String conflictingArchiveNameOrNull(Hive db, Table tbl, Map spec = new HashMap(partSpec); List reversedKeys = new ArrayList(); - for (FieldSchema fs : tbl.getEffectivePartCols()) { + for (FieldSchema fs : tbl.getPartCols()) { if (spec.containsKey(fs.getName())) { reversedKeys.add(fs.getName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java index 3fe753de5e08..9196a3441200 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java @@ -293,7 +293,7 @@ public String getPartitionActualName(Partition pt) { */ private Map getPartitionColumnToPrimitiveCategory(Partition pt) { Map resultMap = new HashMap<>(); - for (FieldSchema schema: pt.getTable().getEffectivePartCols()) { + for (FieldSchema schema: pt.getTable().getPartCols()) { resultMap.put( schema.getName(), ((PrimitiveTypeInfo) TypeInfoUtils.getTypeInfoFromTypeString(schema.getType())).getPrimitiveCategory() @@ -975,7 +975,7 @@ private String getComment(Table table) { } private String getPartitionsForView(Table table) { - List partitionKeys = table.getEffectivePartCols(); + List partitionKeys = table.getPartCols(); if (partitionKeys.isEmpty()) { return ""; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java index a5ec3a023092..3eca5531f127 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java @@ -633,7 +633,7 @@ public void logMessage(LoadTableDesc tbd) { private DataContainer handleStaticParts(Hive db, Table table, LoadTableDesc tbd, TaskInformation ti) throws HiveException, IOException, InvalidOperationException { - List partVals = MetaStoreUtils.getPvals(table.getEffectivePartCols(), tbd.getPartitionSpec()); + List partVals = MetaStoreUtils.getPvals(table.getPartCols(), tbd.getPartitionSpec()); db.validatePartitionNameCharacters(partVals); if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("loadPartition called from " + tbd.getSourcePath() diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 9cd67fcb9619..a29e532b113b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -4321,7 +4321,7 @@ public static void setPartitionColumnNames(Configuration conf, TableScanOperator if (metadata == null) { return; } - List partCols = metadata.getEffectivePartCols(); + List partCols = metadata.getPartCols(); if (partCols != null && !partCols.isEmpty()) { conf.set(serdeConstants.LIST_PARTITION_COLUMNS, MetaStoreUtils.getColumnNamesFromFieldSchema(partCols)); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java index a7829d7b78bf..9287fd75e766 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java @@ -548,7 +548,7 @@ public static Task createViewTask(MetaData metaData, String dbNameToLoadIn, H } CreateViewDesc desc = new CreateViewDesc(dbDotView, table.getCols(), null, table.getParameters(), - table.getPartColNames(), false, false, viewOriginalText, viewExpandedText, table.getEffectivePartCols()); + table.getPartColNames(), false, false, viewOriginalText, viewExpandedText, table.getPartCols()); desc.setReplicationSpec(metaData.getReplicationSpec()); desc.setOwnerName(table.getOwner()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java index 782b4f6e5258..9f871d05feb3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java @@ -90,7 +90,7 @@ public List getValues() { Table table = this.getTable(); values = new ArrayList<>(); - for (FieldSchema fs : table.getEffectivePartCols()) { + for (FieldSchema fs : table.getPartCols()) { String val = partSpec.get(fs.getName()); values.add(val); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 0c583264816b..cd33896807bc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -869,7 +869,7 @@ public void createTable(String tableName, List columns, List par FieldSchema part = new FieldSchema(); part.setName(partCol); part.setType(STRING_TYPE_NAME); // default partition key - tbl.getEffectivePartCols().add(part); + tbl.getPartCols().add(part); } } tbl.setSerializationLib(LazySimpleSerDe.class.getName()); @@ -1246,8 +1246,8 @@ public void renamePartition(Table tbl, Map oldPartSpec, Partitio throws HiveException { try { Map newPartSpec = newPart.getSpec(); - if (oldPartSpec.keySet().size() != tbl.getEffectivePartCols().size() - || newPartSpec.keySet().size() != tbl.getEffectivePartCols().size()) { + if (oldPartSpec.keySet().size() != tbl.getPartCols().size() + || newPartSpec.keySet().size() != tbl.getPartCols().size()) { throw new HiveException("Unable to rename partition to the same name: number of partition cols don't match. "); } if (!oldPartSpec.keySet().equals(newPartSpec.keySet())){ @@ -1255,7 +1255,7 @@ public void renamePartition(Table tbl, Map oldPartSpec, Partitio } List pvals = new ArrayList(); - for (FieldSchema field : tbl.getEffectivePartCols()) { + for (FieldSchema field : tbl.getPartCols()) { String val = oldPartSpec.get(field.getName()); if (val == null || val.length() == 0) { throw new HiveException("get partition: Value for key " @@ -3832,7 +3832,7 @@ public Partition getPartition(Table tbl, Map partSpec, boolean forceCreate, String partPath, boolean inheritTableSpecs) throws HiveException { tbl.validatePartColumnNames(partSpec, true); List pvals = new ArrayList(); - for (FieldSchema field : tbl.getEffectivePartCols()) { + for (FieldSchema field : tbl.getPartCols()) { String val = partSpec.get(field.getName()); // enable dynamic partitioning if ((val == null && !HiveConf.getBoolVar(conf, HiveConf.ConfVars.DYNAMIC_PARTITIONING)) @@ -4221,7 +4221,7 @@ public List getPartitionNames(Table tbl, Map partSpec, s if (tbl.hasNonNativePartitionSupport()) { return tbl.getStorageHandler().getPartitionNames(tbl, partSpec); } - List pvals = MetaStoreUtils.getPvals(tbl.getEffectivePartCols(), partSpec); + List pvals = MetaStoreUtils.getPvals(tbl.getPartCols(), partSpec); return getPartitionNamesByPartitionVals(tbl, pvals, max); } @@ -4463,7 +4463,7 @@ private List getPartitionsWithAuth(Table tbl, Map par throw new HiveException(ErrorMsg.TABLE_NOT_PARTITIONED, tbl.getTableName()); } - List partialPvals = MetaStoreUtils.getPvals(tbl.getEffectivePartCols(), partialPartSpec); + List partialPvals = MetaStoreUtils.getPvals(tbl.getPartCols(), partialPartSpec); List partitions = null; try { @@ -4772,7 +4772,7 @@ static List convertFromPartSpec(Iterator iterator, Tab || partitionWithoutSD.getRelativePath().isEmpty()) { if (tbl.getDataLocation() != null) { Path partPath = new Path(tbl.getDataLocation(), - Warehouse.makePartName(tbl.getEffectivePartCols(), + Warehouse.makePartName(tbl.getPartCols(), partitionWithoutSD.getValues())); partitionLocation = partPath.toString(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java index 2453a0c64358..ee55fde100f6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java @@ -429,7 +429,7 @@ private static RelNode createMaterializedViewScan(HiveConf conf, Table viewTable // 1.2 Add column info corresponding to partition columns ArrayList partitionColumns = new ArrayList(); - for (FieldSchema partCol : viewTable.getEffectivePartCols()) { + for (FieldSchema partCol : viewTable.getPartCols()) { colName = partCol.getName(); colInfo = new ColumnInfo(colName, TypeInfoFactory.getPrimitiveTypeInfo(partCol.getType()), null, true); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java index 2e24dcd0087b..4715775d3b4c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java @@ -124,7 +124,7 @@ public Partition(Table tbl, Map partSpec, Path location) throws public static org.apache.hadoop.hive.metastore.api.Partition createMetaPartitionObject( Table tbl, Map partSpec, Path location) throws HiveException { List pvals = new ArrayList<>(); - for (FieldSchema field : tbl.getEffectivePartCols()) { + for (FieldSchema field : tbl.getPartCols()) { String val = partSpec.get(field.getName()); if (val == null || val.isEmpty()) { throw new HiveException("partition spec is invalid; field " @@ -173,7 +173,7 @@ protected void initialize(Table table, // table partition (not a view partition) if (table.getDataLocation() != null) { Path partPath = new Path(table.getDataLocation(), - Warehouse.makePartName(table.getEffectivePartCols(), tPartition.getValues())); + Warehouse.makePartName(table.getPartCols(), tPartition.getValues())); tPartition.getSd().setLocation(partPath.toString()); } } @@ -200,7 +200,7 @@ protected void initialize(Table table, public String getName() { try { - return Warehouse.makePartName(table.getEffectivePartCols(), tPartition.getValues()); + return Warehouse.makePartName(table.getPartCols(), tPartition.getValues()); } catch (MetaException e) { throw new RuntimeException(e); } @@ -543,7 +543,7 @@ public void setLocation(String location) { public void setValues(Map partSpec) throws HiveException { List pvals = new ArrayList<>(); - for (FieldSchema field : table.getEffectivePartCols()) { + for (FieldSchema field : table.getPartCols()) { String val = partSpec.get(field.getName()); if (val == null) { throw new HiveException( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index 8587797162b7..3e076e3a8039 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -32,6 +32,7 @@ import java.util.Objects; import java.util.Properties; import java.util.Set; +import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; @@ -601,7 +602,7 @@ public boolean equals(Object obj) { && Objects.equals(snapshotRef, other.snapshotRef); } - public List getPartCols() { + private List getNativePartCols() { List partKeys = tTable.getPartitionKeys(); if (partKeys == null) { partKeys = new ArrayList<>(); @@ -614,14 +615,14 @@ public List getPartCols() { * Returns partition columns, consulting the storage handler for non-native tables (e.g. Iceberg) * where partition columns are not stored in the metastore. */ - public List getEffectivePartCols() { + public List getPartCols() { if (cachedPartCols != null) { return cachedPartCols; } if (isTableTypeSet() && hasNonNativePartitionSupport()) { cachedPartCols = getStorageHandler().getPartitionKeys(this); } else { - cachedPartCols = getPartCols(); + cachedPartCols = getNativePartCols(); } return cachedPartCols; } @@ -641,7 +642,7 @@ public FieldSchema getPartColByName(String colName) { } public List getPartColNames() { - return getEffectivePartCols().stream().map(FieldSchema::getName).toList(); + return getPartCols().stream().map(FieldSchema::getName).toList(); } public boolean hasNonNativePartitionSupport() { @@ -756,7 +757,17 @@ private boolean isField(String col) { } public List getCols() { - return getColsInternal(false); + if (!isNonNative()) { + return getColsInternal(false); + } + List nonPartFields = new ArrayList<>(); + Set partFieldsName = getPartCols().stream().map(FieldSchema::getName).collect(Collectors.toSet()); + for (FieldSchema field : getColsInternal(false)) { + if (!partFieldsName.contains(field.getName())) { + nonPartFields.add(field); + } + } + return nonPartFields; } public List getColsForMetastore() { @@ -840,7 +851,7 @@ public void setOutputFormatClass(String name) throws HiveException { public boolean isPartitioned() { return hasNonNativePartitionSupport() ? getStorageHandler().isPartitioned(this) : - CollectionUtils.isNotEmpty(getEffectivePartCols()); + CollectionUtils.isNotEmpty(getPartCols()); } public void setFields(List fields) { @@ -1038,7 +1049,7 @@ public boolean isMaterializedView() { public Map createSpec( org.apache.hadoop.hive.metastore.api.Partition tp) { - List fsl = getEffectivePartCols(); + List fsl = getPartCols(); List tpl = tp.getValues(); Map spec = LinkedHashMap.newLinkedHashMap(fsl.size()); for (int i = 0; i < fsl.size(); i++) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index c560783f070f..bd1be003a512 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -2149,7 +2149,7 @@ static void usePartitionColumns(Properties properties, Table table, List if (properties.containsKey(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS)) { usePartitionColumns(properties, partColNames); } else { - List partCols = table.getEffectivePartCols(); + List partCols = table.getPartCols(); String partNames = partCols.stream().map(FieldSchema::getName).collect(Collectors.joining("/")); String partTypes = partCols.stream().map(FieldSchema::getType).collect(Collectors.joining(":")); properties.setProperty( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java index dd4478e0acc9..d08fe92208ad 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java @@ -58,7 +58,7 @@ static public Object evalExprWithPart(ExprNodeDesc expr, Partition p) throws Hiv if (!partSpec.keySet().containsAll(expr.getCols())) { return null; } - partKeyTypes = p.getTable().getEffectivePartCols().stream().map(FieldSchema::getType) + partKeyTypes = p.getTable().getPartCols().stream().map(FieldSchema::getType) .toArray(String[]::new); } else { String pcolTypes = partProps.getProperty(hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java index 428f1f8cf537..b1bc9eaf0a75 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java @@ -525,7 +525,7 @@ static private boolean pruneBySequentialScan(Table tab, List partitio } private static List extractPartColTypes(Table tab) { - List pCols = tab.getEffectivePartCols(); + List pCols = tab.getPartCols(); List partColTypeInfos = new ArrayList<>(pCols.size()); for (FieldSchema pCol : pCols) { partColTypeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(pCol.getType())); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java index c42f39fa6309..05f3b85f271f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java @@ -175,7 +175,7 @@ private void analyzeAcidExport(ASTNode ast, Table exportTable, ASTNode tokRefOrN //now generate insert statement //insert into newTableName select * from ts StringBuilder rewrittenQueryStr = generateExportQuery( - newTable.getEffectivePartCols(), + newTable.getPartCols(), tokRefOrNameExportTable, (ASTNode) tokRefOrNameExportTable.parent, newTableName); ReparseResult rr = ParseUtils.parseRewrittenQuery(ctx, rewrittenQueryStr); Context rewrittenCtx = rr.rewrittenCtx; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 211462775911..f616049d8591 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -1202,7 +1202,7 @@ public TableSpec(Table tableHandle, List partitions) { if (partitions != null && !partitions.isEmpty()) { this.specType = SpecType.STATIC_PARTITION; this.partitions = partitions; - List partCols = this.tableHandle.getEffectivePartCols(); + List partCols = this.tableHandle.getPartCols(); this.partSpec = new LinkedHashMap<>(); for (FieldSchema partCol : partCols) { partSpec.put(partCol.getName(), null); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java index 5c9f898bea13..9bf6269334e7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java @@ -207,7 +207,7 @@ private static CharSequence genPartitionClause(Table tbl, List pa private static String getColTypeOf(Table tbl, String partKey) { - for (FieldSchema fs : tbl.getEffectivePartCols()) { + for (FieldSchema fs : tbl.getPartCols()) { if (partKey.equalsIgnoreCase(fs.getName())) { return fs.getType().toLowerCase(); } @@ -309,7 +309,7 @@ private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, if (isPartitionStats) { if (partTransformSpec == null) { - for (FieldSchema fs : tbl.getEffectivePartCols()) { + for (FieldSchema fs : tbl.getPartCols()) { String identifier = unparseIdentifier(fs.getName(), conf); rewrittenQueryBuilder.append(", ").append(identifier); columnNamesBuilder.append(", ").append(identifier); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java index e671ad24d7b9..dcf197a2c201 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java @@ -876,7 +876,7 @@ private static void checkTable(Table table, ImportTableDesc tableDesc, } { // check partitioning column order and types - List existingTablePartCols = table.getEffectivePartCols(); + List existingTablePartCols = table.getPartCols(); List importedTablePartCols = tableDesc.getPartCols(); if (!EximUtil.schemaCompare(importedTablePartCols, existingTablePartCols)) { throw new SemanticException( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java index 6f32b0a9293a..ddde37433b5a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java @@ -431,7 +431,7 @@ private static final class OnClauseAnalyzer { HiveConf conf, String onClauseAsString) { this.onClause = onClause; allTargetTableColumns.addAll(targetTable.getCols()); - allTargetTableColumns.addAll(targetTable.getEffectivePartCols()); + allTargetTableColumns.addAll(targetTable.getPartCols()); this.targetTableNameInSourceQuery = unescapeIdentifier(targetTableNameInSourceQuery); this.conf = conf; this.onClauseAsString = onClauseAsString; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java index 0904e2c9b5c5..9edbe5b05e13 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java @@ -575,7 +575,7 @@ public static Map> getFullPartitionSpecs( CommonTree ast, Table table, Configuration conf, boolean canGroupExprs) throws SemanticException { String defaultPartitionName = HiveConf.getVar(conf, HiveConf.ConfVars.DEFAULT_PARTITION_NAME); Map colTypes = new HashMap<>(); - List partitionKeys = table.getEffectivePartCols(); + List partitionKeys = table.getPartCols(); for (FieldSchema fs : partitionKeys) { colTypes.put(fs.getName().toLowerCase(), fs.getType()); } @@ -685,7 +685,7 @@ public static Map> getFullPartitionSpecs( */ private static int calculatePartPrefix(Table tbl, Set partSpecKeys) { int partPrefixToDrop = 0; - for (FieldSchema fs : tbl.getEffectivePartCols()) { + for (FieldSchema fs : tbl.getPartCols()) { if (!partSpecKeys.contains(fs.getName())) { break; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index e1ca15a76b5e..50d02acf2464 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -2210,7 +2210,7 @@ private void handleInsertStatementSpecPhase1(ASTNode ast, QBParseInfo qbp, Phase } } else { // partition spec is not specified but column schema can have partitions specified - for(FieldSchema f : targetTable.getEffectivePartCols()) { + for(FieldSchema f : targetTable.getPartCols()) { //parser only allows foo(a,b), not foo(foo.a, foo.b) targetColumns.remove(f.getName()); } @@ -12306,7 +12306,7 @@ private void setupStats(TableScanDesc tsDesc, QBParseInfo qbp, Table tab, String if (tab.isPartitioned() && !tab.hasNonNativePartitionSupport()) { List cols = new ArrayList(); if (qbp.getAnalyzeRewrite() != null) { - List partitionCols = tab.getEffectivePartCols(); + List partitionCols = tab.getPartCols(); for (FieldSchema fs : partitionCols) { cols.add(fs.getName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java index 928564012985..7587daf13055 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java @@ -152,7 +152,7 @@ public void appendPartColsOfTargetTableWithComma(String alias) { return; } queryStr.append(','); - appendCols(targetTable.getEffectivePartCols(), alias, null, FieldSchema::getName); + appendCols(targetTable.getPartCols(), alias, null, FieldSchema::getName); } public void appendAllColsOfTargetTable(String prefix) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java index 94cda746396b..87e426800442 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java @@ -36,7 +36,7 @@ public NativeAcidMultiInsertSqlGenerator(Table table, String targetTableFullName @Override public void appendAcidSelectColumns(Operation operation) { queryStr.append("ROW__ID,"); - for (FieldSchema fieldSchema : targetTable.getEffectivePartCols()) { + for (FieldSchema fieldSchema : targetTable.getPartCols()) { String identifier = HiveUtils.unparseIdentifier(fieldSchema.getName(), this.conf); queryStr.append(identifier); queryStr.append(","); @@ -45,9 +45,9 @@ public void appendAcidSelectColumns(Operation operation) { @Override public List getDeleteValues(Operation operation) { - List deleteValues = new ArrayList<>(1 + targetTable.getEffectivePartCols().size()); + List deleteValues = new ArrayList<>(1 + targetTable.getPartCols().size()); deleteValues.add(qualify("ROW__ID")); - for (FieldSchema fieldSchema : targetTable.getEffectivePartCols()) { + for (FieldSchema fieldSchema : targetTable.getPartCols()) { deleteValues.add(qualify(HiveUtils.unparseIdentifier(fieldSchema.getName(), conf))); } return deleteValues; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java index cb257ee216f2..be62d94019ed 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java @@ -156,7 +156,7 @@ private boolean constructColumnStatsFromPackedRows(Table tbl, List Date: Wed, 20 May 2026 17:04:48 +0530 Subject: [PATCH 09/10] commit # Conflicts: # ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java --- ...rg_insert_overwrite_partition_transforms.q | 25 ----- .../positive/describe_iceberg_table.q.out | 15 --- .../apache/hadoop/hive/ql/metadata/Table.java | 94 +++++++++++++------ .../hadoop/hive/ql/parse/CalcitePlanner.java | 19 ++-- .../parse/ColumnStatsAutoGatherContext.java | 23 +++-- .../ql/parse/ColumnStatsSemanticAnalyzer.java | 41 ++++---- .../hive/ql/parse/SemanticAnalyzer.java | 10 +- .../hive/ql/stats/ColStatsProcessor.java | 18 ++++ 8 files changed, 138 insertions(+), 107 deletions(-) diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_insert_overwrite_partition_transforms.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_insert_overwrite_partition_transforms.q index 3c968a163ac6..e838c7bd5693 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_insert_overwrite_partition_transforms.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_insert_overwrite_partition_transforms.q @@ -19,30 +19,6 @@ --! qt:replace:/(\S\"iceberg-version\\\":\\\")(\w+\s\w+\s\d+\.\d+\.\d+\s\(\w+\s\w+\))(\\\")/$1#Masked#$3/ set hive.explain.user=false; -create external table ice_parquet_date_transform_year( - bigintcol bigint, - intcol integer, - pcol date -) partitioned by spec (year(pcol)) -stored by iceberg; - -explain insert overwrite table ice_parquet_date_transform_year partition (pcol = '1999-12-05') values (1234567890123345, 2), (23456789012345678, 4); -insert overwrite table ice_parquet_date_transform_year partition (pcol = '1999-12-05') values (1234567890123345, 2), (23456789012345678, 4); -explain insert overwrite table ice_parquet_date_transform_year partition (pcol = '1999-12-26') values (1234567890123345, 3), (23456789012345678, 5); -insert overwrite table ice_parquet_date_transform_year partition (pcol = '1999-12-26') values (1234567890123345, 3), (23456789012345678, 5); -explain insert overwrite table ice_parquet_date_transform_year partition (pcol = '1999-12-12') values (3456789012345678, 4), (34567890123456789, 6); -insert overwrite table ice_parquet_date_transform_year partition (pcol = '1999-12-12') values (3456789012345678, 4), (34567890123456789, 6); - -select * from ice_parquet_date_transform_year; - -explain insert overwrite table ice_parquet_date_transform_year partition (pcol = '1999-12-13') select bigintcol, intcol from ice_parquet_date_transform_year; -insert overwrite table ice_parquet_date_transform_year partition (pcol = '1999-12-13') select bigintcol, intcol from ice_parquet_date_transform_year; -explain insert overwrite table ice_parquet_date_transform_year partition (pcol = '1999-12-02') select 234675894076895090, intcol from ice_parquet_date_transform_year; -insert overwrite table ice_parquet_date_transform_year partition (pcol = '1999-12-02') select 234675894076895090, intcol from ice_parquet_date_transform_year; - -describe formatted ice_parquet_date_transform_year; -select * from ice_parquet_date_transform_year; - create external table ice_parquet_date_transform_month( bigintcol bigint, pcol date, @@ -50,7 +26,6 @@ create external table ice_parquet_date_transform_month( ) partitioned by spec (month(pcol)) stored by iceberg; -explain insert overwrite table ice_parquet_date_transform_month partition (pcol = '1999-12-31') values (1234567890123345, 2), (23456789012345678, 4); insert overwrite table ice_parquet_date_transform_month partition (pcol = '1999-12-31') values (1234567890123345, 2), (23456789012345678, 4); explain insert overwrite table ice_parquet_date_transform_month partition (pcol = '1999-12-26') values (1234567890123345, 3), (23456789012345678, 5); insert overwrite table ice_parquet_date_transform_month partition (pcol = '1999-12-26') values (1234567890123345, 3), (23456789012345678, 5); diff --git a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out index fb1cdbcaf12a..b19a88b6eb90 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out @@ -109,13 +109,6 @@ POSTHOOK: query: DESCRIBE FORMATTED ice_t_transform POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@ice_t_transform # col_name data_type comment -year_field date -month_field date -day_field date -hour_field timestamp -truncate_field string -bucket_field int -identity_field int # Partition Information # col_name data_type comment @@ -182,13 +175,6 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@ice_t_transform_prop # col_name data_type comment id int -year_field date -month_field date -day_field date -hour_field timestamp -truncate_field string -bucket_field int -identity_field int # Partition Information # col_name data_type comment @@ -255,7 +241,6 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@ice_t_identity_part # col_name data_type comment a int -b string # Partition Information # col_name data_type comment diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index 3e076e3a8039..eb6fdc76bbaf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -21,22 +21,13 @@ import java.io.IOException; import java.io.Serial; import java.io.Serializable; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Properties; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -113,7 +104,10 @@ public class Table implements Serializable { /** * These fields are all cached fields. The information comes from tTable. */ - private List cachedPartCols; + private List tablePartCols; + private List tableNonPartCols; + private List tableAllCols; + private Map> inputColumnIndexByName; private transient Deserializer deserializer; private Class outputFormatClass; private Class inputFormatClass; @@ -198,8 +192,8 @@ public Table makeCopy() { newTab.setMetaTable(this.getMetaTable()); newTab.setSnapshotRef(this.getSnapshotRef()); - if (this.cachedPartCols != null) { - newTab.cachedPartCols = new ArrayList<>(this.cachedPartCols); + if (this.tablePartCols != null) { + newTab.tablePartCols = new ArrayList<>(this.tablePartCols); } return newTab; } @@ -616,15 +610,15 @@ private List getNativePartCols() { * where partition columns are not stored in the metastore. */ public List getPartCols() { - if (cachedPartCols != null) { - return cachedPartCols; + if (tablePartCols != null) { + return tablePartCols; } if (isTableTypeSet() && hasNonNativePartitionSupport()) { - cachedPartCols = getStorageHandler().getPartitionKeys(this); + tablePartCols = getStorageHandler().getPartitionKeys(this); } else { - cachedPartCols = getNativePartCols(); + tablePartCols = getNativePartCols(); } - return cachedPartCols; + return tablePartCols; } private boolean isTableTypeSet() { @@ -756,18 +750,49 @@ private boolean isField(String col) { return false; } - public List getCols() { + private void fillColumnIndexByName() { + inputColumnIndexByName = new HashMap<>(); + List fsList = new ArrayList<>(getColsInternal(false)); if (!isNonNative()) { - return getColsInternal(false); + fsList.addAll(getNativePartCols()); + } + for (int i = 0; i < fsList.size(); i++) { + inputColumnIndexByName.put(fsList.get(i).getName(), Pair.of(i, fsList.get(i))); + } + } + + public int getColumnIndexByName(String colName) { + if (inputColumnIndexByName == null) { + fillColumnIndexByName(); + } + return inputColumnIndexByName.get(colName.toLowerCase()).getLeft(); + } + + public FieldSchema getFieldSchemaByName(String colName) { + if (inputColumnIndexByName == null) { + fillColumnIndexByName(); + } + return inputColumnIndexByName.get(colName).getRight(); + } + + public List getCols() { + if (tableNonPartCols != null) { + return tableNonPartCols; } - List nonPartFields = new ArrayList<>(); - Set partFieldsName = getPartCols().stream().map(FieldSchema::getName).collect(Collectors.toSet()); - for (FieldSchema field : getColsInternal(false)) { - if (!partFieldsName.contains(field.getName())) { - nonPartFields.add(field); + tableNonPartCols = new ArrayList<>(); + if (!isNonNative()) { + tableNonPartCols.addAll(getColsInternal(false)); + } else { + List nonPartFields = new ArrayList<>(); + Set partFieldsName = getPartCols().stream().map(FieldSchema::getName).collect(Collectors.toSet()); + for (FieldSchema field : getColsInternal(false)) { + if (!partFieldsName.contains(field.getName())) { + nonPartFields.add(field); + } } + tableNonPartCols = nonPartFields; } - return nonPartFields; + return tableNonPartCols; } public List getColsForMetastore() { @@ -800,9 +825,18 @@ private List getColsInternal(boolean forMs) { * @return List<FieldSchema> */ public List getAllCols() { - ArrayList allCols = new ArrayList<>(getCols()); - allCols.addAll(getPartCols()); - return allCols; + if (tableAllCols != null) { + return tableAllCols; + } + if (inputColumnIndexByName == null) { + fillColumnIndexByName(); + } + TreeMap orderedMap = new TreeMap<>(); + for (Map.Entry> e : inputColumnIndexByName.entrySet()) { + orderedMap.put(e.getValue().getLeft(), e.getValue().getRight()); + } + tableAllCols = orderedMap.values().stream().toList(); + return tableAllCols; } public void setPartCols(List partCols) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 00cc988eb0a8..09eed2dc526b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -26,6 +26,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; import java.util.Map.Entry; import java.util.Optional; @@ -3013,14 +3014,16 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc ArrayList partitionColumns = new ArrayList(); // 3.2 Add column info corresponding to partition columns - for (FieldSchema part_col : tabMetaData.getPartCols()) { - colName = part_col.getName(); - colInfo = new ColumnInfo(colName, - TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), - isNullable(colName, nnc, pkc), tableAlias, true); - rr.put(tableAlias, colName, colInfo); - cInfoLst.add(colInfo); - partitionColumns.add(colInfo); + if (!tabMetaData.hasNonNativePartitionSupport()) { + for (FieldSchema part_col : tabMetaData.getPartCols()) { + colName = part_col.getName(); + colInfo = new ColumnInfo(colName, + TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), + isNullable(colName, nnc, pkc), tableAlias, true); + rr.put(tableAlias, colName, colInfo); + cInfoLst.add(colInfo); + partitionColumns.add(colInfo); + } } final TableType tableType = obtainTableType(tabMetaData); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java index 9109f9cb6086..165c580db612 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java @@ -236,7 +236,7 @@ private Operator genSelOp(String command, boolean rewritten, Context origCtx) /** * @param operator : the select operator in the analyze statement * @param input : the operator right before FS in the insert overwrite statement - * @throws HiveException + * @throws HiveException */ private void replaceSelectOperatorProcess(SelectOperator operator, Operator input) throws HiveException { @@ -244,16 +244,11 @@ private void replaceSelectOperatorProcess(SelectOperator operator, Operator signature = new ArrayList<>(); OpParseContext inputCtx = sa.opParseCtx.get(input); RowResolver inputRR = inputCtx.getRowResolver(); - List columns = inputRR.getColumnInfos(); + List inputColumns = inputRR.getColumnInfos(); List colList = new ArrayList(); List columnNames = new ArrayList(); Map columnExprMap = new HashMap(); - // the column positions in the operator should be like this - // <----non-partition columns---->|<--static partition columns-->|<--dynamic partition columns--> - // ExprNodeColumnDesc | ExprNodeConstantDesc | ExprNodeColumnDesc - // from input | generate itself | from input - // | // 1. deal with non-partition columns Map columnNameToIndex = new HashMap<>(); @@ -262,7 +257,12 @@ private void replaceSelectOperatorProcess(SelectOperator operator, Operator= inputColumns.size()) { + continue; + } + ColumnInfo col = inputColumns.get(inputIdx); Integer selRSIdx = getSelRSColumnIndex(i, col, columnNameToIndex); if (selRSIdx == null) { continue; @@ -294,7 +294,12 @@ private void replaceSelectOperatorProcess(SelectOperator operator, Operator= inputColumns.size()) { + throw new SemanticException("Unable to resolve dynamic partition column '" + partColName + + "' from input columns " + inputRR.getColumnInfos()); + } + ColumnInfo col = inputColumns.get(inputIdx); exprNodeDesc = new ExprNodeColumnDesc(col); srcType = col.getType(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java index 9bf6269334e7..1fd943cf1573 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java @@ -159,7 +159,7 @@ private void handlePartialPartitionSpec(Map partSpec, ColumnStat } // User might have only specified partial list of partition keys, in which case add other partition keys in partSpec - List partKeys = Utilities.getColumnNamesFromFieldSchema(tbl.getPartitionKeys()); + List partKeys = Utilities.getColumnNamesFromFieldSchema(tbl.getPartCols()); for (String partKey : partKeys) { if (!partSpec.containsKey(partKey)) { partSpec.put(partKey, null); @@ -174,14 +174,14 @@ private void handlePartialPartitionSpec(Map partSpec, ColumnStat } } - private static CharSequence genPartitionClause(Table tbl, List partTransformSpec, int specId, + private static CharSequence genPartitionClause(Table tbl, List partTransformSpec, int specId, Map partSpec, HiveConf conf) { boolean predPresent = partSpec.values().stream().anyMatch(Objects::nonNull); - + StringBuilder whereClause = new StringBuilder(" where ").append( partSpec.entrySet().stream() .filter(part -> part.getValue() != null) - .map(part -> unparseIdentifier(part.getKey(), conf) + " = " + .map(part -> unparseIdentifier(part.getKey(), conf) + " = " + genPartValueString(getColTypeOf(tbl, part.getKey()), part.getValue())) .collect(Collectors.joining(" and ")) ); @@ -271,7 +271,7 @@ private String genRewrittenQuery(FieldSchemas columnSchemas, HiveConf conf, * included in the input table. */ protected static String genRewrittenQuery(Table tbl, - HiveConf conf, List partTransformSpec, Map partSpec, + HiveConf conf, List partTransformSpec, Map partSpec, boolean isPartitionStats) { return ColumnStatsSemanticAnalyzer.genRewrittenQuery(tbl, getStatsEligibleFieldSchemas(tbl), conf, partTransformSpec, -1, partSpec, isPartitionStats, true); @@ -296,7 +296,7 @@ private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, final TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(columnSchema.getType()); try { - genComputeStats(rewrittenQueryBuilder, conf, i, columnName, typeInfo); + genComputeStats(rewrittenQueryBuilder, conf, tbl.getColumnIndexByName(columnSchema.getName()), columnName, typeInfo); } catch (SemanticException e) { throw new RuntimeException(e); } @@ -308,17 +308,26 @@ private static String genRewrittenQuery(Table tbl, FieldSchemas columnSchemas, } if (isPartitionStats) { - if (partTransformSpec == null) { - for (FieldSchema fs : tbl.getPartCols()) { - String identifier = unparseIdentifier(fs.getName(), conf); - rewrittenQueryBuilder.append(", ").append(identifier); - columnNamesBuilder.append(", ").append(identifier); - - columnDummyValuesBuilder.append(", cast(null as ") - .append(TypeInfoUtils.getTypeInfoFromTypeString(fs.getType()).toString()) - .append(")"); + for (FieldSchema fs : tbl.getPartCols()) { + String identifier = unparseIdentifier(fs.getName(), conf); + TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(fs.getType().toString()); + rewrittenQueryBuilder.append(", "); + if (tbl.hasNonNativePartitionSupport()) { + try { + genComputeStats(rewrittenQueryBuilder, conf, tbl.getColumnIndexByName(fs.getName()), identifier, typeInfo); + } catch (SemanticException e) { + throw new RuntimeException(e); + } + } else { + rewrittenQueryBuilder.append(identifier); } - } else { + columnNamesBuilder.append(", ").append(identifier); + + columnDummyValuesBuilder.append(", cast(null as ") + .append(typeInfo) + .append(")"); + } + if (partTransformSpec != null) { rewrittenQueryBuilder.append(", ") .append(TransformSpec.toNamedStruct(partTransformSpec, conf)); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 50d02acf2464..4823ebad9578 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -12040,10 +12040,12 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { } // Hack!! - refactor once the metadata APIs with types are ready // Finally add the partitioning columns - for (FieldSchema partCol : tab.getPartCols()) { - LOG.trace("Adding partition col: " + partCol); - rwsch.put(alias, partCol.getName(), new ColumnInfo(partCol.getName(), - TypeInfoFactory.getPrimitiveTypeInfo(partCol.getType()), alias, true)); + if(!tab.hasNonNativePartitionSupport()){ + for (FieldSchema partCol : tab.getPartCols()) { + LOG.trace("Adding partition col: " + partCol); + rwsch.put(alias, partCol.getName(), new ColumnInfo(partCol.getName(), + TypeInfoFactory.getPrimitiveTypeInfo(partCol.getType()), alias, true)); + } } // put virtual columns into RowResolver. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java index be62d94019ed..4ff462d45ceb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java @@ -145,6 +145,24 @@ private boolean constructColumnStatsFromPackedRows(Table tbl, List partVals = new ArrayList<>(); if (tbl.hasNonNativePartitionSupport()) { + for (FieldSchema partCol : tbl.getPartCols()) { + PrimitiveTypeInfo typeInfo = (PrimitiveTypeInfo) TypeInfoUtils.getTypeInfoFromTypeString(partCol.getType()); + List columnStatsFields = ColumnStatsType.getColumnStats(typeInfo); + columnStatsFields = ColumnStatsType.removeDisabledStatistics(conf, columnStatsFields); + try { + ColumnStatisticsObj statObj = ColumnStatisticsObjTranslator.readHiveColumnStatistics( + partCol.getName(), partCol.getType(), columnStatsFields, pos, fields, values); + statsObjs.add(statObj); + numStats++; + } catch (Exception e) { + if (isStatsReliable) { + throw new HiveException("Statistics collection failed while (hive.stats.reliable)", e); + } else { + LOG.debug("Because {} is infinite or NaN, we skip stats.", partCol.getName(), e); + } + } + pos += columnStatsFields.size(); + } ObjectInspector inspector = fields.get(pos).getFieldObjectInspector(); if (inspector.getCategory() == ObjectInspector.Category.STRUCT) { Object obj = values.get(pos); From 6cc371cfece2556b9729a62b4098fbcd3503c6ee Mon Sep 17 00:00:00 2001 From: Ramit Gupta Date: Wed, 20 May 2026 17:31:11 +0530 Subject: [PATCH 10/10] commit2 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java | 6 +++++- .../ql/optimizer/DynamicPartitionPruningOptimization.java | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index eb6fdc76bbaf..d4f93cdb037f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -644,7 +644,11 @@ public boolean hasNonNativePartitionSupport() { } public boolean isPartitionKey(String colName) { - return getPartColByName(colName) != null; + List partKeys = getPartitionKeys().stream().map(FieldSchema::getName).toList(); + if (partKeys.isEmpty()) { + return false; + } + return partKeys.contains(colName); } // TODO merge this with getBucketCols function diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java index dba737e382c0..e92efedc4615 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java @@ -194,7 +194,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Obje } else if (table.isNonNative() && table.getStorageHandler().addDynamicSplitPruningEdge(table, ctx.parent)) { generateEventOperatorPlan(ctx, parseContext, ts, column, - table.getCols().stream().filter(e -> e.getName().equals(column)). + table.getAllCols().stream().filter(e -> e.getName().equals(column)). map(e -> e.getType()).findFirst().get(), ctx.parent); } else { // semijoin LOG.debug("Column " + column + " is not a partition column");