Skip to content

Commit 4b54b61

Browse files
author
Hongdan Zhu
committed
HIVE-28655: Implement HMS Related Drop Stats Changes, Reset COLUMN_STAT_ACCURATE After Dropping
1 parent 063cfa3 commit 4b54b61

File tree

4 files changed

+101
-1
lines changed

4 files changed

+101
-1
lines changed

‎standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7440,6 +7440,15 @@ public boolean delete_column_statistics_req(DeleteColumnStatisticsRequest req) t
74407440
}
74417441
events.add(new DeleteTableColumnStatEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, colName, engine, this));
74427442
}
7443+
/**
7444+
if (colNames == null){
7445+
// remove all column names in parameter COLUMN_STATS_ACCURATE
7446+
StatsSetupConst.clearColumnStatsState(table.getParameters());
7447+
} else {
7448+
// remove the deleted column names in parameter COLUMN_STATS_ACCURATE
7449+
StatsSetupConst.removeColumnStatsState(table.getParameters(), colNames);
7450+
}
7451+
**/
74437452
}
74447453
} else {
74457454
List<String> partNames = new ArrayList<>();
@@ -7460,13 +7469,24 @@ public boolean delete_column_statistics_req(DeleteColumnStatisticsRequest req) t
74607469
.collect(Collectors.toList()) : colNames) {
74617470
for (String partName : partNames) {
74627471
List<String> partVals = getPartValsFromName(table, partName);
7472+
Partition partition = rawStore.getPartition(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, partVals);
7473+
Map<String, String> partParams = partition.getParameters();
74637474
if (transactionalListeners != null && !transactionalListeners.isEmpty()) {
74647475
MetaStoreListenerNotifier.notifyEvent(transactionalListeners, eventType,
74657476
new DeletePartitionColumnStatEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName,
74667477
partName, partVals, colName, engine, this));
74677478
}
74687479
events.add(new DeletePartitionColumnStatEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName,
74697480
partName, partVals, colName, engine, this));
7481+
/**
7482+
if (colNames == null) {
7483+
// remove all column names in parameter COLUMN_STATS_ACCURATE
7484+
StatsSetupConst.clearColumnStatsState(partParams);
7485+
} else {
7486+
// remove the deleted column names in parameter COLUMN_STATS_ACCURATE
7487+
StatsSetupConst.removeColumnStatsState(partParams, colNames);
7488+
}
7489+
**/
74707490
}
74717491
}
74727492
}

‎standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java‎

Lines changed: 61 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,7 @@ public static <T> String getIdListForIn(Collection<T> objectIds) throws MetaExce
183183

184184
// Table names with schema name, if necessary
185185
@TableName
186-
private String DBS, TBLS, PARTITIONS, DATABASE_PARAMS, PARTITION_PARAMS, SORT_COLS, SD_PARAMS,
186+
private String DBS, TBLS, PARTITIONS, DATABASE_PARAMS, TABLE_PARAMS, PARTITION_PARAMS, SORT_COLS, SD_PARAMS,
187187
SDS, SERDES, SKEWED_STRING_LIST_VALUES, SKEWED_VALUES, BUCKETING_COLS, SKEWED_COL_NAMES,
188188
SKEWED_COL_VALUE_LOC_MAP, COLUMNS_V2, PARTITION_KEYS, SERDE_PARAMS, PART_COL_STATS, KEY_CONSTRAINTS,
189189
TAB_COL_STATS, PARTITION_KEY_VALS, PART_PRIVS, PART_COL_PRIVS, SKEWED_STRING_LIST, CDS,
@@ -3211,6 +3211,45 @@ public void lockDbTable(String tableName) throws MetaException {
32113211
}
32123212
}
32133213

3214+
public Long getTableId(String dbName, String tableName) throws MetaException {
3215+
String queryText;
3216+
switch (dbType.dbType) {
3217+
case MYSQL:
3218+
queryText = ""
3219+
+ "SELECT t.TBL_ID FROM " + TBLS + " t "
3220+
+ " JOIN " + DBS + " d ON t.DB_ID = d.DB_ID "
3221+
+ " WHERE d.NAME = ? AND t.TBL_NAME = ?";
3222+
break;
3223+
case DERBY:
3224+
case POSTGRES:
3225+
default:
3226+
queryText = ""
3227+
+ "SELECT t.\"TBL_ID\" FROM " + TBLS + " t "
3228+
+ " JOIN " + DBS + " d ON t.\"DB_ID\" = d.\"DB_ID\" "
3229+
+ " WHERE d.\"NAME\" = ? AND t.\"TBL_NAME\" = ?";
3230+
break;
3231+
}
3232+
3233+
List<String> params = Arrays.asList(dbName, tableName);
3234+
3235+
try (QueryWrapper queryWrapper = new QueryWrapper(pm.newQuery("javax.jdo.query.SQL", queryText))) {
3236+
List<Object> results = executeWithArray(queryWrapper.getInnerQuery(), params.toArray(), queryText, 1);
3237+
3238+
if (results.isEmpty()) {
3239+
return null;
3240+
}
3241+
3242+
Object row = results.get(0);
3243+
if (row instanceof Number) {
3244+
return ((Number) row).longValue();
3245+
} else if (row instanceof Object[]) {
3246+
return ((Number) ((Object[]) row)[0]).longValue();
3247+
} else {
3248+
throw new MetaException("Unexpected result type for table ID: " + row.getClass());
3249+
}
3250+
}
3251+
}
3252+
32143253
public void deleteColumnStatsState(long tbl_id) throws MetaException {
32153254
String queryText;
32163255
switch (dbType.dbType) {
@@ -3235,13 +3274,34 @@ public void deleteColumnStatsState(long tbl_id) throws MetaException {
32353274
// @formatter:on
32363275
}
32373276

3277+
executeWithArray(query.getInnerQuery(), null, queryText);
32383278
try {
32393279
executeNoResult(queryText);
32403280
} catch (SQLException e) {
32413281
throw new MetaException("Error removing column stat states:" + e.getMessage());
32423282
}
32433283
}
32443284

3285+
public void deleteTblColumnStatsState(long tbl_id) throws MetaException {
3286+
String queryText;
3287+
switch (dbType.dbType) {
3288+
// delete the table parameter COLUMN_STATS_ACCURATE
3289+
default:
3290+
// @formatter:off
3291+
queryText = ""
3292+
+ "DELETE FROM " + TABLE_PARAMS
3293+
+ " WHERE \"TBL_ID\" = " + tbl_id
3294+
+ " AND \"PARAM_KEY\" = '" + StatsSetupConst.COLUMN_STATS_ACCURATE + "'";
3295+
// @formatter:on
3296+
}
3297+
3298+
try {
3299+
executeNoResult(queryText);
3300+
} catch (SQLException e) {
3301+
throw new MetaException("Error removing column stats states: " + e.getMessage());
3302+
}
3303+
}
3304+
32453305
public boolean deleteTableColumnStatistics(long tableId, List<String> colNames, String engine) {
32463306
String deleteSql = "delete from " + TAB_COL_STATS + " where \"TBL_ID\" = " + tableId;
32473307
if (colNames != null && !colNames.isEmpty()) {

‎standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10278,6 +10278,8 @@ public List<Void> run(List<String> input) throws Exception {
1027810278
List<MPartitionColumnStatistics> mStatsObjColl =
1027910279
(List<MPartitionColumnStatistics>) query.executeWithArray(params.toArray());
1028010280
pm.retrieveAll(mStatsObjColl);
10281+
Long tableID = directSql.getTableId(dbName, tableName);
10282+
directSql.deleteColumnStatsState(tableID);
1028110283
if (mStatsObjColl != null) {
1028210284
pm.deletePersistentAll(mStatsObjColl);
1028310285
} else {
@@ -10363,6 +10365,8 @@ private boolean deleteTableColumnStatisticsViaJdo(String catName, String dbName,
1036310365
}
1036410366
mStatsObjColl = (List<MTableColumnStatistics>) query.executeWithArray(params.toArray());
1036510367
pm.retrieveAll(mStatsObjColl);
10368+
Long tableID = directSql.getTableId(dbName, tableName);
10369+
directSql.deleteTblColumnStatsState(tableID);
1036610370
if (mStatsObjColl != null) {
1036710371
pm.deletePersistentAll(mStatsObjColl);
1036810372
} else {

‎standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1846,6 +1846,11 @@ public void testColumnStatistics() throws Throwable {
18461846
List<ColumnStatisticsObj> stats = client.getTableColumnStatistics(
18471847
dbName, tblName, Lists.newArrayList(colName[1]), ENGINE);
18481848
assertTrue("stats are not empty: " + stats, stats.isEmpty());
1849+
// test if all columns are deleted from parameter COLUMN_STATS_ACCURATE
1850+
Map<String, String> tableParams = client.getTable(dbName, tblName).getParameters();
1851+
String table_column_stats_accurate = tableParams.get("COLUMN_STATS_ACCURATE");
1852+
assertTrue("parameter COLUMN_STATS_ACCURATE is not accurate in " + tblName, table_column_stats_accurate == null ||
1853+
(!table_column_stats_accurate.contains(colName[0]) && !table_column_stats_accurate.contains(colName[1])));
18491854

18501855
colStats.setStatsDesc(statsDesc);
18511856
colStats.setStatsObj(statsObjs);
@@ -1863,6 +1868,11 @@ public void testColumnStatistics() throws Throwable {
18631868
// multiple columns
18641869
request.setCol_names(Arrays.asList(colName));
18651870
assertTrue(client.deleteColumnStatistics(request));
1871+
// test if the columns in colName array are deleted from parameter COLUMN_STATS_ACCURATE
1872+
tableParams = client.getTable(dbName, tblName).getParameters();
1873+
table_column_stats_accurate = tableParams.get("COLUMN_STATS_ACCURATE");
1874+
assertTrue("parameter COLUMN_STATS_ACCURATE is not accurate in " + tblName, table_column_stats_accurate == null ||
1875+
(!table_column_stats_accurate.contains(colName[0]) && !table_column_stats_accurate.contains(colName[1])));
18661876
colStats3 = client.getTableColumnStatistics(
18671877
dbName, tblName, Lists.newArrayList(colName), ENGINE);
18681878
assertTrue("stats are not empty: " + colStats3, colStats3.isEmpty());
@@ -1958,6 +1968,12 @@ public void testColumnStatistics() throws Throwable {
19581968
Lists.newArrayList(partitions.get(0), partitions.get(1), partitions.get(2)), Lists.newArrayList(colName), ENGINE);
19591969
assertEquals(1, stats2.size());
19601970
assertEquals(2, stats2.get(partitions.get(2)).size());
1971+
// test if all columns are deleted from parameter COLUMN_STATS_ACCURATE
1972+
Partition partition_0 = client.getPartition(dbName, tblName, partitions.get(0));
1973+
Map<String, String> partitionParams = partition_0.getParameters();
1974+
String partition_column_stats_accurate = partitionParams.get("COLUMN_STATS_ACCURATE");
1975+
assertTrue("parameter COLUMN_STATS_ACCURATE is not accurate in " + partitions.get(0),partition_column_stats_accurate == null ||
1976+
(!table_column_stats_accurate.contains(colName[0]) && !table_column_stats_accurate.contains(colName[1])));
19611977

19621978
// no partition or column name is set
19631979
request.unsetPart_names();

0 commit comments

Comments
 (0)