HIVE-28655: Implement HMS Related Drop Stats Changes, Reset COLUMN_STAT_ACCURATE After Dropping

Author: Hongdan Zhu

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java

@@ -7431,6 +7431,8 @@ public boolean delete_column_statistics_req(DeleteColumnStatisticsRequest req) t
       if (!isPartitioned || req.isTableLevel()) {
         ret = rawStore.deleteTableColumnStatistics(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, colNames, engine);
         if (ret) {
+          // remove the deleted column names in parameter COLUMN_STATS_ACCURATE
+          StatsSetupConst.removeColumnStatsState(table.getParameters(), colNames);
           eventType = EventType.DELETE_TABLE_COLUMN_STAT;
           for (String colName :
               colNames == null ? table.getSd().getCols().stream().map(FieldSchema::getName).collect(Collectors.toList()) : colNames) {
@@ -7460,6 +7462,10 @@ public boolean delete_column_statistics_req(DeleteColumnStatisticsRequest req) t
               .collect(Collectors.toList()) : colNames) {
             for (String partName : partNames) {
               List<String> partVals = getPartValsFromName(table, partName);
+              Partition partition = rawStore.getPartition(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, partVals);
+              Map<String, String> partParams = partition.getParameters();
+              // remove the deleted column names in parameter COLUMN_STATS_ACCURATE
+              StatsSetupConst.removeColumnStatsState(partParams, colNames);
               if (transactionalListeners != null && !transactionalListeners.isEmpty()) {
                 MetaStoreListenerNotifier.notifyEvent(transactionalListeners, eventType,
                     new DeletePartitionColumnStatEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName,

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java

@@ -3242,6 +3242,34 @@ public void deleteColumnStatsState(long tbl_id) throws MetaException {
     }
   }
 
+  /**
+  public Long getTableId(String dbName, String tableName) throws MetaException {
+    String queryText;
+    Long result;
+    switch (dbType.dbType) {
+      case MYSQL:
+      case POSTGRES:
+      case DERBY:
+      default:
+        // @formatter:off
+        queryText = ""
+                + "select t.\"TBL_ID\" from " + TBLS + " t "
+                + " join " + DBS + " d on t.\"DB_ID\" = d.\"DB_ID\" "
+                + " where "
+                + " d.\"NAME\" = " + dbName
+                + " and t.\"TBL_NAME\" = " + tableName;
+        // @formatter:on
+    }
+    try (QueryWrapper query = new QueryWrapper(pm.newQuery("javax.jdo.query.SQL", queryText))) {
+      result = executeWithArray(query.getInnerQuery(), null, queryText);
+    } catch (Throwable t) {
+      throw new MetaException("Failed to fetch table ID: " + t.getMessage());
+    }
+    return result;
+  }
+   **/
+
+
   public boolean deleteTableColumnStatistics(long tableId, List<String> colNames, String engine) {
     String deleteSql = "delete from " + TAB_COL_STATS + " where \"TBL_ID\" = " + tableId;
     if (colNames != null && !colNames.isEmpty()) {

standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java

@@ -1863,6 +1863,10 @@ public void testColumnStatistics() throws Throwable {
       // multiple columns
       request.setCol_names(Arrays.asList(colName));
       assertTrue(client.deleteColumnStatistics(request));
+      Map<String, String> tableParams = client.getTable(dbName, tblName).getParameters();
+      String table_column_stats_accurate = tableParams.get("COLUMN_STATS_ACCURATE");
+      assertTrue(table_column_stats_accurate == null ||
+              (!table_column_stats_accurate.contains(colName[0]) && !table_column_stats_accurate.contains(colName[1])));
       colStats3 = client.getTableColumnStatistics(
           dbName, tblName, Lists.newArrayList(colName), ENGINE);
       assertTrue("stats are not empty: " + colStats3, colStats3.isEmpty());