branch-4.0: [feature](fe) Add partition filter sql block rule (#62196)

be/src/exec/schema_scanner/schema_sql_block_rule_status_scanner.cpp

@@ -32,6 +32,11 @@
 
 namespace doris {
 
+namespace {
+constexpr size_t LEGACY_SQL_BLOCK_RULE_STATUS_COLUMN_SIZE = 12;
+constexpr size_t REQUIRE_PARTITION_FILTER_INDEX = 8;
+} // namespace
+
 std::vector<SchemaScanner::ColumnDesc>
         SchemaSqlBlockRuleStatusScanner::_s_sql_block_rule_status_columns = {
                 {"NAME", TYPE_STRING, sizeof(StringRef), true},
@@ -42,6 +47,7 @@ std::vector<SchemaScanner::ColumnDesc>
                 {"CARDINALITY", TYPE_BIGINT, sizeof(int64_t), true},
                 {"GLOBAL", TYPE_BOOLEAN, sizeof(bool), true},
                 {"ENABLE", TYPE_BOOLEAN, sizeof(bool), true},
+                {"REQUIRE_PARTITION_FILTER", TYPE_BOOLEAN, sizeof(bool), true},
                 {"BLOCKS", TYPE_BIGINT, sizeof(int64_t), true},
                 {"AVERAGE_DURATION", TYPE_BIGINT, sizeof(int64_t), true},
                 {"LONGEST_DURATION", TYPE_BIGINT, sizeof(int64_t), true},
@@ -65,11 +71,6 @@ Status SchemaSqlBlockRuleStatusScanner::_get_sql_block_rule_status_block_from_fe
     }
 
     TSchemaTableRequestParams schema_table_request_params;
-    for (int i = 0; i < _s_sql_block_rule_status_columns.size(); i++) {
-        schema_table_request_params.__isset.columns_name = true;
-        schema_table_request_params.columns_name.emplace_back(
-                _s_sql_block_rule_status_columns[i].name);
-    }
     schema_table_request_params.__set_current_user_ident(*_param->common_param->current_user_ident);
     schema_table_request_params.__set_frontend_conjuncts(*_param->common_param->frontend_conjuncts);
 
@@ -112,16 +113,31 @@ Status SchemaSqlBlockRuleStatusScanner::_get_sql_block_rule_status_block_from_fe
 
         if (result_data.size() > 0) {
             auto col_size = result_data[0].column_value.size();
-            if (col_size != _s_sql_block_rule_status_columns.size()) {
+            if (col_size != LEGACY_SQL_BLOCK_RULE_STATUS_COLUMN_SIZE &&
+                col_size != _s_sql_block_rule_status_columns.size()) {
                 return Status::InternalError("col size not equal");
             }
         }
 
         // Add rows from this FE to the result block
         for (int i = 0; i < result_data.size(); i++) {
-            TRow row = result_data[i];
+            const TRow& row = result_data[i];
+            auto col_size = row.column_value.size();
             for (int j = 0; j < _s_sql_block_rule_status_columns.size(); j++) {
-                RETURN_IF_ERROR(insert_block_column(row.column_value[j], j,
+                if (col_size == LEGACY_SQL_BLOCK_RULE_STATUS_COLUMN_SIZE &&
+                    j == REQUIRE_PARTITION_FILTER_INDEX) {
+                    TCell default_cell;
+                    default_cell.__set_boolVal(false);
+                    RETURN_IF_ERROR(insert_block_column(default_cell, j,
+                                                        _sql_block_rule_status_block.get(),
+                                                        _s_sql_block_rule_status_columns[j].type));
+                    continue;
+                }
+                int row_index = col_size == LEGACY_SQL_BLOCK_RULE_STATUS_COLUMN_SIZE &&
+                                                j > REQUIRE_PARTITION_FILTER_INDEX
+                                        ? j - 1
+                                        : j;
+                RETURN_IF_ERROR(insert_block_column(row.column_value[row_index], j,
                                                     _sql_block_rule_status_block.get(),
                                                     _s_sql_block_rule_status_columns[j].type));
             }

fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRule.java

@@ -75,6 +75,10 @@ public class SqlBlockRule implements Writable, GsonPostProcessable {
     @SerializedName(value = "enable")
     private Boolean enable;
 
+    // whether partitioned table scans require partition filters
+    @SerializedName(value = "requirePartitionFilter")
+    private Boolean requirePartitionFilter;
+
     private Pattern sqlPattern;
     private Histogram tryBlockHistogram;
     private LongCounterMetric blockCount;
@@ -83,13 +87,14 @@ public class SqlBlockRule implements Writable, GsonPostProcessable {
      * Create SqlBlockRule.
      **/
     public SqlBlockRule(String name, String sql, String sqlHash, Long partitionNum, Long tabletNum, Long cardinality,
-            Boolean global, Boolean enable) {
+            Boolean requirePartitionFilter, Boolean global, Boolean enable) {
         this.name = name;
         this.sql = sql;
         this.sqlHash = sqlHash;
         this.partitionNum = partitionNum;
         this.tabletNum = tabletNum;
         this.cardinality = cardinality;
+        this.requirePartitionFilter = requirePartitionFilter;
         this.global = global;
         this.enable = enable;
         if (StringUtils.isNotEmpty(sql)) {
@@ -103,6 +108,7 @@ public SqlBlockRule(String name, String sql, String sqlHash, Long partitionNum,
     public SqlBlockRule() {
         this.tryBlockHistogram = new Histogram(new SlidingWindowReservoir(1000));
         this.blockCount = new LongCounterMetric("blocks", MetricUnit.ROWS, "");
+        this.requirePartitionFilter = false;
     }
 
     public String getName() {
@@ -141,6 +147,10 @@ public Boolean getEnable() {
         return enable;
     }
 
+    public Boolean getRequirePartitionFilter() {
+        return requirePartitionFilter;
+    }
+
     public void setSql(String sql) {
         this.sql = sql;
     }
@@ -173,6 +183,10 @@ public void setEnable(Boolean enable) {
         this.enable = enable;
     }
 
+    public void setRequirePartitionFilter(Boolean requirePartitionFilter) {
+        this.requirePartitionFilter = requirePartitionFilter;
+    }
+
     /**
      * Show SqlBlockRule info.
      **/
@@ -181,7 +195,11 @@ public List<String> getShowInfo() {
                 this.partitionNum == null ? "0" : Long.toString(this.partitionNum),
                 this.tabletNum == null ? "0" : Long.toString(this.tabletNum),
                 this.cardinality == null ? "0" : Long.toString(this.cardinality), String.valueOf(this.global),
-                String.valueOf(this.enable));
+                String.valueOf(this.enable), toShowBoolean(this.requirePartitionFilter));
+    }
+
+    private static String toShowBoolean(Boolean value) {
+        return Boolean.TRUE.equals(value) ? "1" : "0";
     }
 
     public Histogram getTryBlockHistogram() {
@@ -211,5 +229,8 @@ public void gsonPostProcess() {
                 this.getSql())) {
             this.setSqlPattern(Pattern.compile(this.getSql()));
         }
+        if (this.getRequirePartitionFilter() == null) {
+            this.setRequirePartitionFilter(false);
+        }
     }
 }

fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java

@@ -161,6 +161,9 @@ public void alterSqlBlockRule(SqlBlockRule sqlBlockRule) throws AnalysisExceptio
             if (sqlBlockRule.getEnable() == null) {
                 sqlBlockRule.setEnable(originRule.getEnable());
             }
+            if (sqlBlockRule.getRequirePartitionFilter() == null) {
+                sqlBlockRule.setRequirePartitionFilter(originRule.getRequirePartitionFilter());
+            }
             sqlBlockRule.setSqlPattern(Pattern.compile(sqlBlockRule.getSql()));
             verifyLimitations(sqlBlockRule);
             SqlBlockUtil.checkAlterValidate(sqlBlockRule);
@@ -269,13 +272,22 @@ private void matchSql(SqlBlockRule rule, String originSql, String sqlHash) throw
      **/
     public void checkLimitations(Long partitionNum, Long tabletNum, Long cardinality, String user)
             throws AnalysisException {
+        checkLimitations(partitionNum, tabletNum, cardinality, false, false, user);
+    }
+
+    /**
+     * Check scan limitations whether legal by user.
+     **/
+    public void checkLimitations(Long partitionNum, Long tabletNum, Long cardinality,
+            boolean isPartitionedTable, boolean hasPartitionPredicate, String user) throws AnalysisException {
         if (ConnectContext.get().getState().isInternal()) {
             return;
         }
         // match global rule
         for (SqlBlockRule rule : nameToSqlBlockRuleMap.values()) {
             if (rule.getGlobal()) {
-                checkLimitations(rule, partitionNum, tabletNum, cardinality);
+                checkLimitations(rule, partitionNum, tabletNum, cardinality,
+                        isPartitionedTable, hasPartitionPredicate);
             }
         }
         // match user rule
@@ -285,33 +297,42 @@ public void checkLimitations(Long partitionNum, Long tabletNum, Long cardinality
             if (rule == null) {
                 continue;
             }
-            checkLimitations(rule, partitionNum, tabletNum, cardinality);
+            checkLimitations(rule, partitionNum, tabletNum, cardinality,
+                    isPartitionedTable, hasPartitionPredicate);
         }
     }
 
     /**
      * Check number whether legal by SqlBlockRule.
      **/
-    private void checkLimitations(SqlBlockRule rule, Long partitionNum, Long tabletNum, Long cardinality)
+    @VisibleForTesting
+    void checkLimitations(SqlBlockRule rule, Long partitionNum, Long tabletNum, Long cardinality,
+            boolean isPartitionedTable, boolean hasPartitionPredicate)
             throws AnalysisException {
+        if (!rule.getEnable()) {
+            return;
+        }
+        if (Boolean.TRUE.equals(rule.getRequirePartitionFilter()) && isPartitionedTable && !hasPartitionPredicate) {
+            MetricRepo.COUNTER_HIT_SQL_BLOCK_RULE.increase(1L);
+            throw new AnalysisException("sql hits sql block rule: " + rule.getName() + ", missing partition filter");
+        }
         if (rule.getPartitionNum() == 0 && rule.getTabletNum() == 0 && rule.getCardinality() == 0) {
             return;
-        } else if (rule.getEnable()) {
-            if ((rule.getPartitionNum() != 0 && rule.getPartitionNum() < partitionNum) || (rule.getTabletNum() != 0
-                    && rule.getTabletNum() < tabletNum) || (rule.getCardinality() != 0
-                    && rule.getCardinality() < cardinality)) {
-                MetricRepo.COUNTER_HIT_SQL_BLOCK_RULE.increase(1L);
-                if (rule.getPartitionNum() < partitionNum && rule.getPartitionNum() != 0) {
-                    throw new AnalysisException(
-                            "sql hits sql block rule: " + rule.getName() + ", reach partition_num : "
-                                    + rule.getPartitionNum());
-                } else if (rule.getTabletNum() < tabletNum && rule.getTabletNum() != 0) {
-                    throw new AnalysisException("sql hits sql block rule: " + rule.getName() + ", reach tablet_num : "
-                            + rule.getTabletNum());
-                } else if (rule.getCardinality() < cardinality && rule.getCardinality() != 0) {
-                    throw new AnalysisException("sql hits sql block rule: " + rule.getName() + ", reach cardinality : "
-                            + rule.getCardinality());
-                }
+        }
+        if ((rule.getPartitionNum() != 0 && rule.getPartitionNum() < partitionNum) || (rule.getTabletNum() != 0
+                && rule.getTabletNum() < tabletNum) || (rule.getCardinality() != 0
+                && rule.getCardinality() < cardinality)) {
+            MetricRepo.COUNTER_HIT_SQL_BLOCK_RULE.increase(1L);
+            if (rule.getPartitionNum() < partitionNum && rule.getPartitionNum() != 0) {
+                throw new AnalysisException(
+                        "sql hits sql block rule: " + rule.getName() + ", reach partition_num : "
+                                + rule.getPartitionNum());
+            } else if (rule.getTabletNum() < tabletNum && rule.getTabletNum() != 0) {
+                throw new AnalysisException("sql hits sql block rule: " + rule.getName() + ", reach tablet_num : "
+                        + rule.getTabletNum());
+            } else if (rule.getCardinality() < cardinality && rule.getCardinality() != 0) {
+                throw new AnalysisException("sql hits sql block rule: " + rule.getName() + ", reach cardinality : "
+                        + rule.getCardinality());
             }
         }
     }

fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java

@@ -761,6 +761,8 @@ public class SchemaTable extends Table {
                                     .column("CARDINALITY", ScalarType.createType(PrimitiveType.BIGINT))
                                     .column("GLOBAL", ScalarType.createType(PrimitiveType.BOOLEAN))
                                     .column("ENABLE", ScalarType.createType(PrimitiveType.BOOLEAN))
+                                    .column("REQUIRE_PARTITION_FILTER",
+                                            ScalarType.createType(PrimitiveType.BOOLEAN))
                                     .column("BLOCKS", ScalarType.createType(PrimitiveType.BIGINT),
                                             SchemaTableAggregateType.SUM, false)
                                     .column("AVERAGE_DURATION", ScalarType.createType(PrimitiveType.BIGINT),