[feature] (sql-digest) support sql digest (#8919)

apache · May 8, 2022 · c633402 · c633402
1 parent 52a2db1
commit c633402
Show file tree

Hide file tree

Showing 37 changed files with 735 additions and 5 deletions.
diff --git a/docs/en/ecosystem/audit-plugin.md b/docs/en/ecosystem/audit-plugin.md
@@ -72,6 +72,7 @@ create table doris_audit_tbl__
     frontend_ip varchar(32) comment "Frontend ip of executing this statement",
     cpu_time_ms bigint comment "Total scan cpu time in millisecond of this query",
     sql_hash varchar(50) comment "Hash value for this query",
+    sql_digest varchar(48) comment "Sql digest for this query",
     peak_memory_bytes bigint comment "Peak memory bytes used on all backends of this query",
     stmt string comment "The original statement, trimed if longer than 2G"
 ) engine=OLAP
@@ -97,4 +98,4 @@ The `dynamic_partition` attribute selects the number of days to keep the audit l
 
 After that, connect to Doris and use the `INSTALL PLUGIN` command to complete the installation. After successful installation, you can see the installed plug-ins through `SHOW PLUGINS`, and the status is `INSTALLED`.
 
-Upon completion, the plug-in will continuously import audit date into this table at specified intervals.
+Upon completion, the plug-in will continuously import audit date into this table at specified intervals.
diff --git a/docs/zh-CN/ecosystem/audit-plugin.md b/docs/zh-CN/ecosystem/audit-plugin.md
@@ -72,6 +72,7 @@ create table doris_audit_tbl__
     frontend_ip varchar(32) comment "Frontend ip of executing this statement",
     cpu_time_ms bigint comment "Total scan cpu time in millisecond of this query",
     sql_hash varchar(48) comment "Hash value for this query",
+    sql_digest varchar(48) comment "Sql digest for this query",
     peak_memory_bytes bigint comment "Peak memory bytes used on all backends of this query",
     stmt string comment "The original statement, trimed if longer than 2G "
 ) engine=OLAP

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java
@@ -838,6 +838,36 @@ public String toSqlImpl() {
         return sb.toString();
     }
 
+    @Override
+    public String toDigestImpl() {
+        StringBuilder sb = new StringBuilder();
+        sb.append(fnCall.toDigest()).append(" OVER (");
+        boolean needsSpace = false;
+        if (!partitionExprs.isEmpty()) {
+            sb.append("PARTITION BY ").append(exprListToDigest(partitionExprs));
+            needsSpace = true;
+        }
+        if (!orderByElements.isEmpty()) {
+            List<String> orderByStrings = Lists.newArrayList();
+            for (OrderByElement e : orderByElements) {
+                orderByStrings.add(e.toDigest());
+            }
+            if (needsSpace) {
+                sb.append(" ");
+            }
+            sb.append("ORDER BY ").append(Joiner.on(", ").join(orderByStrings));
+            needsSpace = true;
+        }
+        if (window != null) {
+            if (needsSpace) {
+                sb.append(" ");
+            }
+            sb.append(window.toDigest());
+        }
+        sb.append(")");
+        return sb.toString();
+    }
+
     private String exprListToSql(List<? extends Expr> exprs) {
         if (exprs == null || exprs.isEmpty())
             return "";
@@ -847,4 +877,15 @@ private String exprListToSql(List<? extends Expr> exprs) {
         }
         return Joiner.on(", ").join(strings);
     }
+
+    private String exprListToDigest(List<? extends Expr> exprs) {
+        if (exprs == null || exprs.isEmpty()) {
+            return "";
+        }
+        List<String> strings = Lists.newArrayList();
+        for (Expr expr : exprs) {
+            strings.add(expr.toDigest());
+        }
+        return Joiner.on(", ").join(strings);
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticWindow.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticWindow.java
@@ -167,6 +167,17 @@ public String toSql() {
             return sb.toString();
         }
 
+        public String toDigest() {
+            StringBuilder sb = new StringBuilder();
+
+            if (expr != null) {
+                sb.append(expr.toDigest()).append(" ");
+            }
+
+            sb.append(type.toString());
+            return sb.toString();
+        }
+
         public TAnalyticWindowBoundary toThrift(Type windowType) {
             TAnalyticWindowBoundary result = new TAnalyticWindowBoundary(type.toThrift());
 
@@ -296,6 +307,21 @@ public String toSql() {
         return sb.toString();
     }
 
+    public String toDigest() {
+        StringBuilder sb = new StringBuilder();
+        sb.append(type_.toString()).append(" ");
+
+        if (rightBoundary_ == null) {
+            sb.append(leftBoundary_.toDigest());
+        } else {
+            sb.append("BETWEEN ").append(leftBoundary_.toDigest()).append(" AND ");
+            sb.append(rightBoundary_.toDigest());
+        }
+
+        return sb.toString();
+    }
+
+
     public TAnalyticWindow toThrift() {
         TAnalyticWindow result = new TAnalyticWindow(type_.toThrift());
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java
@@ -221,6 +221,15 @@ public String toSqlImpl() {
         }
     }
 
+    @Override
+    public String toDigestImpl() {
+        if (children.size() == 1) {
+            return op.toString() + " " + getChild(0).toDigest();
+        } else {
+            return getChild(0).toDigest() + " " + op.toString() + " " + getChild(1).toDigest();
+        }
+    }
+
     @Override
     protected void toThrift(TExprNode msg) {
         msg.node_type = TExprNodeType.ARITHMETIC_EXPR;

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
@@ -72,6 +72,14 @@ protected String toSqlImpl() {
         return "ARRAY(" + StringUtils.join(list, ", ") + ")";
     }
 
+    @Override
+    public String toDigestImpl() {
+        List<String> list = new ArrayList<>(children.size());
+        children.forEach(v -> list.add(v.toDigestImpl()));
+
+        return "ARRAY(" + StringUtils.join(list, ", ") + ")";
+    }
+
     @Override
     public String getStringValue() {
         List<String> list = new ArrayList<>(children.size());

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BetweenPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BetweenPredicate.java
@@ -96,7 +96,14 @@ protected void toThrift(TExprNode msg) {
     public String toSqlImpl() {
         String notStr = (isNotBetween) ? "NOT " : "";
         return children.get(0).toSql() + " " + notStr + "BETWEEN " +
-          children.get(1).toSql() + " AND " + children.get(2).toSql();
+                children.get(1).toSql() + " AND " + children.get(2).toSql();
+    }
+
+    @Override
+    public String toDigestImpl() {
+        String notStr = (isNotBetween) ? "NOT " : "";
+        return children.get(0).toDigest() + " " + notStr + "BETWEEN " +
+                children.get(1).toDigest() + " AND " + children.get(2).toDigest();
     }
 
     @Override

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java
@@ -242,6 +242,11 @@ public String toSqlImpl() {
         return getChild(0).toSql() + " " + op.toString() + " " + getChild(1).toSql();
     }
 
+    @Override
+    public String toDigestImpl() {
+        return getChild(0).toDigest() + " " + op.toString() + " " + getChild(1).toDigest();
+    }
+
     @Override
     protected void toThrift(TExprNode msg) {
         msg.node_type = TExprNodeType.BINARY_PRED;

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CaseExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CaseExpr.java
@@ -129,6 +129,24 @@ public String toSqlImpl() {
         return output.toString();
     }
 
+    @Override
+    public String toDigestImpl() {
+        StringBuilder sb = new StringBuilder("CASE");
+        int childIdx = 0;
+        if (hasCaseExpr) {
+            sb.append(" ").append(children.get(childIdx++).toDigest());
+        }
+        while (childIdx + 2 <= children.size()) {
+            sb.append(" WHEN ").append(children.get(childIdx++).toDigest());
+            sb.append(" THEN ").append(children.get(childIdx++).toDigest());
+        }
+        if (hasElseExpr) {
+            sb.append(" ELSE ").append(children.get(children.size() - 1).toDigest());
+        }
+        sb.append(" END");
+        return sb.toString();
+    }
+
     @Override
     public boolean isVectorized() {
         return false;

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
@@ -206,6 +206,27 @@ public String toSqlImpl() {
         }
     }
 
+    @Override
+    public String toDigestImpl() {
+        boolean isVerbose = ConnectContext.get() != null &&
+                ConnectContext.get().getExecutor() != null &&
+                ConnectContext.get().getExecutor().getParsedStmt() != null &&
+                ConnectContext.get().getExecutor().getParsedStmt().getExplainOptions() != null &&
+                ConnectContext.get().getExecutor().getParsedStmt().getExplainOptions().isVerbose();
+        if (isImplicit && !isVerbose) {
+            return getChild(0).toDigest();
+        }
+        if (isAnalyzed) {
+            if (type.isStringType()) {
+                return "CAST(" + getChild(0).toDigest() + " AS " + "CHARACTER" + ")";
+            } else {
+                return "CAST(" + getChild(0).toDigest() + " AS " + type.toString() + ")";
+            }
+        } else {
+            return "CAST(" + getChild(0).toDigest() + " AS " + targetTypeDef.toString() + ")";
+        }
+    }
+
     @Override
     protected void treeToThriftHelper(TExpr container) {
         if (noOp) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java
@@ -94,6 +94,15 @@ public String toSqlImpl() {
         }
     }
 
+    @Override
+    public String toDigestImpl() {
+        if (children.size() == 1) {
+            return "NOT " + getChild(0).toDigest();
+        } else {
+            return getChild(0).toDigest() + " " + op.toString() + " " + getChild(1).toDigest();
+        }
+    }
+
     @Override
     protected void toThrift(TExprNode msg) {
         msg.node_type = TExprNodeType.COMPOUND_PRED;

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExistsPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExistsPredicate.java
@@ -58,6 +58,7 @@ protected void toThrift(TExprNode msg) {
     @Override
     public Expr clone() { return new ExistsPredicate(this); }
 
+    @Override
     public String toSqlImpl() {
         StringBuilder strBuilder = new StringBuilder();
         if (notExists) {
@@ -69,6 +70,18 @@ public String toSqlImpl() {
         return strBuilder.toString();
     }
 
+    @Override
+    public String toDigestImpl() {
+        StringBuilder strBuilder = new StringBuilder();
+        if (notExists) {
+            strBuilder.append("NOT ");
+
+        }
+        strBuilder.append("EXISTS ");
+        strBuilder.append(getChild(0).toDigest());
+        return strBuilder.toString();
+    }
+
     @Override
     public int hashCode() {
         return 31 * super.hashCode() + Boolean.hashCode(notExists);

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
@@ -874,12 +874,25 @@ public String toSql() {
         return (printSqlInParens) ? "(" + toSqlImpl() + ")" : toSqlImpl();
     }
 
+    public String toDigest() {
+        return (printSqlInParens) ? "(" + toDigestImpl() + ")" : toDigestImpl();
+    }
+
     /**
      * Returns a SQL string representing this expr. Subclasses should override this method
      * instead of toSql() to ensure that parenthesis are properly added around the toSql().
      */
     protected abstract String toSqlImpl();
 
+    /**
+     * !!!!!! Important !!!!!!
+     * Subclasses should override this method if
+     * sql digest should be represented different from tosqlImpl().
+     */
+    protected String toDigestImpl() {
+        return toSqlImpl();
+    }
+
     public String toMySql() {
         return toSql();
     }
@@ -952,6 +965,14 @@ public List<String> childrenToSql() {
     //        }
     //    }
 
+    public List<String> childrenToDigest() {
+        List<String> childrenDigestList = Lists.newArrayList();
+        for (Expr child : children) {
+            childrenDigestList.add(child.toDigest());
+        }
+        return childrenDigestList;
+    }
+
     public static com.google.common.base.Predicate<Expr> isAggregatePredicate() {
         return IS_AGGREGATE_PREDICATE;
     }

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java
@@ -193,6 +193,17 @@ public String toSql() {
         return builder.toString();
     }
 
+    public String toDigest() {
+        StringBuilder builder = new StringBuilder();
+        if (!tableRefs_.isEmpty()) {
+            builder.append(" FROM");
+            for (int i = 0; i < tableRefs_.size(); ++i) {
+                builder.append(" " + tableRefs_.get(i).toDigest());
+            }
+        }
+        return builder.toString();
+    }
+
     public boolean isEmpty() { return tableRefs_.isEmpty(); }
 
     @Override

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -292,6 +292,61 @@ public String toSqlImpl() {
         return sb.toString();
     }
 
+    private String paramsToDigest() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("(");
+
+        if (fnParams.isStar()) {
+            sb.append("*");
+        }
+        if (fnParams.isDistinct()) {
+            sb.append("DISTINCT ");
+        }
+        int len = children.size();
+        List<String> result = Lists.newArrayList();
+        if (fnName.getFunction().equalsIgnoreCase("json_array") ||
+                fnName.getFunction().equalsIgnoreCase("json_object")) {
+            len = len - 1;
+        }
+        if (fnName.getFunction().equalsIgnoreCase("aes_decrypt") ||
+                fnName.getFunction().equalsIgnoreCase("aes_encrypt") ||
+                fnName.getFunction().equalsIgnoreCase("sm4_decrypt") ||
+                fnName.getFunction().equalsIgnoreCase("sm4_encrypt")) {
+            len = len - 1;
+        }
+        for (int i = 0; i < len; ++i) {
+            if (i == 1 && (fnName.getFunction().equalsIgnoreCase("aes_decrypt") ||
+                    fnName.getFunction().equalsIgnoreCase("aes_encrypt") ||
+                    fnName.getFunction().equalsIgnoreCase("sm4_decrypt") ||
+                    fnName.getFunction().equalsIgnoreCase("sm4_encrypt"))) {
+                result.add("\'***\'");
+            } else {
+                result.add(children.get(i).toDigest());
+            }
+        }
+        sb.append(Joiner.on(", ").join(result)).append(")");
+        return sb.toString();
+    }
+
+    @Override
+    public String toDigestImpl() {
+        Expr expr;
+        if (originStmtFnExpr != null) {
+            expr = originStmtFnExpr;
+        } else {
+            expr = this;
+        }
+        StringBuilder sb = new StringBuilder();
+        sb.append(((FunctionCallExpr) expr).fnName);
+        sb.append(paramsToDigest());
+        if (fnName.getFunction().equalsIgnoreCase("json_quote") ||
+                fnName.getFunction().equalsIgnoreCase("json_array") ||
+                fnName.getFunction().equalsIgnoreCase("json_object")) {
+            return forJSON(sb.toString());
+        }
+        return sb.toString();
+    }
+
     @Override
     public String debugString() {
         return MoreObjects.toStringHelper(this)/*.add("op", aggOp)*/.add("name", fnName).add("isStar",