Skip to content
Permalink
Browse files
[feature] (sql-digest) support sql digest (#8919)
  • Loading branch information
Henry2SS committed May 8, 2022
1 parent 52a2db1 commit c633402ce3753f3726ec0bbbe3b966f8e998625b
Showing 37 changed files with 735 additions and 5 deletions.
@@ -72,6 +72,7 @@ create table doris_audit_tbl__
frontend_ip varchar(32) comment "Frontend ip of executing this statement",
cpu_time_ms bigint comment "Total scan cpu time in millisecond of this query",
sql_hash varchar(50) comment "Hash value for this query",
sql_digest varchar(48) comment "Sql digest for this query",
peak_memory_bytes bigint comment "Peak memory bytes used on all backends of this query",
stmt string comment "The original statement, trimed if longer than 2G"
) engine=OLAP
@@ -97,4 +98,4 @@ The `dynamic_partition` attribute selects the number of days to keep the audit l

After that, connect to Doris and use the `INSTALL PLUGIN` command to complete the installation. After successful installation, you can see the installed plug-ins through `SHOW PLUGINS`, and the status is `INSTALLED`.

Upon completion, the plug-in will continuously import audit date into this table at specified intervals.
Upon completion, the plug-in will continuously import audit date into this table at specified intervals.
@@ -72,6 +72,7 @@ create table doris_audit_tbl__
frontend_ip varchar(32) comment "Frontend ip of executing this statement",
cpu_time_ms bigint comment "Total scan cpu time in millisecond of this query",
sql_hash varchar(48) comment "Hash value for this query",
sql_digest varchar(48) comment "Sql digest for this query",
peak_memory_bytes bigint comment "Peak memory bytes used on all backends of this query",
stmt string comment "The original statement, trimed if longer than 2G "
) engine=OLAP
@@ -838,6 +838,36 @@ public String toSqlImpl() {
return sb.toString();
}

@Override
public String toDigestImpl() {
StringBuilder sb = new StringBuilder();
sb.append(fnCall.toDigest()).append(" OVER (");
boolean needsSpace = false;
if (!partitionExprs.isEmpty()) {
sb.append("PARTITION BY ").append(exprListToDigest(partitionExprs));
needsSpace = true;
}
if (!orderByElements.isEmpty()) {
List<String> orderByStrings = Lists.newArrayList();
for (OrderByElement e : orderByElements) {
orderByStrings.add(e.toDigest());
}
if (needsSpace) {
sb.append(" ");
}
sb.append("ORDER BY ").append(Joiner.on(", ").join(orderByStrings));
needsSpace = true;
}
if (window != null) {
if (needsSpace) {
sb.append(" ");
}
sb.append(window.toDigest());
}
sb.append(")");
return sb.toString();
}

private String exprListToSql(List<? extends Expr> exprs) {
if (exprs == null || exprs.isEmpty())
return "";
@@ -847,4 +877,15 @@ private String exprListToSql(List<? extends Expr> exprs) {
}
return Joiner.on(", ").join(strings);
}

private String exprListToDigest(List<? extends Expr> exprs) {
if (exprs == null || exprs.isEmpty()) {
return "";
}
List<String> strings = Lists.newArrayList();
for (Expr expr : exprs) {
strings.add(expr.toDigest());
}
return Joiner.on(", ").join(strings);
}
}
@@ -167,6 +167,17 @@ public String toSql() {
return sb.toString();
}

public String toDigest() {
StringBuilder sb = new StringBuilder();

if (expr != null) {
sb.append(expr.toDigest()).append(" ");
}

sb.append(type.toString());
return sb.toString();
}

public TAnalyticWindowBoundary toThrift(Type windowType) {
TAnalyticWindowBoundary result = new TAnalyticWindowBoundary(type.toThrift());

@@ -296,6 +307,21 @@ public String toSql() {
return sb.toString();
}

public String toDigest() {
StringBuilder sb = new StringBuilder();
sb.append(type_.toString()).append(" ");

if (rightBoundary_ == null) {
sb.append(leftBoundary_.toDigest());
} else {
sb.append("BETWEEN ").append(leftBoundary_.toDigest()).append(" AND ");
sb.append(rightBoundary_.toDigest());
}

return sb.toString();
}


public TAnalyticWindow toThrift() {
TAnalyticWindow result = new TAnalyticWindow(type_.toThrift());

@@ -221,6 +221,15 @@ public String toSqlImpl() {
}
}

@Override
public String toDigestImpl() {
if (children.size() == 1) {
return op.toString() + " " + getChild(0).toDigest();
} else {
return getChild(0).toDigest() + " " + op.toString() + " " + getChild(1).toDigest();
}
}

@Override
protected void toThrift(TExprNode msg) {
msg.node_type = TExprNodeType.ARITHMETIC_EXPR;
@@ -72,6 +72,14 @@ protected String toSqlImpl() {
return "ARRAY(" + StringUtils.join(list, ", ") + ")";
}

@Override
public String toDigestImpl() {
List<String> list = new ArrayList<>(children.size());
children.forEach(v -> list.add(v.toDigestImpl()));

return "ARRAY(" + StringUtils.join(list, ", ") + ")";
}

@Override
public String getStringValue() {
List<String> list = new ArrayList<>(children.size());
@@ -96,7 +96,14 @@ protected void toThrift(TExprNode msg) {
public String toSqlImpl() {
String notStr = (isNotBetween) ? "NOT " : "";
return children.get(0).toSql() + " " + notStr + "BETWEEN " +
children.get(1).toSql() + " AND " + children.get(2).toSql();
children.get(1).toSql() + " AND " + children.get(2).toSql();
}

@Override
public String toDigestImpl() {
String notStr = (isNotBetween) ? "NOT " : "";
return children.get(0).toDigest() + " " + notStr + "BETWEEN " +
children.get(1).toDigest() + " AND " + children.get(2).toDigest();
}

@Override
@@ -242,6 +242,11 @@ public String toSqlImpl() {
return getChild(0).toSql() + " " + op.toString() + " " + getChild(1).toSql();
}

@Override
public String toDigestImpl() {
return getChild(0).toDigest() + " " + op.toString() + " " + getChild(1).toDigest();
}

@Override
protected void toThrift(TExprNode msg) {
msg.node_type = TExprNodeType.BINARY_PRED;
@@ -129,6 +129,24 @@ public String toSqlImpl() {
return output.toString();
}

@Override
public String toDigestImpl() {
StringBuilder sb = new StringBuilder("CASE");
int childIdx = 0;
if (hasCaseExpr) {
sb.append(" ").append(children.get(childIdx++).toDigest());
}
while (childIdx + 2 <= children.size()) {
sb.append(" WHEN ").append(children.get(childIdx++).toDigest());
sb.append(" THEN ").append(children.get(childIdx++).toDigest());
}
if (hasElseExpr) {
sb.append(" ELSE ").append(children.get(children.size() - 1).toDigest());
}
sb.append(" END");
return sb.toString();
}

@Override
public boolean isVectorized() {
return false;
@@ -206,6 +206,27 @@ public String toSqlImpl() {
}
}

@Override
public String toDigestImpl() {
boolean isVerbose = ConnectContext.get() != null &&
ConnectContext.get().getExecutor() != null &&
ConnectContext.get().getExecutor().getParsedStmt() != null &&
ConnectContext.get().getExecutor().getParsedStmt().getExplainOptions() != null &&
ConnectContext.get().getExecutor().getParsedStmt().getExplainOptions().isVerbose();
if (isImplicit && !isVerbose) {
return getChild(0).toDigest();
}
if (isAnalyzed) {
if (type.isStringType()) {
return "CAST(" + getChild(0).toDigest() + " AS " + "CHARACTER" + ")";
} else {
return "CAST(" + getChild(0).toDigest() + " AS " + type.toString() + ")";
}
} else {
return "CAST(" + getChild(0).toDigest() + " AS " + targetTypeDef.toString() + ")";
}
}

@Override
protected void treeToThriftHelper(TExpr container) {
if (noOp) {
@@ -94,6 +94,15 @@ public String toSqlImpl() {
}
}

@Override
public String toDigestImpl() {
if (children.size() == 1) {
return "NOT " + getChild(0).toDigest();
} else {
return getChild(0).toDigest() + " " + op.toString() + " " + getChild(1).toDigest();
}
}

@Override
protected void toThrift(TExprNode msg) {
msg.node_type = TExprNodeType.COMPOUND_PRED;
@@ -58,6 +58,7 @@ protected void toThrift(TExprNode msg) {
@Override
public Expr clone() { return new ExistsPredicate(this); }

@Override
public String toSqlImpl() {
StringBuilder strBuilder = new StringBuilder();
if (notExists) {
@@ -69,6 +70,18 @@ public String toSqlImpl() {
return strBuilder.toString();
}

@Override
public String toDigestImpl() {
StringBuilder strBuilder = new StringBuilder();
if (notExists) {
strBuilder.append("NOT ");

}
strBuilder.append("EXISTS ");
strBuilder.append(getChild(0).toDigest());
return strBuilder.toString();
}

@Override
public int hashCode() {
return 31 * super.hashCode() + Boolean.hashCode(notExists);
@@ -874,12 +874,25 @@ public String toSql() {
return (printSqlInParens) ? "(" + toSqlImpl() + ")" : toSqlImpl();
}

public String toDigest() {
return (printSqlInParens) ? "(" + toDigestImpl() + ")" : toDigestImpl();
}

/**
* Returns a SQL string representing this expr. Subclasses should override this method
* instead of toSql() to ensure that parenthesis are properly added around the toSql().
*/
protected abstract String toSqlImpl();

/**
* !!!!!! Important !!!!!!
* Subclasses should override this method if
* sql digest should be represented different from tosqlImpl().
*/
protected String toDigestImpl() {
return toSqlImpl();
}

public String toMySql() {
return toSql();
}
@@ -952,6 +965,14 @@ public List<String> childrenToSql() {
// }
// }

public List<String> childrenToDigest() {
List<String> childrenDigestList = Lists.newArrayList();
for (Expr child : children) {
childrenDigestList.add(child.toDigest());
}
return childrenDigestList;
}

public static com.google.common.base.Predicate<Expr> isAggregatePredicate() {
return IS_AGGREGATE_PREDICATE;
}
@@ -193,6 +193,17 @@ public String toSql() {
return builder.toString();
}

public String toDigest() {
StringBuilder builder = new StringBuilder();
if (!tableRefs_.isEmpty()) {
builder.append(" FROM");
for (int i = 0; i < tableRefs_.size(); ++i) {
builder.append(" " + tableRefs_.get(i).toDigest());
}
}
return builder.toString();
}

public boolean isEmpty() { return tableRefs_.isEmpty(); }

@Override
@@ -292,6 +292,61 @@ public String toSqlImpl() {
return sb.toString();
}

private String paramsToDigest() {
StringBuilder sb = new StringBuilder();
sb.append("(");

if (fnParams.isStar()) {
sb.append("*");
}
if (fnParams.isDistinct()) {
sb.append("DISTINCT ");
}
int len = children.size();
List<String> result = Lists.newArrayList();
if (fnName.getFunction().equalsIgnoreCase("json_array") ||
fnName.getFunction().equalsIgnoreCase("json_object")) {
len = len - 1;
}
if (fnName.getFunction().equalsIgnoreCase("aes_decrypt") ||
fnName.getFunction().equalsIgnoreCase("aes_encrypt") ||
fnName.getFunction().equalsIgnoreCase("sm4_decrypt") ||
fnName.getFunction().equalsIgnoreCase("sm4_encrypt")) {
len = len - 1;
}
for (int i = 0; i < len; ++i) {
if (i == 1 && (fnName.getFunction().equalsIgnoreCase("aes_decrypt") ||
fnName.getFunction().equalsIgnoreCase("aes_encrypt") ||
fnName.getFunction().equalsIgnoreCase("sm4_decrypt") ||
fnName.getFunction().equalsIgnoreCase("sm4_encrypt"))) {
result.add("\'***\'");
} else {
result.add(children.get(i).toDigest());
}
}
sb.append(Joiner.on(", ").join(result)).append(")");
return sb.toString();
}

@Override
public String toDigestImpl() {
Expr expr;
if (originStmtFnExpr != null) {
expr = originStmtFnExpr;
} else {
expr = this;
}
StringBuilder sb = new StringBuilder();
sb.append(((FunctionCallExpr) expr).fnName);
sb.append(paramsToDigest());
if (fnName.getFunction().equalsIgnoreCase("json_quote") ||
fnName.getFunction().equalsIgnoreCase("json_array") ||
fnName.getFunction().equalsIgnoreCase("json_object")) {
return forJSON(sb.toString());
}
return sb.toString();
}

@Override
public String debugString() {
return MoreObjects.toStringHelper(this)/*.add("op", aggOp)*/.add("name", fnName).add("isStar",

0 comments on commit c633402

Please sign in to comment.