Support non-correlated subquery in having clause

fe/src/main/java/org/apache/doris/analysis/Analyzer.java

@@ -17,18 +17,6 @@
 
 package org.apache.doris.analysis;
 
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.IdentityHashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
 import org.apache.doris.catalog.Catalog;
 import org.apache.doris.catalog.Column;
 import org.apache.doris.catalog.Database;
@@ -53,8 +41,6 @@
 import org.apache.doris.rewrite.FoldConstantsRule;
 import org.apache.doris.rewrite.NormalizeBinaryPredicatesRule;
 import org.apache.doris.thrift.TQueryGlobals;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
@@ -65,6 +51,21 @@
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Sets;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 /**
  * Repository of analysis state for single select block.
  * <p/>
@@ -295,6 +296,9 @@ public Analyzer(Catalog catalog, ConnectContext context) {
      */
     public Analyzer(Analyzer parentAnalyzer) {
         this(parentAnalyzer, parentAnalyzer.globalState);
+        if (parentAnalyzer.isSubquery) {
+            this.isSubquery = true;
+        }
     }
 
     /**
@@ -549,6 +553,16 @@ public SlotDescriptor registerColumnRef(TableName tblName, String colName) throw
             }
             d = resolveColumnRef(newTblName, colName);
         }
+        /*
+         * Now, we only support the columns in the subquery to associate the outer query columns in parent level.
+         * If the level of the association exceeds one level, the associated columns in subquery could not be resolved.
+         * For example:
+         * Select k1 from table a where k1=(select k1 from table b where k1=(select k1 from table c where a.k1=k1));
+         * The inner subquery: select k1 from table c where a.k1=k1;
+         * There is a associated column (a.k1) which belongs to the outer query appears in the inner subquery.
+         * This column could not be resolved because doris can only resolved the parent column instead of grandpa.
+         * The exception of this query like that: Unknown column 'k1' in 'a'
+         */
         if (d == null && hasAncestors() && isSubquery) {
             // analyzer father for subquery
             if (newTblName == null) {

fe/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java

@@ -17,9 +17,6 @@
 
 package org.apache.doris.analysis;
 
-import java.util.List;
-import java.util.Objects;
-
 import org.apache.doris.catalog.Function;
 import org.apache.doris.catalog.FunctionSet;
 import org.apache.doris.catalog.PrimitiveType;
@@ -29,12 +26,16 @@
 import org.apache.doris.thrift.TExprNode;
 import org.apache.doris.thrift.TExprNodeType;
 import org.apache.doris.thrift.TExprOpcode;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.List;
+import java.util.Objects;
+
 public class ArithmeticExpr extends Expr {
     private static final Logger LOG = LogManager.getLogger(ArithmeticExpr.class);
 
@@ -216,7 +217,6 @@ private Type findCommonType(Type t1, Type t2) {
 
     @Override
     public void analyzeImpl(Analyzer analyzer) throws AnalysisException {
-
         // bitnot is the only unary op, deal with it here
         if (op == Operator.BITNOT) {
             type = Type.BIGINT;
@@ -231,6 +231,12 @@ public void analyzeImpl(Analyzer analyzer) throws AnalysisException {
             return;
         }
 
+        analyzeSubqueryInChildren();
+        // if children has subquery, it will be rewritten and reanalyzed in the future.
+        if (contains(Subquery.class)) {
+            return;
+        }
+
         Type t1 = getChild(0).getType().getNumResultType();
         Type t2 = getChild(1).getType().getNumResultType();
         // Find result type of this operator
@@ -275,6 +281,31 @@ public void analyzeImpl(Analyzer analyzer) throws AnalysisException {
         }
     }
 
+    public void analyzeSubqueryInChildren() throws AnalysisException {
+        for (Expr child : children) {
+            if (child instanceof Subquery) {
+                Subquery subquery = (Subquery) child;
+                if (!subquery.returnsScalarColumn()) {
+                    String msg = "Subquery of arithmetic expr must return a single column: " + child.toSql();
+                    throw new AnalysisException(msg);
+                }
+                /**
+                 * Situation: The expr is a binary predicate and the type of subquery is not scalar type.
+                 * Add assert: The stmt of subquery is added an assert condition (return error if row count > 1).
+                 * Input params:
+                 *     expr: 0.9*(select k1 from t2)
+                 *     subquery stmt: select k1 from t2
+                 * Output params:
+                 *     new expr: 0.9 * (select k1 from t2 (assert row count: return error if row count > 1 ))
+                 *     subquery stmt: select k1 from t2 (assert row count: return error if row count > 1 )
+                 */
+                if (!subquery.getType().isScalarType()) {
+                    subquery.getStatement().setAssertNumRowsElement(1);
+                }
+            }
+        }
+    }
+
     @Override
     public int hashCode() {
         return 31 * super.hashCode() + Objects.hashCode(op);

fe/src/main/java/org/apache/doris/analysis/BinaryPredicate.java

@@ -17,11 +17,6 @@
 
 package org.apache.doris.analysis;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Objects;
-
 import org.apache.doris.catalog.Function;
 import org.apache.doris.catalog.FunctionSet;
 import org.apache.doris.catalog.PrimitiveType;
@@ -35,12 +30,18 @@
 import org.apache.doris.thrift.TExprNode;
 import org.apache.doris.thrift.TExprNodeType;
 import org.apache.doris.thrift.TExprOpcode;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Objects;
+
 /**
  * Most predicates with two operands..
  */
@@ -313,14 +314,30 @@ public void analyzeImpl(Analyzer analyzer) throws AnalysisException {
         super.analyzeImpl(analyzer);
 
         for (Expr expr : children) {
-            if (expr instanceof Subquery && !((Subquery) expr).returnsScalarColumn()) {
-                String msg = "Subquery of binary predicate must return a single column: " + expr.toSql();
-                throw new AnalysisException(msg);
+            if (expr instanceof Subquery) {
+                Subquery subquery = (Subquery) expr;
+                if (!subquery.returnsScalarColumn()) {
+                    String msg = "Subquery of binary predicate must return a single column: " + expr.toSql();
+                    throw new AnalysisException(msg);
+                }
+                /**
+                 * Situation: The expr is a binary predicate and the type of subquery is not scalar type.
+                 * Add assert: The stmt of subquery is added an assert condition (return error if row count > 1).
+                 * Input params:
+                 *     expr: k1=(select k1 from t2)
+                 *     subquery stmt: select k1 from t2
+                 * Output params:
+                 *     new expr: k1 = (select k1 from t2 (assert row count: return error if row count > 1 ))
+                 *     subquery stmt: select k1 from t2 (assert row count: return error if row count > 1 )
+                 */
+                if (!subquery.getType().isScalarType()) {
+                    subquery.getStatement().setAssertNumRowsElement(1);
+                }
             }
         }
 
-        // if children has subquery, it will be written and reanalyzed in the future.
-        if (children.get(0) instanceof Subquery || children.get(1) instanceof Subquery) {
+        // if children has subquery, it will be rewritten and reanalyzed in the future.
+        if (contains(Subquery.class)) {
             return;
         }
 
@@ -558,4 +575,3 @@ public int hashCode() {
         return 31 * super.hashCode() + Objects.hashCode(op);
     }
 }
-

fe/src/main/java/org/apache/doris/analysis/Expr.java

@@ -1462,6 +1462,18 @@ public Subquery getSubquery() {
         return subqueries.get(0);
     }
 
+    public boolean isCorrelatedPredicate(List<TupleId> tupleIdList) {
+        if ((this instanceof BinaryPredicate || this instanceof SlotRef) && !this.isBoundByTupleIds(tupleIdList)) {
+            return true;
+        }
+        for (Expr child : this.getChildren()) {
+            if (child.isCorrelatedPredicate(tupleIdList)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     @Override
     public void write(DataOutput out) throws IOException {
         throw new IOException("Not implemented serializable ");

fe/src/main/java/org/apache/doris/analysis/InlineViewRef.java

@@ -58,7 +58,7 @@ public class InlineViewRef extends TableRef {
     // BEGIN: Members that need to be reset()
 
     // The select or union statement of the inline view
-    private final QueryStmt queryStmt;
+    private QueryStmt queryStmt;
 
     // queryStmt has its own analysis context
     private Analyzer inlineViewAnalyzer;
@@ -397,6 +397,10 @@ public QueryStmt getViewStmt() {
         return queryStmt;
     }
 
+    public void setViewStmt(QueryStmt queryStmt) {
+        this.queryStmt = queryStmt;
+    }
+
     public Analyzer getAnalyzer() {
         Preconditions.checkState(isAnalyzed);
         return inlineViewAnalyzer;

fe/src/main/java/org/apache/doris/analysis/InsertStmt.java

@@ -89,7 +89,7 @@ public class InsertStmt extends DdlStmt {
     // parsed from targetPartitionNames. empty means no partition specified
     private List<Long> targetPartitionIds = Lists.newArrayList();
     private final List<String> targetColumnNames;
-    private final QueryStmt queryStmt;
+    private QueryStmt queryStmt;
     private final List<String> planHints;
     private Boolean isRepartition;
     private boolean isStreaming = false;
@@ -204,6 +204,10 @@ public QueryStmt getQueryStmt() {
         return queryStmt;
     }
 
+    public void setQueryStmt(QueryStmt queryStmt) {
+        this.queryStmt = queryStmt;
+    }
+
 
     @Override
     public void rewriteExprs(ExprRewriter rewriter) throws AnalysisException {

fe/src/main/java/org/apache/doris/analysis/OrderByElement.java

@@ -129,6 +129,12 @@ public String toSql() {
 
         return strBuilder.toString();
     }
+
+    @Override
+    public String toString() {
+        return toSql();
+    }
+
     @Override
     public boolean equals(Object obj) {
         if (obj == null) {

fe/src/main/java/org/apache/doris/analysis/QueryStmt.java

@@ -100,6 +100,11 @@ public abstract class QueryStmt extends StatementBase {
     // used by hll
     protected boolean fromInsert = false;
 
+    // order by elements which has been analyzed
+    // For example: select k1 a from t order by a;
+    // this parameter: order by t.k1
+    protected List<OrderByElement> orderByElementsAfterAnalyzed;
+
     /////////////////////////////////////////
     // END: Members that need to be reset()
 
@@ -226,6 +231,14 @@ protected void createSortInfo(Analyzer analyzer) throws AnalysisException {
         }
         substituteOrdinalsAliases(orderingExprs, "ORDER BY", analyzer);
 
+        // save the order by element after analyzed
+        orderByElementsAfterAnalyzed = Lists.newArrayList();
+        for (int i = 0; i < orderByElements.size(); i++) {
+            OrderByElement orderByElement = new OrderByElement(orderingExprs.get(i), isAscOrder.get(i),
+                    nullsFirstParams.get(i));
+            orderByElementsAfterAnalyzed.add(orderByElement);
+        }
+
         if (!analyzer.isRootAnalyzer() && hasOffset() && !hasLimit()) {
             throw new AnalysisException("Order-by with offset without limit not supported" +
                     " in nested queries.");
@@ -392,6 +405,10 @@ public ArrayList<OrderByElement> getOrderByElements() {
         return orderByElements;
     }
 
+    public List<OrderByElement> getOrderByElementsAfterAnalyzed() {
+        return orderByElementsAfterAnalyzed;
+    }
+
     public void removeOrderByElements() {
         orderByElements = null;
     }
@@ -415,6 +432,10 @@ public void setLimit(long limit) throws AnalysisException {
         limitElement = new LimitElement(newLimit);
     }
 
+    public void removeLimitElement() {
+        limitElement = LimitElement.NO_LIMIT;
+    }
+
     public long getOffset() {
         return limitElement.getOffset();
     }
@@ -494,6 +515,8 @@ public WithClause cloneWithClause() {
         return withClause_ != null ? withClause_.clone() : null;
     }
 
+    public abstract boolean containsCorrelatedPredicate();
+
     /**
      * C'tor for cloning.
      */
@@ -523,6 +546,7 @@ public void reset() {
         baseTblResultExprs.clear();
         aliasSMap.clear();
         ambiguousAliasList.clear();
+        orderByElementsAfterAnalyzed = null;
         sortInfo = null;
         evaluateOrderBy = false;
         fromInsert = false;