diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/AlterViewStmt.java b/fe/fe-core/src/main/java/com/starrocks/analysis/AlterViewStmt.java index 0262ee46b1b25..9e160fe572315 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/AlterViewStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/AlterViewStmt.java @@ -21,15 +21,8 @@ package com.starrocks.analysis; -import com.starrocks.catalog.Table; -import com.starrocks.catalog.View; import com.starrocks.common.AnalysisException; -import com.starrocks.common.ErrorCode; -import com.starrocks.common.ErrorReport; import com.starrocks.common.UserException; -import com.starrocks.mysql.privilege.PrivPredicate; -import com.starrocks.qe.ConnectContext; -import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.ast.AstVisitor; import com.starrocks.sql.ast.QueryStatement; @@ -52,36 +45,6 @@ public TableName getTbl() { @Override public void analyze(Analyzer analyzer) throws AnalysisException, UserException { - super.analyze(analyzer); - if (tableName == null) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_TABLES_USED); - } - tableName.analyze(analyzer); - - Table table = analyzer.getTable(tableName); - if (!(table instanceof View)) { - throw new AnalysisException( - String.format("ALTER VIEW not allowed on a table:%s.%s", getDbName(), getTable())); - } - - if (!GlobalStateMgr.getCurrentState().getAuth() - .checkTblPriv(ConnectContext.get(), tableName.getDb(), tableName.getTbl(), - PrivPredicate.ALTER)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "ALTER VIEW", - ConnectContext.get().getQualifiedUser(), - ConnectContext.get().getRemoteIP(), - tableName.getTbl()); - } - - if (cols != null) { - cloneStmt = viewDefStmt.clone(); - } - - viewDefStmt.setNeedToSql(true); - Analyzer viewAnalyzer = new Analyzer(analyzer); - viewDefStmt.analyze(viewAnalyzer); - - createColumnAndViewDefs(analyzer); } @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/Analyzer.java b/fe/fe-core/src/main/java/com/starrocks/analysis/Analyzer.java index 9dd254b41c10f..c01ccb4a0792f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/Analyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/Analyzer.java @@ -21,7 +21,6 @@ package com.starrocks.analysis; -import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Lists; @@ -105,33 +104,13 @@ public class Analyzer { // Flag indicating whether this analyzer belongs to a WITH clause view. private boolean isWithClause_ = false; - // By default, all registered semi-joined tuples are invisible, i.e., their slots - // cannot be referenced. If set, this semi-joined tuple is made visible. Such a tuple - // should only be made visible for analyzing the On-clause of its semi-join. - // In particular, if there are multiple semi-joins in the same query block, then the - // On-clause of any such semi-join is not allowed to reference other semi-joined tuples - // except its own. Therefore, only a single semi-joined tuple can be visible at a time. - private TupleId visibleSemiJoinedTupleId_ = null; - // for some situation that udf is not allowed. - private boolean isUDFAllowed = true; // timezone specified for some operation, such as broker load private String timezone = TimeUtils.DEFAULT_TIME_ZONE; - // Whether to ignore cast expressions - // Compatibility with older versions, maybe delete in near future - private boolean ignoreCast = false; - private String schemaDb; - private String schemaTable; - private String schemaWild; - public boolean isWithClause() { return isWithClause_; } - public void setUDFAllowed(boolean val) { - this.isUDFAllowed = val; - } - public void setTimezone(String timezone) { this.timezone = timezone; } @@ -239,15 +218,6 @@ public GlobalState(GlobalStateMgr globalStateMgr, ConnectContext context) { // Set of lowercase ambiguous implicit table aliases. private final Set ambiguousAliases_ = Sets.newHashSet(); - // Indicates whether this analyzer/block is guaranteed to have an empty result set - // due to a limit 0 or constant conjunct evaluating to false. - private boolean hasEmptyResultSet_ = false; - - // Indicates whether the select-project-join (spj) portion of this query block - // is guaranteed to return an empty result set. Set due to a constant non-Having - // conjunct evaluating to false. - private boolean hasEmptySpjResultSet_ = false; - public Analyzer(GlobalStateMgr globalStateMgr, ConnectContext context) { ancestors = Lists.newArrayList(); globalState = new GlobalState(globalStateMgr, context); @@ -295,20 +265,6 @@ public int getCallDepth() { return callDepth; } - /** - * Substitute analyzer's internal expressions (conjuncts) with the given - * substitution map - */ - public void substitute(ExprSubstitutionMap sMap) { - for (ExprId id : globalState.conjuncts.keySet()) { - // TODO(dhc): next three lines for subquery - if (globalState.conjuncts.get(id).substitute(sMap) instanceof BoolLiteral) { - continue; - } - globalState.conjuncts.put(id, (Predicate) globalState.conjuncts.get(id).substitute(sMap)); - } - } - /** * Creates an returns an empty TupleDescriptor for the given table ref and registers * it against all its legal aliases. For tables refs with an explicit alias, only the @@ -551,27 +507,6 @@ public boolean isSemiJoined(TupleId tid) { return globalState.semiJoinedTupleIds.containsKey(tid); } - public boolean isAntiJoinedConjunct(Expr e) { - return getAntiJoinRef(e) != null; - } - - public TableRef getAntiJoinRef(Expr e) { - TableRef tblRef = globalState.sjClauseByConjunct.get(e.getId()); - if (tblRef == null) { - return null; - } - return (tblRef.getJoinOp().isAntiJoin()) ? tblRef : null; - } - - public boolean containsOuterJoinedTid(Set tids) { - for (TupleId tid : tids) { - if (isOuterJoined(tid)) { - return true; - } - } - return false; - } - public DescriptorTable getDescTbl() { return globalState.descTbl; } @@ -584,59 +519,6 @@ public Set getAliases() { return uniqueTableAliasSet_; } - public void setHasEmptyResultSet() { - hasEmptyResultSet_ = true; - } - - public boolean isOjConjunct(Expr e) { - return globalState.ojClauseByConjunct.containsKey(e.getId()); - } - - public boolean isIjConjunct(Expr e) { - return globalState.ijClauseByConjunct.containsKey(e.getId()); - } - - public boolean isSjConjunct(Expr e) { - return globalState.sjClauseByConjunct.containsKey(e.getId()); - } - - public TableRef getFullOuterJoinRef(Expr e) { - return globalState.fullOuterJoinedConjuncts.get(e.getId()); - } - - public boolean isFullOuterJoined(Expr e) { - return globalState.fullOuterJoinedConjuncts.containsKey(e.getId()); - } - - /** - * return equal conjuncts, used by OlapScanNode.normalizePredicate and SelectStmt.reorderTable - */ - public List getEqJoinConjuncts(TupleId id) { - final List conjunctIds = globalState.eqJoinConjuncts.get(id); - if (conjunctIds == null) { - return Lists.newArrayList(); - } - final List result = Lists.newArrayList(); - for (ExprId conjunctId : conjunctIds) { - final Expr e = globalState.conjuncts.get(conjunctId); - Preconditions.checkState(e != null); - result.add(e); - } - return result; - } - - public int getCurrentOutputColumn(TupleId id) { - Integer result = currentOutputColumn.get(id); - if (null == result) { - return this.getTupleDesc(id).getSlots().size(); - } - return result; - } - - public void setCurrentOutputColumn(TupleId id, int v) { - currentOutputColumn.put(id, v); - } - /** * Mark predicates as assigned. */ @@ -763,44 +645,6 @@ public boolean canEvalPredicate(List tupleIds, Expr e) { return true; } - if (e.isOnClauseConjunct()) { - - if (isAntiJoinedConjunct(e)) { - return canEvalAntiJoinedConjunct(e, tupleIds); - } - if (isIjConjunct(e) || isSjConjunct(e)) { - if (!containsOuterJoinedTid(tids)) { - return true; - } - // If the predicate references an outer-joined tuple, then evaluate it at - // the join that the On-clause belongs to. - TableRef onClauseTableRef = null; - if (isIjConjunct(e)) { - onClauseTableRef = globalState.ijClauseByConjunct.get(e.getId()); - } else { - onClauseTableRef = globalState.sjClauseByConjunct.get(e.getId()); - } - Preconditions.checkNotNull(onClauseTableRef); - return tupleIds.containsAll(onClauseTableRef.getAllTableRefIds()); - } - - if (isFullOuterJoined(e)) { - return canEvalFullOuterJoinedConjunct(e, tupleIds); - } - if (isOjConjunct(e)) { - // Force this predicate to be evaluated by the corresponding outer join node. - // The join node will pick up the predicate later via getUnassignedOjConjuncts(). - if (tids.size() > 1) { - return false; - } - // Optimization for single-tid predicates: Legal to assign below the outer join - // if the predicate is from the same On-clause that makes tid nullable - // (otherwise e needn't be true when that tuple is set). - TupleId tid = tids.iterator().next(); - return globalState.ojClauseByConjunct.get(e.getId()) == getLastOjClause(tid); - } - } - for (TupleId tid : tids) { TableRef rhsRef = getLastOjClause(tid); // this is not outer-joined; ignore @@ -815,40 +659,4 @@ public boolean canEvalPredicate(List tupleIds, Expr e) { return true; } - - /** - * Checks if a conjunct from the On-clause of an anti join can be evaluated in a node - * that materializes a given list of tuple ids. - */ - public boolean canEvalAntiJoinedConjunct(Expr e, List nodeTupleIds) { - TableRef antiJoinRef = getAntiJoinRef(e); - if (antiJoinRef == null) { - return true; - } - List tids = Lists.newArrayList(); - e.getIds(tids, null); - if (tids.size() > 1) { - return nodeTupleIds.containsAll(antiJoinRef.getAllTableRefIds()) - && antiJoinRef.getAllTableRefIds().containsAll(nodeTupleIds); - } - // A single tid conjunct that is anti-joined can be safely assigned to a - // node below the anti join that specified it. - return globalState.semiJoinedTupleIds.containsKey(tids.get(0)); - } - - /** - * Returns false if 'e' references a full outer joined tuple and it is incorrect to - * evaluate 'e' at a node materializing 'tids'. Returns true otherwise. - */ - public boolean canEvalFullOuterJoinedConjunct(Expr e, List tids) { - TableRef fullOuterJoin = getFullOuterJoinRef(e); - if (fullOuterJoin == null) { - return true; - } - return tids.containsAll(fullOuterJoin.getAllTableRefIds()); - } - - public boolean isOuterJoined(TupleId tid) { - return globalState.outerJoinedTupleIds.containsKey(tid); - } } diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/ArithmeticExpr.java b/fe/fe-core/src/main/java/com/starrocks/analysis/ArithmeticExpr.java index 0f65e77aa9e7d..c445e838c5685 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/ArithmeticExpr.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/ArithmeticExpr.java @@ -36,7 +36,6 @@ import com.starrocks.thrift.TExprOpcode; import java.util.Arrays; -import java.util.List; import java.util.Objects; public class ArithmeticExpr extends Expr { @@ -391,15 +390,6 @@ public boolean equals(Object obj) { return ((ArithmeticExpr) obj).opcode == opcode; } - @Override - public void computeOutputColumn(Analyzer analyzer) { - super.computeOutputColumn(analyzer); - - List tupleIds = Lists.newArrayList(); - getIds(tupleIds, null); - Preconditions.checkArgument(tupleIds.size() == 1); - } - public static Type getCommonType(Type t1, Type t2) { PrimitiveType pt1 = t1.getNumResultType().getPrimitiveType(); PrimitiveType pt2 = t2.getNumResultType().getPrimitiveType(); diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/BaseTableRef.java b/fe/fe-core/src/main/java/com/starrocks/analysis/BaseTableRef.java index 5cf79cc00e716..461f333f56418 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/BaseTableRef.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/BaseTableRef.java @@ -68,9 +68,6 @@ public void analyze(Analyzer analyzer) throws AnalysisException { name.analyze(analyzer); desc = analyzer.registerTableRef(this); isAnalyzed = true; // true that we have assigned desc - analyzeJoin(analyzer); - analyzeSortHints(); - analyzeHints(); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/BaseViewStmt.java b/fe/fe-core/src/main/java/com/starrocks/analysis/BaseViewStmt.java index 1a2c9a9c52c26..ce366d682385d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/BaseViewStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/BaseViewStmt.java @@ -23,12 +23,8 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import com.starrocks.catalog.Column; -import com.starrocks.catalog.Type; import com.starrocks.common.AnalysisException; -import com.starrocks.common.ErrorCode; -import com.starrocks.common.ErrorReport; import com.starrocks.common.UserException; import com.starrocks.sql.ast.AstVisitor; import com.starrocks.sql.ast.QueryStatement; @@ -36,8 +32,6 @@ import org.apache.logging.log4j.Logger; import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; public class BaseViewStmt extends DdlStmt { private static final Logger LOG = LogManager.getLogger(BaseViewStmt.class); @@ -108,66 +102,6 @@ public List getCols() { return cols; } - /** - * Sets the originalViewDef and the expanded inlineViewDef based on viewDefStmt. - * If columnNames were given, checks that they do not contain duplicate column names - * and throws an exception if they do. - */ - protected void createColumnAndViewDefs(Analyzer analyzer) throws AnalysisException, UserException { - if (cols != null) { - if (cols.size() != viewDefStmt.getColLabels().size()) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_VIEW_WRONG_LIST); - } - for (int i = 0; i < cols.size(); ++i) { - Type type = viewDefStmt.getBaseTblResultExprs().get(i).getType().clone(); - Column col = new Column(cols.get(i).getColName(), type); - col.setComment(cols.get(i).getComment()); - finalCols.add(col); - } - } else { - for (int i = 0; i < viewDefStmt.getBaseTblResultExprs().size(); ++i) { - Type type = viewDefStmt.getBaseTblResultExprs().get(i).getType().clone(); - finalCols.add(new Column(viewDefStmt.getColLabels().get(i), type)); - } - } - // Set for duplicate columns - Set colSets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); - for (Column col : finalCols) { - if (!colSets.add(col.getName())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DUP_FIELDNAME, col.getName()); - } - } - - // format view def string - originalViewDef = viewDefStmt.toSql(); - - if (cols == null) { - inlineViewDef = originalViewDef; - return; - } - - Analyzer tmpAnalyzer = new Analyzer(analyzer); - List colNames = cols.stream().map(c -> c.getColName()).collect(Collectors.toList()); - cloneStmt.substituteSelectList(tmpAnalyzer, colNames); - inlineViewDef = cloneStmt.toSql(); - - // StringBuilder sb = new StringBuilder(); - // sb.append("SELECT "); - // for (int i = 0; i < finalCols.size(); ++i) { - // if (i != 0) { - // sb.append(", "); - // } - // String colRef = viewDefStmt.getColLabels().get(i); - // if (!colRef.startsWith("`")) { - // colRef = "`" + colRef + "`"; - // } - // String colAlias = finalCols.get(i).getName(); - // sb.append(String.format("`%s`.%s AS `%s`", tableName.getTbl(), colRef, colAlias)); - // } - // sb.append(String.format(" FROM (%s) %s", originalViewDef, tableName.getTbl())); - // inlineViewDef = sb.toString(); - } - @Override public void analyze(Analyzer analyzer) throws AnalysisException, UserException { super.analyze(analyzer); diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/BinaryPredicate.java b/fe/fe-core/src/main/java/com/starrocks/analysis/BinaryPredicate.java index 50945c78c64ce..265237ff12e05 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/BinaryPredicate.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/BinaryPredicate.java @@ -22,7 +22,6 @@ package com.starrocks.analysis; import com.google.common.base.Preconditions; -import com.starrocks.catalog.Function; import com.starrocks.catalog.PrimitiveType; import com.starrocks.catalog.Type; import com.starrocks.common.AnalysisException; @@ -255,21 +254,6 @@ protected void toThrift(TExprNode msg) { msg.setChild_type(getChild(0).getType().getPrimitiveType().toThrift()); } - @Override - public void vectorizedAnalyze(Analyzer analyzer) { - super.vectorizedAnalyze(analyzer); - Function match = null; - - try { - match = getBuiltinFunction(analyzer, op.name, collectChildReturnTypes(), - Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF); - } catch (AnalysisException e) { - Preconditions.checkState(false); - } - Preconditions.checkState(match != null); - Preconditions.checkState(match.getReturnType().getPrimitiveType() == PrimitiveType.BOOLEAN); - } - private static boolean canCompareDate(PrimitiveType t1, PrimitiveType t2) { if (t1.isDateType()) { return t2.isDateType() || t2.isStringType(); diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/CreateViewStmt.java b/fe/fe-core/src/main/java/com/starrocks/analysis/CreateViewStmt.java index 1aacc7c0deb38..1398ab71192f2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/CreateViewStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/CreateViewStmt.java @@ -22,24 +22,14 @@ package com.starrocks.analysis; import com.google.common.base.Strings; -import com.google.common.collect.Sets; -import com.starrocks.catalog.Column; -import com.starrocks.catalog.Type; import com.starrocks.common.AnalysisException; -import com.starrocks.common.ErrorCode; -import com.starrocks.common.ErrorReport; import com.starrocks.common.UserException; -import com.starrocks.mysql.privilege.PrivPredicate; -import com.starrocks.qe.ConnectContext; -import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.ast.AstVisitor; import com.starrocks.sql.ast.QueryStatement; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; public class CreateViewStmt extends BaseViewStmt { @@ -70,64 +60,6 @@ public String getComment() { @Override public void analyze(Analyzer analyzer) throws AnalysisException, UserException { - tableName.analyze(analyzer); - viewDefStmt.setNeedToSql(true); - - // check privilege - if (!GlobalStateMgr.getCurrentState().getAuth().checkTblPriv(ConnectContext.get(), tableName.getDb(), - tableName.getTbl(), PrivPredicate.CREATE)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "CREATE"); - } - - if (cols != null) { - cloneStmt = viewDefStmt.clone(); - } - - // Analyze view define statement - Analyzer viewAnalyzer = new Analyzer(analyzer); - viewDefStmt.analyze(viewAnalyzer); - - createView(analyzer); - } - - void createView(Analyzer analyzer) throws UserException { - if (cols != null) { - if (cols.size() != viewDefStmt.getColLabels().size()) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_VIEW_WRONG_LIST); - } - for (int i = 0; i < cols.size(); ++i) { - Type type = viewDefStmt.getBaseTblResultExprs().get(i).getType().clone(); - Column col = new Column(cols.get(i).getColName(), type); - col.setComment(cols.get(i).getComment()); - finalCols.add(col); - } - } else { - for (int i = 0; i < viewDefStmt.getBaseTblResultExprs().size(); ++i) { - Type type = viewDefStmt.getBaseTblResultExprs().get(i).getType().clone(); - finalCols.add(new Column(viewDefStmt.getColLabels().get(i), type)); - } - } - // Set for duplicate columns - Set colSets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); - for (Column col : finalCols) { - if (!colSets.add(col.getName())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DUP_FIELDNAME, col.getName()); - } - } - - // format view def string - originalViewDef = viewDefStmt.toSql(); - - if (cols == null) { - inlineViewDef = originalViewDef; - return; - } - - // It's different with createColumnAndViewDefs in here, cloneStmt is origin stmt which analyze without set `setNeedSql` - Analyzer tmpAnalyzer = new Analyzer(analyzer); - List colNames = cols.stream().map(ColWithComment::getColName).collect(Collectors.toList()); - viewDefStmt.substituteSelectListForCreateView(tmpAnalyzer, colNames); - inlineViewDef = viewDefStmt.toSql(); } public R accept(AstVisitor visitor, C context) { diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/Expr.java b/fe/fe-core/src/main/java/com/starrocks/analysis/Expr.java index 587ea28b08c90..ec111ef90ea46 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/Expr.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/Expr.java @@ -265,15 +265,6 @@ public int getOutputColumn() { public boolean isFilter() { return isFilter; } - - public boolean isOnClauseConjunct() { - return isOnClauseConjunct_; - } - - public void setIsOnClauseConjunct(boolean b) { - isOnClauseConjunct_ = b; - } - public boolean isAuxExpr() { return isAuxExpr; } @@ -642,31 +633,6 @@ public static void getIds(List exprs, List tupleIds = Lists.newArrayList(); - getIds(tupleIds, null); - Preconditions.checkArgument(tupleIds.size() == 1); - - int currentOutputColumn = analyzer.getCurrentOutputColumn(tupleIds.get(0)); - this.outputColumn = currentOutputColumn; - LOG.info(debugString() + " outputColumn: " + this.outputColumn); - ++currentOutputColumn; - analyzer.setCurrentOutputColumn(tupleIds.get(0), currentOutputColumn); - } - } - public String toSql() { return (printSqlInParens) ? "(" + toSqlImpl() + ")" : toSqlImpl(); } @@ -1324,18 +1290,6 @@ public Subquery getSubquery() { return subqueries.get(0); } - public boolean isCorrelatedPredicate(List tupleIdList) { - if (this instanceof SlotRef && !this.isBoundByTupleIds(tupleIdList)) { - return true; - } - for (Expr child : this.getChildren()) { - if (child.isCorrelatedPredicate(tupleIdList)) { - return true; - } - } - return false; - } - @Override public void write(DataOutput out) throws IOException { throw new IOException("Not implemented serializable "); diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/FromClause.java b/fe/fe-core/src/main/java/com/starrocks/analysis/FromClause.java index 425e7307a0714..fabc9cbde30bd 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/FromClause.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/FromClause.java @@ -58,10 +58,6 @@ public List getTableRefs() { return tableRefs_; } - public void setNeedToSql(boolean needToSql) { - this.needToSql = needToSql; - } - @Override public void analyze(Analyzer analyzer) throws AnalysisException, UserException { if (analyzed_) { diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/InPredicate.java b/fe/fe-core/src/main/java/com/starrocks/analysis/InPredicate.java index b5b0a6be44d13..517b2333ac65b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/InPredicate.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/InPredicate.java @@ -22,9 +22,6 @@ package com.starrocks.analysis; import com.google.common.base.Preconditions; -import com.starrocks.catalog.Function; -import com.starrocks.catalog.PrimitiveType; -import com.starrocks.catalog.Type; import com.starrocks.common.AnalysisException; import com.starrocks.sql.analyzer.SemanticException; import com.starrocks.sql.ast.AstVisitor; @@ -33,7 +30,6 @@ import com.starrocks.thrift.TExprOpcode; import com.starrocks.thrift.TInPredicate; -import java.util.ArrayList; import java.util.List; /** @@ -107,64 +103,8 @@ public boolean isLiteralChildren() { return true; } - @Override - public void vectorizedAnalyze(Analyzer analyzer) { - super.vectorizedAnalyze(analyzer); - - PrimitiveType type = getChild(0).getType().getPrimitiveType(); - } - @Override public void analyzeImpl(Analyzer analyzer) throws AnalysisException { - super.analyzeImpl(analyzer); - - if (contains(Subquery.class)) { - // An [NOT] IN predicate with a subquery must contain two children, the second of - // which is a Subquery. - if (children.size() != 2 || !(getChild(1) instanceof Subquery)) { - throw new AnalysisException("Unsupported IN predicate with a subquery: " + - toSql()); - } - Subquery subquery = (Subquery) getChild(1); - if (!subquery.returnsScalarColumn()) { - throw new AnalysisException("Subquery must return a single column: " + - subquery.toSql()); - } - - // Ensure that the column in the lhs of the IN predicate and the result of - // the subquery are type compatible. No need to perform any - // casting at this point. Any casting needed will be performed when the - // subquery is unnested. - ArrayList subqueryExprs = subquery.getStatement().getResultExprs(); - Expr compareExpr = children.get(0); - Expr subqueryExpr = subqueryExprs.get(0); - analyzer.getCompatibleType(compareExpr.getType(), compareExpr, subqueryExpr); - } else { - analyzer.castAllToCompatibleType(children); - vectorizedAnalyze(analyzer); - } - - boolean allConstant = true; - for (int i = 1; i < children.size(); ++i) { - if (!children.get(i).isConstant()) { - allConstant = false; - break; - } - } - boolean useSetLookup = allConstant; - // Only lookup fn_ if all subqueries have been rewritten. If the second child is a - // subquery, it will have type ArrayType, which cannot be resolved to a builtin - // function and will fail analysis. - Type[] argTypes = {getChild(0).type, getChild(1).type}; - if (useSetLookup) { - opcode = isNotIn ? TExprOpcode.FILTER_NOT_IN : TExprOpcode.FILTER_IN; - } else { - fn = getBuiltinFunction(analyzer, isNotIn ? NOT_IN_ITERATE : IN_ITERATE, - argTypes, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF); - opcode = isNotIn ? TExprOpcode.FILTER_NEW_NOT_IN : TExprOpcode.FILTER_NEW_IN; - } - - selectivity = Expr.DEFAULT_SELECTIVITY; } @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/IndexDef.java b/fe/fe-core/src/main/java/com/starrocks/analysis/IndexDef.java index 2bc490d721bb4..d82458d44f00b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/IndexDef.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/IndexDef.java @@ -141,24 +141,6 @@ public void checkColumn(Column column, KeysType keysType) { } } - public void verifyColumn(Column column, KeysType keysType) { - if (indexType == IndexType.BITMAP) { - String indexColName = column.getName(); - PrimitiveType colType = column.getPrimitiveType(); - if (!(colType.isDateType() || - colType.isFixedPointType() || colType.isStringType() || colType == PrimitiveType.BOOLEAN)) { - throw new SemanticException(colType + " is not supported in bitmap index. " - + "invalid column: " + indexColName); - } else if ((keysType == KeysType.AGG_KEYS || keysType == KeysType.UNIQUE_KEYS) && !column.isKey()) { - throw new SemanticException( - "BITMAP index only used in columns of DUP_KEYS/PRIMARY_KEYS table or key columns of" - + " UNIQUE_KEYS/AGG_KEYS table. invalid column: " + indexColName); - } - } else { - throw new SemanticException("Unsupported index type: " + indexType); - } - } - public enum IndexType { BITMAP, } diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/LimitElement.java b/fe/fe-core/src/main/java/com/starrocks/analysis/LimitElement.java index 31a3381b4f068..1fd11fd462cd9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/LimitElement.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/LimitElement.java @@ -112,9 +112,6 @@ public String toDigest() { } public void analyze(Analyzer analyzer) { - if (limit == 0) { - analyzer.setHasEmptyResultSet(); - } } public void reset() { diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/QueryStmt.java b/fe/fe-core/src/main/java/com/starrocks/analysis/QueryStmt.java index 4569635a57fc0..20391430ef212 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/QueryStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/QueryStmt.java @@ -29,8 +29,6 @@ import com.starrocks.common.ErrorReport; import com.starrocks.common.UserException; import com.starrocks.qe.ConnectContext; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.List; @@ -217,8 +215,6 @@ public void getWithClauseDbs(ConnectContext context, Map dbs) */ public abstract void collectTableRefs(List tblRefs); - abstract List collectTupleIds(); - public ArrayList getOrderByElements() { return orderByElements; } @@ -359,14 +355,6 @@ public void setFromInsert(boolean value) { @Override public abstract QueryStmt clone(); - public abstract void substituteSelectList(Analyzer analyzer, List newColLabels) - throws AnalysisException, UserException; - - public void substituteSelectListForCreateView(Analyzer analyzer, List newColLabels) - throws AnalysisException, UserException { - substituteSelectList(analyzer, newColLabels); - } - public void setOutFileClause(OutFileClause outFileClause) { this.outFileClause = outFileClause; } diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/SelectStmt.java b/fe/fe-core/src/main/java/com/starrocks/analysis/SelectStmt.java index 77ee25278b70a..6d96d4718d855 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/SelectStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/SelectStmt.java @@ -21,28 +21,22 @@ package com.starrocks.analysis; -import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; -import com.starrocks.catalog.Column; import com.starrocks.catalog.Database; import com.starrocks.catalog.View; import com.starrocks.cluster.ClusterNamespace; import com.starrocks.common.AnalysisException; import com.starrocks.common.ErrorCode; import com.starrocks.common.ErrorReport; -import com.starrocks.common.TableAliasGenerator; import com.starrocks.common.UserException; import com.starrocks.common.util.SqlUtils; import com.starrocks.common.util.UUIDUtil; import com.starrocks.mysql.privilege.PrivPredicate; import com.starrocks.qe.ConnectContext; import com.starrocks.server.GlobalStateMgr; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.UUID; @@ -65,8 +59,6 @@ public class SelectStmt extends QueryStmt { // private Expr havingClause; // original having clause protected Expr whereClause; - // havingClause with aliases and agg output resolved - private Expr havingPred; // set if we have any kind of aggregation operation, include SELECT DISTINCT private AggregateInfo aggInfo; @@ -78,11 +70,6 @@ public class SelectStmt extends QueryStmt { private ValueList valueList; - // having clause which has been analyzed - // For example: select k1, sum(k2) a from t group by k1 having a>1; - // this parameter: sum(t.k2) > 1 - private Expr havingClauseAfterAnaylzed; - // END: Members that need to be reset() // /////////////////////////////////////// @@ -90,9 +77,6 @@ public class SelectStmt extends QueryStmt { // Set in analyze(). protected String sqlString_; - // Table alias generator used during query rewriting. - private TableAliasGenerator tableAliasGenerator = null; - public SelectStmt(ValueList valueList, ArrayList orderByElement, LimitElement limitElement) { super(orderByElement, limitElement); this.valueList = valueList; @@ -121,7 +105,6 @@ public SelectStmt( this.havingClause = havingPredicate; this.colLabels = Lists.newArrayList(); - this.havingPred = null; this.aggInfo = null; this.sortInfo = null; } @@ -157,8 +140,6 @@ public void reset() { if (havingClause != null) { havingClause.reset(); } - havingClauseAfterAnaylzed = null; - havingPred = null; aggInfo = null; analyticInfo = null; baseTblSmap.clear(); @@ -272,70 +253,6 @@ private boolean isViewTableRef(TableRef tblRef) { public void analyze(Analyzer analyzer) throws AnalysisException, UserException { } - public List getTableRefIds() { - List result = Lists.newArrayList(); - - for (TableRef ref : fromClause_) { - result.add(ref.getId()); - } - - return result; - } - - @Override - public List collectTupleIds() { - List result = Lists.newArrayList(); - resultExprs.stream().forEach(expr -> expr.getIds(result, null)); - result.addAll(getTableRefIds()); - if (whereClause != null) { - whereClause.getIds(result, null); - } - if (havingClauseAfterAnaylzed != null) { - havingClauseAfterAnaylzed.getIds(result, null); - } - return result; - } - - /** - * Expand "*" select list item. - */ - private void expandStar(Analyzer analyzer) throws AnalysisException { - if (fromClause_.isEmpty()) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_TABLES_USED); - } - // expand in From clause order - for (TableRef tableRef : fromClause_) { - if (analyzer.isSemiJoined(tableRef.getId())) { - continue; - } - expandStar(tableRef.getAliasAsName(), tableRef.getDesc()); - } - } - - /** - * Expand ".*" select list item. - */ - private void expandStar(Analyzer analyzer, TableName tblName) throws AnalysisException { - Collection descs = analyzer.getDescriptor(tblName); - if (descs == null || descs.isEmpty()) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR, tblName.getTbl()); - } - for (TupleDescriptor desc : descs) { - expandStar(tblName, desc); - } - } - - /** - * Expand "*" for a particular tuple descriptor by appending - * refs for each column to selectListExprs. - */ - private void expandStar(TableName tblName, TupleDescriptor desc) { - for (Column col : desc.getTable().getBaseSchema()) { - resultExprs.add(new SlotRef(tblName, col.getName())); - colLabels.add(col.getName()); - } - } - @Override public String toSql() { if (sqlString_ != null) { @@ -501,75 +418,6 @@ public String toDigest() { return strBuilder.toString(); } - @Override - public void substituteSelectList(Analyzer analyzer, List newColLabels) - throws AnalysisException, UserException { - // analyze with clause - if (hasWithClause()) { - withClause_.analyze(analyzer); - } - // start out with table refs to establish aliases - TableRef leftTblRef = null; // the one to the left of tblRef - for (int i = 0; i < fromClause_.size(); ++i) { - // Resolve and replace non-InlineViewRef table refs with a BaseTableRef or ViewRef. - TableRef tblRef = fromClause_.get(i); - tblRef = analyzer.resolveTableRef(tblRef); - Preconditions.checkNotNull(tblRef); - fromClause_.set(i, tblRef); - tblRef.setLeftTblRef(leftTblRef); - tblRef.analyze(analyzer); - leftTblRef = tblRef; - } - // populate selectListExprs, aliasSMap, and colNames - for (SelectListItem item : selectList.getItems()) { - if (item.isStar()) { - TableName tblName = item.getTblName(); - if (tblName == null) { - expandStar(analyzer); - } else { - expandStar(analyzer, tblName); - } - } else { - // to make sure the sortinfo's AnalyticExpr and resultExprs's AnalyticExpr analytic once - if (item.getExpr() instanceof AnalyticExpr) { - item.getExpr().analyze(analyzer); - } - if (item.getAlias() != null) { - SlotRef aliasRef = new SlotRef(null, item.getAlias()); - SlotRef newAliasRef = new SlotRef(null, newColLabels.get(resultExprs.size())); - newAliasRef.analysisDone(); - aliasSMap.put(aliasRef, newAliasRef); - } - resultExprs.add(item.getExpr()); - } - } - // substitute group by - if (groupByClause != null) { - substituteOrdinalsAliases(groupByClause.getGroupingExprs(), "GROUP BY", analyzer); - } - // substitute having - if (havingClause != null) { - havingClause = havingClause.clone(aliasSMap); - } - // substitute order by - if (orderByElements != null) { - for (int i = 0; i < orderByElements.size(); ++i) { - orderByElements = OrderByElement.substitute(orderByElements, aliasSMap, analyzer); - } - } - - colLabels.clear(); - colLabels.addAll(newColLabels); - } - - @Override - public void substituteSelectListForCreateView(Analyzer analyzer, List newColLabels) - throws UserException { - resultExprs = Lists.newArrayList(); - substituteSelectList(analyzer, newColLabels); - sqlString_ = null; - } - public boolean hasAggInfo() { return aggInfo != null; } diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/SetOperationStmt.java b/fe/fe-core/src/main/java/com/starrocks/analysis/SetOperationStmt.java index d3a818abe85d9..c87db36edfcbd 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/SetOperationStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/SetOperationStmt.java @@ -180,27 +180,15 @@ public List getAllOperands() { return allOperands_; } - public AggregateInfo getDistinctAggInfo() { - return distinctAggInfo; - } - public boolean hasAnalyticExprs() { return hasAnalyticExprs_; } - public TupleId getTupleId() { - return tupleId; - } - public void removeAllOperands() { operands.removeAll(allOperands_); allOperands_.clear(); } - public List getSetOpsResultExprs() { - return setOpsResultExprs_; - } - @Override public void getDbs(ConnectContext context, Map dbs) throws AnalysisException { getWithClauseDbs(context, dbs); @@ -217,74 +205,6 @@ public void getDbs(ConnectContext context, Map dbs) throws Ana public void analyze(Analyzer analyzer) throws UserException { } - /** - * Analyzes all operands and checks that they return an equal number of exprs. - * Throws an AnalysisException if that is not the case, or if analyzing - * an operand fails. - */ - private void analyzeOperands(Analyzer analyzer) throws AnalysisException, UserException { - for (int i = 0; i < operands.size(); ++i) { - operands.get(i).analyze(analyzer); - QueryStmt firstQuery = operands.get(0).getQueryStmt(); - List firstExprs = operands.get(0).getQueryStmt().getResultExprs(); - QueryStmt query = operands.get(i).getQueryStmt(); - List exprs = query.getResultExprs(); - if (firstExprs.size() != exprs.size()) { - throw new AnalysisException("Operands have unequal number of columns:\n" + - "'" + queryStmtToSql(firstQuery) + "' has " + - firstExprs.size() + " column(s)\n" + - "'" + queryStmtToSql(query) + "' has " + exprs.size() + " column(s)"); - } - } - } - - /** - * Fill distinct-/allOperands and performs possible unnesting of SetOperationStmt - * operands in the process. - */ - private void unnestOperands(Analyzer analyzer) throws AnalysisException { - if (operands.size() == 1) { - // ValuesStmt for a single row. - allOperands_.add(operands.get(0)); - return; - } - // find index of first ALL operand - int firstAllIdx = operands.size(); - for (int i = 1; i < operands.size(); ++i) { - SetOperand operand = operands.get(i); - if (operand.getQualifier() == Qualifier.ALL) { - firstAllIdx = (i == 1 ? 0 : i); - break; - } - } - // operands[0] is always implicitly ALL, so operands[1] can't be the - // first one - Preconditions.checkState(firstAllIdx != 1); - - // unnest DISTINCT operands - Preconditions.checkState(distinctOperands_.isEmpty()); - for (int i = 0; i < firstAllIdx; ++i) { - unnestOperand(distinctOperands_, Qualifier.DISTINCT, operands.get(i)); - } - - // unnest ALL operands - Preconditions.checkState(allOperands_.isEmpty()); - for (int i = firstAllIdx; i < operands.size(); ++i) { - unnestOperand(allOperands_, Qualifier.ALL, operands.get(i)); - } - - for (SetOperand op : distinctOperands_) { - op.setQualifier(Qualifier.DISTINCT); - } - for (SetOperand op : allOperands_) { - op.setQualifier(Qualifier.ALL); - } - - operands.clear(); - operands.addAll(distinctOperands_); - operands.addAll(allOperands_); - } - /** * Add a single operand to the target list; if the operand itself is a SetOperationStmt, apply * unnesting to the extent possible (possibly modifying 'operand' in the process). @@ -326,23 +246,6 @@ private void unnestOperand( } } - /** - * Sets the smap for the given operand. It maps from the output slots this SetOps's - * tuple to the corresponding result exprs of the operand. - */ - private void setOperandSmap(SetOperand operand, Analyzer analyzer) { - TupleDescriptor tupleDesc = analyzer.getDescTbl().getTupleDesc(tupleId); - // operands' smaps were already set in the operands' analyze() - operand.getSmap().clear(); - List resultExprs = operand.getQueryStmt().getResultExprs(); - Preconditions.checkState(resultExprs.size() == tupleDesc.getSlots().size()); - for (int i = 0; i < tupleDesc.getSlots().size(); ++i) { - SlotDescriptor outputSlot = tupleDesc.getSlots().get(i); - // Map to the original (uncast) result expr of the operand. - Expr origExpr = resultExprs.get(i).unwrapExpr(true).clone(); - operand.getSmap().put(new SlotRef(outputSlot), origExpr); - } - } /** * String representation of queryStmt used in reporting errors. @@ -352,92 +255,6 @@ protected String queryStmtToSql(QueryStmt queryStmt) { return queryStmt.toSql(); } - /** - * Propagates DISTINCT (if present) from right to left. - * Implied associativity: - * A UNION ALL B UNION DISTINCT C = (A UNION ALL B) UNION DISTINCT C - * = A UNION DISTINCT B UNION DISTINCT C - */ - private void propagateDistinct() { - int firstDistinctPos = -1; - for (int i = operands.size() - 1; i > 0; --i) { - SetOperand operand = operands.get(i); - if (firstDistinctPos != -1) { - // There is a DISTINCT somewhere to the right. - operand.setQualifier(Qualifier.DISTINCT); - } else if (operand.getQualifier() == Qualifier.DISTINCT) { - firstDistinctPos = i; - } - } - } - - /** - * Create a descriptor for the tuple materialized by the setOps. - * Set resultExprs to be slot refs into that tuple. - * Also fills the substitution map, such that "order by" can properly resolve - * column references from the result of the setOps. - */ - private void createMetadata(Analyzer analyzer) throws AnalysisException { - // Create tuple descriptor for materialized tuple created by the setOps. - TupleDescriptor tupleDesc = analyzer.getDescTbl().createTupleDescriptor("SetOps"); - tupleDesc.setIsMaterialized(true); - tupleId = tupleDesc.getId(); - if (LOG.isTraceEnabled()) { - LOG.trace("SetOperationStmt.createMetadata: tupleId=" + tupleId.toString()); - } - - // One slot per expr in the select blocks. Use first select block as representative. - List firstSelectExprs = operands.get(0).getQueryStmt().getResultExprs(); - - // Create tuple descriptor and slots. - for (int i = 0; i < firstSelectExprs.size(); ++i) { - Expr expr = firstSelectExprs.get(i); - SlotDescriptor slotDesc = analyzer.addSlotDescriptor(tupleDesc); - slotDesc.setLabel(getColLabels().get(i)); - slotDesc.setType(expr.getType()); - SlotRef outputSlotRef = new SlotRef(slotDesc); - resultExprs.add(outputSlotRef); - - // Add to aliasSMap so that column refs in "order by" can be resolved. - if (orderByElements != null) { - SlotRef aliasRef = new SlotRef(null, getColLabels().get(i)); - if (aliasSMap.containsMappingFor(aliasRef)) { - ambiguousAliasList.add(aliasRef); - } else { - aliasSMap.put(aliasRef, outputSlotRef); - } - } - - boolean isNullable = false; - // register single-directional value transfers from output slot - // to operands' result exprs (if those happen to be slotrefs); - // don't do that if the operand computes analytic exprs - // (see Planner.createInlineViewPlan() for the reasoning) - for (SetOperand op : operands) { - Expr resultExpr = op.getQueryStmt().getResultExprs().get(i); - slotDesc.addSourceExpr(resultExpr); - SlotRef slotRef = resultExpr.unwrapSlotRef(false); - if (slotRef == null || slotRef.getDesc().getIsNullable() - || analyzer.isOuterJoined(slotRef.getDesc().getParent().getId()) - || resultExpr.isNullable()) { - isNullable = true; - } - if (op.hasAnalyticExprs()) { - continue; - } - slotRef = resultExpr.unwrapSlotRef(true); - if (slotRef == null) { - continue; - } - // analyzer.registerValueTransfer(outputSlotRef.getSlotId(), slotRef.getSlotId()); - } - // If all the child slots are not nullable, then the SetOps output slot should not - // be nullable as well. - slotDesc.setIsNullable(isNullable); - } - baseTblResultExprs = resultExprs; - } - @Override public void collectTableRefs(List tblRefs) { for (SetOperand op : operands) { @@ -445,15 +262,6 @@ public void collectTableRefs(List tblRefs) { } } - @Override - public List collectTupleIds() { - List result = Lists.newArrayList(); - for (SetOperand op : operands) { - result.addAll(op.getQueryStmt().collectTupleIds()); - } - return result; - } - @Override public String toSql() { if (toSqlString != null) { @@ -575,34 +383,6 @@ public void setNeedToSql(boolean needToSql) { } } - @Override - public void substituteSelectList(Analyzer analyzer, List newColLabels) - throws UserException { - for (SetOperand operand : operands) { - operand.getQueryStmt().substituteSelectList(analyzer, newColLabels); - } - // substitute order by - if (orderByElements != null) { - orderByElements = OrderByElement.substitute(orderByElements, - operands.get(0).getQueryStmt().aliasSMap, analyzer); - } - } - - @Override - public void substituteSelectListForCreateView(Analyzer analyzer, List newColLabels) - throws UserException { - for (SetOperand operand : operands) { - operand.getQueryStmt().substituteSelectListForCreateView(analyzer, newColLabels); - } - // substitute order by - if (orderByElements != null) { - orderByElements = OrderByElement.substitute(orderByElements, - operands.get(0).getQueryStmt().aliasSMap, analyzer); - } - - toSqlString = null; - } - /** * Represents an operand to a SetOperand. It consists of a query statement and its left * all/distinct qualifier (null for the first operand). diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/SlotRef.java b/fe/fe-core/src/main/java/com/starrocks/analysis/SlotRef.java index da164503723c1..bae6b3c7c4276 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/SlotRef.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/SlotRef.java @@ -133,17 +133,6 @@ public void setDesc(SlotDescriptor desc) { this.desc = desc; } - @Override - public void vectorizedAnalyze(Analyzer analyzer) { - computeOutputColumn(analyzer); - } - - @Override - public void computeOutputColumn(Analyzer analyzer) { - outputColumn = desc.getSlotOffset(); - LOG.debug("SlotRef: " + debugString() + " outputColumn: " + outputColumn); - } - @Override public void analyzeImpl(Analyzer analyzer) throws AnalysisException { } diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/Subquery.java b/fe/fe-core/src/main/java/com/starrocks/analysis/Subquery.java index 3c2b6695e060e..89558a51f729d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/Subquery.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/Subquery.java @@ -32,7 +32,6 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.List; /** * Class representing a subquery. A Subquery consists of a QueryStmt and has @@ -128,17 +127,6 @@ public boolean returnsScalarColumn() { return false; } - @Override - public boolean isCorrelatedPredicate(List tupleIdList) { - List tupleIdFromSubquery = stmt.collectTupleIds(); - for (TupleId tupleId : tupleIdList) { - if (tupleIdFromSubquery.contains(tupleId)) { - return true; - } - } - return false; - } - /** * Returns true if the toSql() of the Subqueries is identical. May return false for * equivalent statements even due to minor syntactic differences like parenthesis. diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/TableRef.java b/fe/fe-core/src/main/java/com/starrocks/analysis/TableRef.java index 262f9a70cd150..42aa8190c065c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/TableRef.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/TableRef.java @@ -24,7 +24,6 @@ import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import com.starrocks.catalog.Table; import com.starrocks.common.AnalysisException; import com.starrocks.common.ErrorCode; import com.starrocks.common.ErrorReport; @@ -243,30 +242,6 @@ public void setName(TableName name) { this.name = name; } - /** - * This method should only be called after the TableRef has been analyzed. - */ - public TupleDescriptor getDesc() { - Preconditions.checkState(isAnalyzed); - // after analyze(), desc should be set. - Preconditions.checkState(desc != null); - return desc; - } - - /** - * This method should only be called after the TableRef has been analyzed. - */ - public TupleId getId() { - Preconditions.checkState(isAnalyzed); - // after analyze(), desc should be set. - Preconditions.checkState(desc != null); - return desc.getId(); - } - - public Table getTable() { - return desc.getTable(); - } - public void setUsingClause(List colNames) { this.usingColNames = colNames; } @@ -292,33 +267,6 @@ public void setLateral(boolean lateral) { isLateral = lateral; } - protected void analyzeSortHints() throws AnalysisException { - } - - /** - * Parse PreAgg hints. - */ - protected void analyzeHints() throws AnalysisException { - if (commonHints == null || commonHints.isEmpty()) { - return; - } - // Currently only 'PREAGGOPEN' is supported - for (String hint : commonHints) { - if (hint.equalsIgnoreCase("PREAGGOPEN")) { - isForcePreAggOpened = true; - break; - } - } - } - - /** - * Analyze the join clause. - * The join clause can only be analyzed after the left table has been analyzed - * and the TupleDescriptor (desc) of this table has been created. - */ - public void analyzeJoin(Analyzer analyzer) throws AnalysisException { - } - private String joinOpToSql() { Preconditions.checkState(joinOp != null); switch (joinOp) { diff --git a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadingTaskPlanner.java b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadingTaskPlanner.java index 77e479c66f12f..d96b11de5ab26 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadingTaskPlanner.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadingTaskPlanner.java @@ -107,7 +107,6 @@ public LoadingTaskPlanner(Long loadJobId, long txnId, long dbId, OlapTable table this.partialUpdate = partialUpdate; this.parallelInstanceNum = Config.load_parallel_instance_num; this.startTime = startTime; - this.analyzer.setUDFAllowed(false); } public void plan(TUniqueId loadId, List> fileStatusesList, int filesAdded) diff --git a/fe/fe-core/src/main/java/com/starrocks/planner/AnalyticEvalNode.java b/fe/fe-core/src/main/java/com/starrocks/planner/AnalyticEvalNode.java index 2dd24796f7a00..e24429801a285 100644 --- a/fe/fe-core/src/main/java/com/starrocks/planner/AnalyticEvalNode.java +++ b/fe/fe-core/src/main/java/com/starrocks/planner/AnalyticEvalNode.java @@ -58,10 +58,6 @@ public class AnalyticEvalNode extends PlanNode { private final TupleDescriptor intermediateTupleDesc; private final TupleDescriptor outputTupleDesc; - // maps from the logical output slots in logicalTupleDesc_ to their corresponding - // physical output slots in outputTupleDesc_ - private final ExprSubstitutionMap logicalToPhysicalSmap; - // predicates constructed from partitionExprs_/orderingExprs_ to // compare input to buffered tuples private final Expr partitionByEq; @@ -84,7 +80,6 @@ public AnalyticEvalNode( this.analyticWindow = analyticWindow; this.intermediateTupleDesc = intermediateTupleDesc; this.outputTupleDesc = outputTupleDesc; - this.logicalToPhysicalSmap = logicalToPhysicalSmap; this.partitionByEq = partitionByEq; this.orderByEq = orderByEq; this.bufferedTupleDesc = bufferedTupleDesc; diff --git a/fe/fe-core/src/main/java/com/starrocks/planner/OlapScanNode.java b/fe/fe-core/src/main/java/com/starrocks/planner/OlapScanNode.java index cca46ae4069fb..9b023fbb55bdb 100644 --- a/fe/fe-core/src/main/java/com/starrocks/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/com/starrocks/planner/OlapScanNode.java @@ -390,20 +390,8 @@ private void computePartitionInfo() throws AnalysisException { } public void selectBestRollupByRollupSelector(Analyzer analyzer) throws UserException { - // Step2: select best rollup - long start = System.currentTimeMillis(); - if (olapTable.getKeysType() == KeysType.DUP_KEYS) { - //This function is compatible with the INDEX selection logic of ROLLUP, - //so the Duplicate table here returns base index directly - //and the selection logic of materialized view is selected in "MaterializedViewSelector" - selectedIndexId = olapTable.getBaseIndexId(); - LOG.debug("The best index will be selected later in mv selector"); - return; - } - final RollupSelector rollupSelector = new RollupSelector(analyzer, desc, olapTable); - selectedIndexId = rollupSelector.selectBestRollup(selectedPartitionIds, conjuncts, isPreAggregation); - LOG.debug("select best roll up cost: {} ms, best index id: {}", - (System.currentTimeMillis() - start), selectedIndexId); + selectedIndexId = olapTable.getBaseIndexId(); + LOG.debug("The best index will be selected later in mv selector"); } private void getScanRangeLocations() throws UserException { diff --git a/fe/fe-core/src/main/java/com/starrocks/planner/PlanNode.java b/fe/fe-core/src/main/java/com/starrocks/planner/PlanNode.java index c200c5138019d..329bfaa0f7ea5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/planner/PlanNode.java +++ b/fe/fe-core/src/main/java/com/starrocks/planner/PlanNode.java @@ -25,7 +25,6 @@ import com.google.common.base.Predicates; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.google.common.math.LongMath; import com.starrocks.analysis.Analyzer; import com.starrocks.analysis.Expr; import com.starrocks.analysis.ExprSubstitutionMap; @@ -156,19 +155,6 @@ protected PlanNode(PlanNodeId id, PlanNode node, String planNodeName) { this.numInstances = 1; } - /** - * Computes and returns the sum of two cardinalities. If an overflow occurs, - * the maximum Long value is returned (Long.MAX_VALUE). - */ - public static long addCardinalities(long a, long b) { - try { - return LongMath.checkedAdd(a, b); - } catch (ArithmeticException e) { - LOG.warn("overflow when adding cardinalities: " + a + ", " + b); - return Long.MAX_VALUE; - } - } - public List getProbeRuntimeFilters() { return probeRuntimeFilters; } @@ -566,25 +552,10 @@ public void computeStatistics(Statistics statistics) { columnStatistics = statistics.getColumnStatistics(); } - /** - * Compute the product of the selectivies of all conjuncts. - */ - protected double computeSelectivity() { - double prod = 1.0; - for (Expr e : conjuncts) { - prod *= e.getSelectivity(); - } - return prod; - } - public ExprSubstitutionMap getOutputSmap() { return outputSmap; } - public ExprSubstitutionMap getWithoutTupleIsNullOutputSmap() { - return withoutTupleIsNullOutputSmap == null ? outputSmap : withoutTupleIsNullOutputSmap; - } - public void init(Analyzer analyzer) throws UserException { } diff --git a/fe/fe-core/src/main/java/com/starrocks/planner/Planner.java b/fe/fe-core/src/main/java/com/starrocks/planner/Planner.java deleted file mode 100644 index e1bfbaaa69787..0000000000000 --- a/fe/fe-core/src/main/java/com/starrocks/planner/Planner.java +++ /dev/null @@ -1,69 +0,0 @@ -// This file is made available under Elastic License 2.0. -// This file is based on code available under the Apache license here: -// https://github.com/apache/incubator-doris/blob/master/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.starrocks.planner; - -import com.google.common.collect.Lists; -import com.starrocks.thrift.TExplainLevel; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.ArrayList; -import java.util.List; - -@Deprecated -public class Planner { - private static final Logger LOG = LogManager.getLogger(Planner.class); - - private boolean isBlockQuery = false; - - private ArrayList fragments = Lists.newArrayList(); - - public boolean isBlockQuery() { - return isBlockQuery; - } - - public List getFragments() { - return fragments; - } - - /** - * Return combined explain string for all plan fragments. - */ - public String getExplainString(List fragments, TExplainLevel explainLevel) { - StringBuilder str = new StringBuilder(); - for (int i = 0; i < fragments.size(); ++i) { - PlanFragment fragment = fragments.get(i); - if (i > 0) { - // a blank line between plan fragments - str.append("\n"); - } - if (explainLevel.equals(TExplainLevel.NORMAL)) { - str.append("PLAN FRAGMENT ").append(i).append("\n"); - str.append(fragment.getExplainString(TExplainLevel.NORMAL)); - } else { - str.append("PLAN FRAGMENT ").append(i).append("(").append(fragment.getFragmentId()).append(")\n"); - str.append(fragment.getVerboseExplain()); - } - } - return str.toString(); - } -} diff --git a/fe/fe-core/src/main/java/com/starrocks/planner/RollupSelector.java b/fe/fe-core/src/main/java/com/starrocks/planner/RollupSelector.java deleted file mode 100644 index df97ca1d20527..0000000000000 --- a/fe/fe-core/src/main/java/com/starrocks/planner/RollupSelector.java +++ /dev/null @@ -1,309 +0,0 @@ -// This file is made available under Elastic License 2.0. -// This file is based on code available under the Apache license here: -// https://github.com/apache/incubator-doris/blob/master/fe/fe-core/src/main/java/org/apache/doris/planner/RollupSelector.java - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.starrocks.planner; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import com.starrocks.alter.MaterializedViewHandler; -import com.starrocks.analysis.Analyzer; -import com.starrocks.analysis.BinaryPredicate; -import com.starrocks.analysis.CastExpr; -import com.starrocks.analysis.Expr; -import com.starrocks.analysis.InPredicate; -import com.starrocks.analysis.SlotDescriptor; -import com.starrocks.analysis.SlotRef; -import com.starrocks.analysis.TupleDescriptor; -import com.starrocks.catalog.Column; -import com.starrocks.catalog.MaterializedIndex; -import com.starrocks.catalog.OlapTable; -import com.starrocks.common.UserException; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Set; - -@Deprecated -public final class RollupSelector { - private static final Logger LOG = LogManager.getLogger(RollupSelector.class); - - // Rollup's table info. - private final TupleDescriptor tupleDesc; - private final OlapTable table; - private final Analyzer analyzer; - - public RollupSelector(Analyzer analyzer, TupleDescriptor tupleDesc, OlapTable table) { - this.analyzer = analyzer; - this.tupleDesc = tupleDesc; - this.table = table; - } - - public long selectBestRollup( - Collection partitionIds, List conjuncts, boolean isPreAggregation) - throws UserException { - Preconditions.checkArgument(partitionIds != null, "Paritition can't be null."); - // Get first partition to select best prefix index rollups, because MaterializedIndex ids in one rollup's partitions are all same. - final List bestPrefixIndexRollups = selectBestPrefixIndexRollup(conjuncts, isPreAggregation); - return selectBestRowCountRollup(bestPrefixIndexRollups, partitionIds); - } - - private long selectBestRowCountRollup(List bestPrefixIndexRollups, Collection partitionIds) { - long minRowCount = Long.MAX_VALUE; - long selectedIndexId = 0; - for (Long indexId : bestPrefixIndexRollups) { - long rowCount = 0; - for (Long partitionId : partitionIds) { - rowCount += table.getPartition(partitionId).getIndex(indexId).getRowCount(); - } - LOG.debug("rowCount={} for table={}", rowCount, indexId); - if (rowCount < minRowCount) { - minRowCount = rowCount; - selectedIndexId = indexId; - } else if (rowCount == minRowCount) { - // check column number, select one minimum column number - int selectedColumnSize = table.getSchemaByIndexId(selectedIndexId).size(); - int currColumnSize = table.getSchemaByIndexId(indexId).size(); - if (currColumnSize < selectedColumnSize) { - selectedIndexId = indexId; - } - } - } - String tableName = table.getName(); - String v2RollupIndexName = MaterializedViewHandler.NEW_STORAGE_FORMAT_INDEX_NAME_PREFIX + tableName; - Long v2RollupIndexId = table.getIndexIdByName(v2RollupIndexName); - long baseIndexId = table.getBaseIndexId(); - if (v2RollupIndexId != null && v2RollupIndexId == selectedIndexId) { - // if the selectedIndexId is v2RollupIndex - // but useV2Rollup is false, use baseIndexId as selectedIndexId - // just make sure to use baseIndex instead of v2RollupIndex if the useV2Rollup is false - selectedIndexId = baseIndexId; - } - return selectedIndexId; - } - - private List selectBestPrefixIndexRollup(List conjuncts, boolean isPreAggregation) - throws UserException { - - final List outputColumns = Lists.newArrayList(); - for (SlotDescriptor slot : tupleDesc.getMaterializedSlots()) { - Column col = slot.getColumn(); - outputColumns.add(col.getName()); - } - - final List rollups = table.getVisibleIndex(); - LOG.debug("num of rollup(base included): {}, pre aggr: {}", rollups.size(), isPreAggregation); - - // 1. find all rollup indexes which contains all tuple columns - final List rollupsContainsOutput = Lists.newArrayList(); - final List baseTableColumns = table.getKeyColumnsByIndexId(table.getBaseIndexId()); - for (MaterializedIndex rollup : rollups) { - final Set rollupColumns = Sets.newHashSet(); - table.getSchemaByIndexId(rollup.getId()) - .stream().forEach(column -> rollupColumns.add(column.getName())); - - if (rollupColumns.containsAll(outputColumns)) { - // If preAggregation is off, so that we only can use base table - // or those rollup tables which key columns is the same with base table - // (often in different order) - if (isPreAggregation) { - LOG.debug("preAggregation is on. add index {} which contains all output columns", - rollup.getId()); - rollupsContainsOutput.add(rollup); - } else if (table.getKeyColumnsByIndexId(rollup.getId()).size() == baseTableColumns.size()) { - LOG.debug("preAggregation is off, but index {} have same key columns with base index.", - rollup.getId()); - rollupsContainsOutput.add(rollup); - } - } else { - LOG.debug("exclude index {} because it does not contain all output columns", rollup.getId()); - } - } - - Preconditions.checkArgument(rollupsContainsOutput.size() > 0, - "Can't find candicate rollup contains all output columns."); - - // 2. find all rollups which match the prefix most based on predicates column from containTupleIndices. - final Set equivalenceColumns = Sets.newHashSet(); - final Set unequivalenceColumns = Sets.newHashSet(); - collectColumns(conjuncts, equivalenceColumns, unequivalenceColumns); - final List rollupsMatchingBestPrefixIndex = Lists.newArrayList(); - matchPrefixIndex(rollupsContainsOutput, rollupsMatchingBestPrefixIndex, - equivalenceColumns, unequivalenceColumns); - - if (rollupsMatchingBestPrefixIndex.isEmpty()) { - rollupsContainsOutput.stream().forEach(n -> rollupsMatchingBestPrefixIndex.add(n.getId())); - } - - // 3. sorted the final candidate indexes by index id - // this is to make sure that candidate indexes find in all partitions will be returned in same order - Collections.sort(rollupsMatchingBestPrefixIndex, new Comparator() { - @Override - public int compare(Long id1, Long id2) { - return (int) (id1 - id2); - } - }); - return rollupsMatchingBestPrefixIndex; - } - - private void matchPrefixIndex(List candidateRollups, - List rollupsMatchingBestPrefixIndex, - Set equivalenceColumns, - Set unequivalenceColumns) { - if (equivalenceColumns.size() == 0 && unequivalenceColumns.size() == 0) { - return; - } - int maxPrefixMatchCount = 0; - int prefixMatchCount; - for (MaterializedIndex index : candidateRollups) { - prefixMatchCount = 0; - for (Column col : table.getSchemaByIndexId(index.getId())) { - if (equivalenceColumns.contains(col.getName())) { - prefixMatchCount++; - } else if (unequivalenceColumns.contains(col.getName())) { - // Unequivalence predicate's columns can match only first column in rollup. - prefixMatchCount++; - break; - } else { - break; - } - } - - if (prefixMatchCount == maxPrefixMatchCount) { - LOG.debug("s3: find a equal prefix match index {}. match count: {}", index.getId(), prefixMatchCount); - rollupsMatchingBestPrefixIndex.add(index.getId()); - } else if (prefixMatchCount > maxPrefixMatchCount) { - LOG.debug("s3: find a better prefix match index {}. match count: {}", index.getId(), prefixMatchCount); - maxPrefixMatchCount = prefixMatchCount; - rollupsMatchingBestPrefixIndex.clear(); - rollupsMatchingBestPrefixIndex.add(index.getId()); - } - } - } - - private void collectColumns( - List conjuncts, Set equivalenceColumns, Set unequivalenceColumns) { - - // 1. Get columns which has predicate on it. - for (Expr expr : conjuncts) { - if (!isPredicateUsedForPrefixIndex(expr, false)) { - continue; - } - for (SlotDescriptor slot : tupleDesc.getMaterializedSlots()) { - if (expr.isBound(slot.getId())) { - if (!isEquivalenceExpr(expr)) { - unequivalenceColumns.add(slot.getColumn().getName()); - } else { - equivalenceColumns.add(slot.getColumn().getName()); - } - break; - } - } - } - - // 2. Equal join predicates when pushing inner child. - List eqJoinPredicate = analyzer.getEqJoinConjuncts(tupleDesc.getId()); - for (Expr expr : eqJoinPredicate) { - if (!isPredicateUsedForPrefixIndex(expr, true)) { - continue; - } - for (SlotDescriptor slot : tupleDesc.getMaterializedSlots()) { - for (int i = 0; i < 2; i++) { - if (expr.getChild(i).isBound(slot.getId())) { - equivalenceColumns.add(slot.getColumn().getName()); - break; - } - } - } - } - } - - private boolean isEquivalenceExpr(Expr expr) { - if (expr instanceof InPredicate) { - return true; - } - if (expr instanceof BinaryPredicate) { - final BinaryPredicate predicate = (BinaryPredicate) expr; - if (predicate.getOp().isEquivalence()) { - return true; - } - } - return false; - } - - private boolean isPredicateUsedForPrefixIndex(Expr expr, boolean isJoinConjunct) { - if (!(expr instanceof InPredicate) - && !(expr instanceof BinaryPredicate)) { - return false; - } - if (expr instanceof InPredicate) { - return isInPredicateUsedForPrefixIndex((InPredicate) expr); - } else if (expr instanceof BinaryPredicate) { - if (isJoinConjunct) { - return isEqualJoinConjunctUsedForPrefixIndex((BinaryPredicate) expr); - } else { - return isBinaryPredicateUsedForPrefixIndex((BinaryPredicate) expr); - } - } - return true; - } - - private boolean isEqualJoinConjunctUsedForPrefixIndex(BinaryPredicate expr) { - Preconditions.checkArgument(expr.getOp().isEquivalence()); - if (expr.isAuxExpr()) { - return false; - } - for (Expr child : expr.getChildren()) { - for (SlotDescriptor slot : tupleDesc.getMaterializedSlots()) { - if (child.isBound(slot.getId()) && isSlotRefNested(child)) { - return true; - } - } - } - return false; - } - - private boolean isBinaryPredicateUsedForPrefixIndex(BinaryPredicate expr) { - if (expr.isAuxExpr() || expr.getOp().isUnequivalence()) { - return false; - } - return (isSlotRefNested(expr.getChild(0)) && expr.getChild(1).isConstant()) - || (isSlotRefNested(expr.getChild(1)) && expr.getChild(0).isConstant()); - } - - private boolean isInPredicateUsedForPrefixIndex(InPredicate expr) { - if (expr.isNotIn()) { - return false; - } - return isSlotRefNested(expr.getChild(0)) && expr.isLiteralChildren(); - } - - private boolean isSlotRefNested(Expr expr) { - while (expr instanceof CastExpr) { - expr = expr.getChild(0); - } - return expr instanceof SlotRef; - } -} diff --git a/fe/fe-core/src/main/java/com/starrocks/planner/StreamLoadPlanner.java b/fe/fe-core/src/main/java/com/starrocks/planner/StreamLoadPlanner.java index c64bd6df16bd8..a72068d531a87 100644 --- a/fe/fe-core/src/main/java/com/starrocks/planner/StreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/com/starrocks/planner/StreamLoadPlanner.java @@ -89,9 +89,6 @@ public StreamLoadPlanner(Database db, OlapTable destTable, StreamLoadTask stream private void resetAnalyzer() { analyzer = new Analyzer(GlobalStateMgr.getCurrentState(), null); - // TODO(cmy): currently we do not support UDF in stream load command. - // Because there is no way to check the privilege of accessing UDF.. - analyzer.setUDFAllowed(false); descTable = analyzer.getDescTbl(); }