Skip to content

[branch-2.1](auto-partition) forbid create table with illegal auto partition expr #31498 #31599

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiguolei merged 2 commits into from
Feb 29, 2024
Merged

[branch-2.1](auto-partition) forbid create table with illegal auto partition expr #31498 #31599

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7a6e502
[enhancement](Nereids) New optimizer support check column privileges …
924060929 Feb 29, 2024
711cdb4
[fix](nereids) forbid create table with illegal auto partition expr (…
zclllyybb Feb 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 47 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import org.apache.doris.analysis.RecoverPartitionStmt;
import org.apache.doris.analysis.RecoverTableStmt;
import org.apache.doris.analysis.SinglePartitionDesc;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.TableName;
import org.apache.doris.analysis.TableRef;
import org.apache.doris.analysis.TruncateTableStmt;
Expand All @@ -68,6 +69,7 @@
import org.apache.doris.catalog.DynamicPartitionProperty;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.EsTable;
import org.apache.doris.catalog.Function;
import org.apache.doris.catalog.HashDistributionInfo;
import org.apache.doris.catalog.HiveTable;
import org.apache.doris.catalog.Index;
Expand Down Expand Up @@ -1981,6 +1983,17 @@ private Partition createPartitionWithIndices(long dbId, long tableId, String tab
return partition;
}

private Type getChildTypeByName(String name, CreateTableStmt stmt)
throws AnalysisException {
List<Column> columns = stmt.getColumns();
for (Column col : columns) {
if (col.nameEquals(name, false)) {
return col.getType();
}
}
throw new AnalysisException("Cannot find column `" + name + "` in table's columns");
}

// Create olap table and related base index synchronously.
private void createOlapTable(Database db, CreateTableStmt stmt) throws UserException {
String tableName = stmt.getTableName();
Expand Down Expand Up @@ -2025,6 +2038,40 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws UserExcep

// create partition info
PartitionDesc partitionDesc = stmt.getPartitionDesc();

// check legality of partiton exprs
ConnectContext ctx = ConnectContext.get();
Env env = Env.getCurrentEnv();
if (ctx != null && env != null && partitionDesc != null && partitionDesc.getPartitionExprs() != null) {
for (Expr expr : partitionDesc.getPartitionExprs()) {
if (expr != null && expr instanceof FunctionCallExpr) { // test them
FunctionCallExpr func = (FunctionCallExpr) expr;
ArrayList<Expr> children = func.getChildren();
Type[] childTypes = new Type[children.size()];
for (int i = 0; i < children.size(); i++) {
if (children.get(i) instanceof LiteralExpr) {
childTypes[i] = children.get(i).getType();
} else if (children.get(i) instanceof SlotRef) {
childTypes[i] = getChildTypeByName(children.get(i).getExprName(), stmt);
} else {
throw new AnalysisException(String.format(
"partition expr %s has unrecognized parameter in slot %d", func.getExprName(), i));
}
}
Function fn = null;
try {
fn = func.getBuiltinFunction(func.getFnName().getFunction(), childTypes,
Function.CompareMode.IS_INDISTINGUISHABLE); // only for test
} catch (Exception e) {
throw new AnalysisException("partition expr " + func.getExprName() + " is illegal!");
}
if (fn == null) {
throw new AnalysisException("partition expr " + func.getExprName() + " is illegal!");
}
}
}
}

PartitionInfo partitionInfo = null;
Map<String, Long> partitionNameToId = Maps.newHashMap();
if (partitionDesc != null) {
Expand Down Expand Up @@ -2699,7 +2746,6 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws UserExcep
}
throw t;
}

}
}

Expand Down
43 changes: 22 additions & 21 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,6 @@ public class CascadesContext implements ScheduleContext {
private boolean isLeadingDisableJoinReorder = false;

private final Map<String, Hint> hintMap = Maps.newLinkedHashMap();
private final boolean shouldCheckRelationAuthentication;
private final ThreadLocal<Boolean> showPlanProcess = new ThreadLocal<>();

// This list is used to listen the change event of the plan which
Expand All @@ -133,12 +132,12 @@ public class CascadesContext implements ScheduleContext {
/**
* Constructor of OptimizerContext.
*
* @param memo {@link Memo} reference
* @param statementContext {@link StatementContext} reference
* @param memo {@link Memo} reference
*/
private CascadesContext(Optional<CascadesContext> parent, Optional<CTEId> currentTree,
StatementContext statementContext, Plan plan, Memo memo,
CTEContext cteContext, PhysicalProperties requireProperties, boolean shouldCheckRelationAuthentication) {
CTEContext cteContext, PhysicalProperties requireProperties) {
this.parent = Objects.requireNonNull(parent, "parent should not null");
this.currentTree = Objects.requireNonNull(currentTree, "currentTree should not null");
this.statementContext = Objects.requireNonNull(statementContext, "statementContext should not null");
Expand All @@ -152,7 +151,6 @@ private CascadesContext(Optional<CascadesContext> parent, Optional<CTEId> curren
this.subqueryExprIsAnalyzed = new HashMap<>();
this.runtimeFilterContext = new RuntimeFilterContext(getConnectContext().getSessionVariable());
this.materializationContexts = new ArrayList<>();
this.shouldCheckRelationAuthentication = shouldCheckRelationAuthentication;
}

/**
Expand All @@ -161,13 +159,7 @@ private CascadesContext(Optional<CascadesContext> parent, Optional<CTEId> curren
public static CascadesContext initContext(StatementContext statementContext,
Plan initPlan, PhysicalProperties requireProperties) {
return newContext(Optional.empty(), Optional.empty(), statementContext,
initPlan, new CTEContext(), requireProperties, true);
}

public static CascadesContext initViewContext(StatementContext statementContext,
Plan initPlan, PhysicalProperties requireProperties) {
return newContext(Optional.empty(), Optional.empty(), statementContext,
initPlan, new CTEContext(), requireProperties, false);
initPlan, new CTEContext(), requireProperties);
}

/**
Expand All @@ -176,14 +168,14 @@ public static CascadesContext initViewContext(StatementContext statementContext,
public static CascadesContext newContextWithCteContext(CascadesContext cascadesContext,
Plan initPlan, CTEContext cteContext) {
return newContext(Optional.of(cascadesContext), Optional.empty(),
cascadesContext.getStatementContext(), initPlan, cteContext, PhysicalProperties.ANY,
cascadesContext.shouldCheckRelationAuthentication);
cascadesContext.getStatementContext(), initPlan, cteContext, PhysicalProperties.ANY
);
}

public static CascadesContext newCurrentTreeContext(CascadesContext context) {
return CascadesContext.newContext(context.getParent(), context.getCurrentTree(), context.getStatementContext(),
context.getRewritePlan(), context.getCteContext(),
context.getCurrentJobContext().getRequiredProperties(), context.shouldCheckRelationAuthentication);
context.getCurrentJobContext().getRequiredProperties());
}

/**
Expand All @@ -192,14 +184,14 @@ public static CascadesContext newCurrentTreeContext(CascadesContext context) {
public static CascadesContext newSubtreeContext(Optional<CTEId> subtree, CascadesContext context,
Plan plan, PhysicalProperties requireProperties) {
return CascadesContext.newContext(Optional.of(context), subtree, context.getStatementContext(),
plan, context.getCteContext(), requireProperties, context.shouldCheckRelationAuthentication);
plan, context.getCteContext(), requireProperties);
}

private static CascadesContext newContext(Optional<CascadesContext> parent, Optional<CTEId> subtree,
StatementContext statementContext, Plan initPlan, CTEContext cteContext,
PhysicalProperties requireProperties, boolean shouldCheckRelationAuthentication) {
PhysicalProperties requireProperties) {
return new CascadesContext(parent, subtree, statementContext, initPlan, null,
cteContext, requireProperties, shouldCheckRelationAuthentication);
cteContext, requireProperties);
}

public CascadesContext getRoot() {
Expand Down Expand Up @@ -655,10 +647,6 @@ public void setLeadingJoin(boolean leadingJoin) {
isLeadingJoin = leadingJoin;
}

public boolean shouldCheckRelationAuthentication() {
return shouldCheckRelationAuthentication;
}

public boolean isLeadingDisableJoinReorder() {
return isLeadingDisableJoinReorder;
}
Expand All @@ -675,6 +663,10 @@ public void addPlanProcess(PlanProcess planProcess) {
planProcesses.add(planProcess);
}

public void addPlanProcesses(List<PlanProcess> planProcesses) {
this.planProcesses.addAll(planProcesses);
}

public List<PlanProcess> getPlanProcesses() {
return planProcesses;
}
Expand Down Expand Up @@ -706,4 +698,13 @@ public void withPlanProcess(boolean showPlanProcess, Runnable task) {
}
}
}

/** keepOrShowPlanProcess */
public void keepOrShowPlanProcess(boolean showPlanProcess, Runnable task) {
if (showPlanProcess) {
withPlanProcess(showPlanProcess, task);
} else {
task.run();
}
}
}
17 changes: 15 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import org.apache.doris.nereids.rules.rewrite.CheckDataTypes;
import org.apache.doris.nereids.rules.rewrite.CheckMatchExpression;
import org.apache.doris.nereids.rules.rewrite.CheckMultiDistinct;
import org.apache.doris.nereids.rules.rewrite.CheckPrivileges;
import org.apache.doris.nereids.rules.rewrite.CollectFilterAboveConsumer;
import org.apache.doris.nereids.rules.rewrite.CollectProjectAboveConsumer;
import org.apache.doris.nereids.rules.rewrite.ColumnPruning;
Expand All @@ -66,7 +67,7 @@
import org.apache.doris.nereids.rules.rewrite.EliminateOrderByConstant;
import org.apache.doris.nereids.rules.rewrite.EliminateSemiJoin;
import org.apache.doris.nereids.rules.rewrite.EliminateSort;
import org.apache.doris.nereids.rules.rewrite.EliminateSortUnderSubquery;
import org.apache.doris.nereids.rules.rewrite.EliminateSortUnderSubqueryOrView;
import org.apache.doris.nereids.rules.rewrite.EliminateUnnecessaryProject;
import org.apache.doris.nereids.rules.rewrite.EnsureProjectOnTopJoin;
import org.apache.doris.nereids.rules.rewrite.ExtractAndNormalizeWindowExpression;
Expand All @@ -78,6 +79,7 @@
import org.apache.doris.nereids.rules.rewrite.InferJoinNotNull;
import org.apache.doris.nereids.rules.rewrite.InferPredicates;
import org.apache.doris.nereids.rules.rewrite.InferSetOperatorDistinct;
import org.apache.doris.nereids.rules.rewrite.InlineLogicalView;
import org.apache.doris.nereids.rules.rewrite.LimitSortToTopN;
import org.apache.doris.nereids.rules.rewrite.MergeFilters;
import org.apache.doris.nereids.rules.rewrite.MergeOneRowRelationIntoUnion;
Expand Down Expand Up @@ -142,7 +144,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
topic("Plan Normalization",
topDown(
new EliminateOrderByConstant(),
new EliminateSortUnderSubquery(),
new EliminateSortUnderSubqueryOrView(),
new EliminateGroupByConstant(),
// MergeProjects depends on this rule
new LogicalSubQueryAliasToLogicalProject(),
Expand Down Expand Up @@ -231,6 +233,17 @@ public class Rewriter extends AbstractBatchJobExecutor {
new ApplyToJoin()
)
),
// before `Subquery unnesting` topic, some correlate slots should have appeared at LogicalApply.left,
// but it appeared at LogicalApply.right. After the `Subquery unnesting` topic, all slots is placed in a
// normal position, then we can check column privileges by these steps
//
// 1. use ColumnPruning rule to derive the used slots in LogicalView
// 2. and then check the column privileges
// 3. finally, we can eliminate the LogicalView
topic("Inline view and check column privileges",
custom(RuleType.CHECK_PRIVILEGES, CheckPrivileges::new),
bottomUp(new InlineLogicalView())
),
topic("Eliminate optimization",
bottomUp(
new EliminateLimit(),
Expand Down
5 changes: 4 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,8 @@ public enum RuleType {

ELIMINATE_LOGICAL_SELECT_HINT(RuleTypeClass.REWRITE),
ELIMINATE_ORDER_BY_CONSTANT(RuleTypeClass.REWRITE),
ELIMINATE_SUBQUERY_ORDER_BY(RuleTypeClass.REWRITE),
ELIMINATE_ORDER_BY_UNDER_SUBQUERY(RuleTypeClass.REWRITE),
ELIMINATE_ORDER_BY_UNDER_VIEW(RuleTypeClass.REWRITE),
ELIMINATE_HINT(RuleTypeClass.REWRITE),
ELIMINATE_JOIN_ON_EMPTYRELATION(RuleTypeClass.REWRITE),
ELIMINATE_FILTER_ON_EMPTYRELATION(RuleTypeClass.REWRITE),
Expand Down Expand Up @@ -288,6 +289,8 @@ public enum RuleType {
CTE_INLINE(RuleTypeClass.REWRITE),
REWRITE_CTE_CHILDREN(RuleTypeClass.REWRITE),
COLLECT_FILTER_ON_CONSUMER(RuleTypeClass.REWRITE),
INLINE_VIEW(RuleTypeClass.REWRITE),
CHECK_PRIVILEGES(RuleTypeClass.REWRITE),

COLLECT_FILTER(RuleTypeClass.REWRITE),
COLLECT_JOIN_CONSTRAINT(RuleTypeClass.REWRITE),
Expand Down
34 changes: 17 additions & 17 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
import org.apache.doris.nereids.trees.plans.logical.LogicalTestScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalView;
import org.apache.doris.nereids.util.RelationUtil;
import org.apache.doris.qe.ConnectContext;

Expand Down Expand Up @@ -157,7 +159,7 @@ private LogicalPlan bindWithCurrentDb(CascadesContext cascadesContext, UnboundRe
}

// TODO: should generate different Scan sub class according to table's type
LogicalPlan scan = getAndCheckLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
LogicalPlan scan = getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
if (cascadesContext.isLeadingJoin()) {
LeadingHint leading = (LeadingHint) cascadesContext.getHintMap().get("Leading");
leading.putRelationIdAndTableName(Pair.of(unboundRelation.getRelationId(), tableName));
Expand All @@ -178,7 +180,7 @@ private LogicalPlan bind(CascadesContext cascadesContext, UnboundRelation unboun
if (table == null) {
table = RelationUtil.getTable(tableQualifier, cascadesContext.getConnectContext().getEnv());
}
return getAndCheckLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
return getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
}

private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, List<String> tableQualifier) {
Expand Down Expand Up @@ -234,25 +236,18 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation,
return scan;
}

private LogicalPlan getAndCheckLogicalPlan(TableIf table, UnboundRelation unboundRelation,
private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelation,
List<String> tableQualifier, CascadesContext cascadesContext) {
// if current context is in the view, we can skip check authentication because
// the view already checked authentication
if (cascadesContext.shouldCheckRelationAuthentication()) {
UserAuthentication.checkPermission(table, cascadesContext.getConnectContext());
}
return doGetLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
}

private LogicalPlan doGetLogicalPlan(TableIf table, UnboundRelation unboundRelation, List<String> tableQualifier,
CascadesContext cascadesContext) {
switch (table.getType()) {
case OLAP:
case MATERIALIZED_VIEW:
return makeOlapScan(table, unboundRelation, tableQualifier);
case VIEW:
Plan viewPlan = parseAndAnalyzeView(((View) table).getInlineViewDef(), cascadesContext);
return new LogicalSubQueryAlias<>(tableQualifier, viewPlan);
View view = (View) table;
String inlineViewDef = view.getInlineViewDef();
Plan viewBody = parseAndAnalyzeView(inlineViewDef, cascadesContext);
LogicalView<Plan> logicalView = new LogicalView<>(view, viewBody);
return new LogicalSubQueryAlias<>(tableQualifier, logicalView);
case HMS_EXTERNAL_TABLE:
if (Config.enable_query_hive_views && ((HMSExternalTable) table).isView()) {
String hiveCatalog = ((HMSExternalTable) table).getCatalog().getName();
Expand All @@ -276,6 +271,8 @@ private LogicalPlan doGetLogicalPlan(TableIf table, UnboundRelation unboundRelat
return new LogicalOdbcScan(unboundRelation.getRelationId(), table, tableQualifier);
case ES_EXTERNAL_TABLE:
return new LogicalEsScan(unboundRelation.getRelationId(), (EsExternalTable) table, tableQualifier);
case TEST_EXTERNAL_TABLE:
return new LogicalTestScan(unboundRelation.getRelationId(), table, tableQualifier);
default:
throw new AnalysisException("Unsupported tableType " + table.getType());
}
Expand All @@ -299,9 +296,12 @@ private Plan parseAndAnalyzeView(String ddlSql, CascadesContext parentContext) {
if (parsedViewPlan instanceof UnboundResultSink) {
parsedViewPlan = (LogicalPlan) ((UnboundResultSink<?>) parsedViewPlan).child();
}
CascadesContext viewContext = CascadesContext.initViewContext(
CascadesContext viewContext = CascadesContext.initContext(
parentContext.getStatementContext(), parsedViewPlan, PhysicalProperties.ANY);
viewContext.newAnalyzer(true, customTableResolver).analyze();
viewContext.keepOrShowPlanProcess(parentContext.showPlanProcess(), () -> {
viewContext.newAnalyzer(true, customTableResolver).analyze();
});
parentContext.addPlanProcesses(viewContext.getPlanProcesses());
// we should remove all group expression of the plan which in other memo, so the groupId would not conflict
return viewContext.getRewritePlan();
}
Expand Down
1 change: 1 addition & 0 deletions ...ore/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ private void setVar(SelectHintSetVar selectHint, StatementContext context) {
}
throw new AnalysisException("The nereids is disabled in this sql, fallback to original planner");
}
context.invalidCache(selectHint.getHintName());
}

private void extractLeading(SelectHintLeading selectHint, CascadesContext context,
Expand Down
Loading