From f3df797f0379925782ac1b9ee7ad09e91f177e57 Mon Sep 17 00:00:00 2001 From: jackwener Date: Tue, 13 Sep 2022 13:14:40 +0800 Subject: [PATCH 1/4] [fix](Nereids): fix StatsCalculator compute project. --- .../doris/nereids/stats/StatsCalculator.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index ba25d62ce9408f..c75c939ede7aad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -70,7 +70,7 @@ import com.google.common.collect.Maps; -import java.util.AbstractMap; +import java.util.AbstractMap.SimpleEntry; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -341,14 +341,23 @@ private StatsDeriveResult computeProject(Project project) { StatsDeriveResult statsDeriveResult = groupExpression.getCopyOfChildStats(0); Map childColumnStats = statsDeriveResult.getSlotToColumnStats(); Map columnsStats = projections.stream().map(projection -> { + ColumnStats value = null; Set slots = projection.getInputSlots(); if (slots.isEmpty()) { - return new AbstractMap.SimpleEntry<>(projection.toSlot(), ColumnStats.createDefaultColumnStats()); + value = ColumnStats.createDefaultColumnStats(); } else { // TODO: just a trick here, need to do real project on column stats - return new AbstractMap.SimpleEntry<>(projection.toSlot(), - childColumnStats.get(slots.iterator().next())); + for (Slot slot : slots) { + if (childColumnStats.containsKey(slot)) { + value = childColumnStats.get(slot); + break; + } + } + if (value == null) { + value = ColumnStats.createDefaultColumnStats(); + } } + return new SimpleEntry<>(projection.toSlot(), value); }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (item1, item2) -> item1)); statsDeriveResult.setSlotToColumnStats(columnsStats); return statsDeriveResult; From 9d9fac89b4509c162784a0e4e0c35210837c788f Mon Sep 17 00:00:00 2001 From: jackwener Date: Tue, 13 Sep 2022 13:35:51 +0800 Subject: [PATCH 2/4] fix commute join type --- .../doris/nereids/rules/exploration/join/JoinCommute.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java index 1fd5bbfaaa3229..05942b3c8759b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java @@ -50,7 +50,7 @@ enum SwapType { @Override public Rule build() { - return innerLogicalJoin() + return logicalJoin() .when(this::check) .then(join -> { LogicalJoin newJoin = new LogicalJoin<>( From 5754ba34eb9ba2b0720ca5ad55673cd832a14596 Mon Sep 17 00:00:00 2001 From: jackwener Date: Tue, 13 Sep 2022 15:00:10 +0800 Subject: [PATCH 3/4] fix type --- .../doris/nereids/rules/exploration/join/JoinCommute.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java index 05942b3c8759b2..1b40df3bc968b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java @@ -54,7 +54,7 @@ public Rule build() { .when(this::check) .then(join -> { LogicalJoin newJoin = new LogicalJoin<>( - join.getJoinType(), + join.getJoinType().swap(), join.getHashJoinConjuncts(), join.getOtherJoinCondition(), join.right(), join.left(), From b359a898e05d3dd3081bb456bf8c571ec41c0b42 Mon Sep 17 00:00:00 2001 From: jackwener Date: Tue, 13 Sep 2022 19:52:58 +0800 Subject: [PATCH 4/4] add PushdownFilterThroughProject --- .../java/org/apache/doris/nereids/rules/RuleSet.java | 2 ++ .../java/org/apache/doris/nereids/rules/RuleType.java | 10 +++++----- .../rewrite/logical/PushdownFilterThroughProject.java | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index 936834605b2018..70251110ec86a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -34,6 +34,7 @@ import org.apache.doris.nereids.rules.implementation.LogicalTopNToPhysicalTopN; import org.apache.doris.nereids.rules.rewrite.AggregateDisassemble; import org.apache.doris.nereids.rules.rewrite.logical.MergeConsecutiveProjects; +import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughProject; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; @@ -48,6 +49,7 @@ public class RuleSet { .add(JoinCommute.OUTER_LEFT_DEEP) .add(JoinLAsscom.INNER) .add(JoinLAsscomProject.INNER) + .add(new PushdownFilterThroughProject()) .add(new MergeConsecutiveProjects()) .build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 278bf7c9298baf..98aac8e9235ed5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -99,7 +99,6 @@ public enum RuleType { // Eliminate plan ELIMINATE_LIMIT(RuleTypeClass.REWRITE), ELIMINATE_FILTER(RuleTypeClass.REWRITE), - FIND_HASH_CONDITION_FOR_JOIN(RuleTypeClass.REWRITE), ROLLUP_AGG_SCAN(RuleTypeClass.REWRITE), ROLLUP_AGG_FILTER_SCAN(RuleTypeClass.REWRITE), @@ -107,8 +106,9 @@ public enum RuleType { ROLLUP_AGG_PROJECT_FILTER_SCAN(RuleTypeClass.REWRITE), ROLLUP_AGG_FILTER_PROJECT_SCAN(RuleTypeClass.REWRITE), OLAP_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE), - // Swap plan - SWAP_FILTER_AND_PROJECT(RuleTypeClass.REWRITE), + // Pushdown filter + PUSHDOWN_FILTER_THROUGH_PROJET(RuleTypeClass.REWRITE), + LOGICAL_LIMIT_TO_LOGICAL_EMPTY_RELATION_RULE(RuleTypeClass.REWRITE), SWAP_LIMIT_PROJECT(RuleTypeClass.REWRITE), REWRITE_SENTINEL(RuleTypeClass.REWRITE), @@ -152,8 +152,8 @@ public RuleTypeClass getRuleTypeClass() { return ruleTypeClass; } - public Rule build( - PatternMatcher patternMatcher) { + public + Rule build(PatternMatcher patternMatcher) { return patternMatcher.toRule(this); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java index aab04a0ac66b9f..c1e064a5c87927 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java @@ -44,6 +44,6 @@ public Rule build() { project.child() ) ); - }).toRule(RuleType.SWAP_FILTER_AND_PROJECT); + }).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_PROJET); } }