Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Enhancment] Assign tablet ranges for one-phase aggregate without partition and colocate #8771

Merged
merged 2 commits into from Jul 18, 2022
Merged

[Enhancment] Assign tablet ranges for one-phase aggregate without partition and colocate #8771

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2962281
[Enhancment] Assign tablet ranges for one-phase aggregate without par…
ZiheLiu Jul 15, 2022
6802401
add comment
ZiheLiu Jul 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
12 changes: 12 additions & 0 deletions fe/fe-core/src/main/java/com/starrocks/planner/PlanFragment.java
View file
Open in desktop
Expand Up @@ -126,6 +126,10 @@ public class PlanFragment extends TreeNode<PlanFragment> {
// Enable shared_scan for this fragment: OlapScanOperator could share the output data to avoid data skew
protected boolean enableSharedScan = true;

// Whether to assign scan ranges to each driver sequence of pipeline,
// for the normal backend assignment (not colocate, bucket, and replicated join).
protected boolean assignScanRangesPerDriverSeq = false;

protected final Map<Integer, RuntimeFilterDescription> buildRuntimeFilters = Maps.newTreeMap();
protected final Map<Integer, RuntimeFilterDescription> probeRuntimeFilters = Maps.newTreeMap();

Expand Down Expand Up @@ -249,6 +253,14 @@ public boolean isEnableSharedScan() {
return enableSharedScan;
}

public boolean isAssignScanRangesPerDriverSeq() {
return assignScanRangesPerDriverSeq;
}

public void setAssignScanRangesPerDriverSeq(boolean assignScanRangesPerDriverSeq) {
this.assignScanRangesPerDriverSeq = assignScanRangesPerDriverSeq;
}

public void computeLocalRfWaitingSet(PlanNode root, boolean clearGlobalRuntimeFilter) {
root.fillLocalRfWaitingSet(runtimeFilterBuildNodeIds);
if (root instanceof RuntimeFilterBuildNode) {
Expand Down
17 changes: 15 additions & 2 deletions fe/fe-core/src/main/java/com/starrocks/qe/Coordinator.java
View file
Open in desktop
Expand Up @@ -465,7 +465,6 @@ public void exec() throws Exception {
DebugUtil.printId(queryId), descTable);
}


// prepare information
prepare();

Expand Down Expand Up @@ -1708,6 +1707,7 @@ private void computeFragmentHosts() throws Exception {
parallelExecInstanceNum, pipelineDop, enablePipeline, params);
computeBucketSeq2InstanceOrdinal(params, fragmentIdToBucketNumMap.get(fragment.getFragmentId()));
} else {
boolean assignScanRangesPerDriverSeq = enablePipeline && fragment.isAssignScanRangesPerDriverSeq();
for (Map.Entry<TNetworkAddress, Map<Integer, List<TScanRangeParams>>> tNetworkAddressMapEntry :
fragmentExecParamsMap.get(fragment.getFragmentId()).scanRangeAssignment.entrySet()) {
TNetworkAddress key = tNetworkAddressMapEntry.getKey();
Expand All @@ -1729,8 +1729,21 @@ private void computeFragmentHosts() throws Exception {

for (List<TScanRangeParams> scanRangeParams : perInstanceScanRanges) {
FInstanceExecParam instanceParam = new FInstanceExecParam(null, key, 0, params);
instanceParam.perNodeScanRanges.put(planNodeId, scanRangeParams);
params.instanceExecParams.add(instanceParam);

if (!assignScanRangesPerDriverSeq) {
instanceParam.perNodeScanRanges.put(planNodeId, scanRangeParams);
} else {
int expectedDop = Math.max(1, Math.min(pipelineDop, scanRangeParams.size()));
List<List<TScanRangeParams>> scanRangeParamsPerDriverSeq =
ListUtil.splitBySize(scanRangeParams, expectedDop);
instanceParam.pipelineDop = scanRangeParamsPerDriverSeq.size();
Map<Integer, List<TScanRangeParams>> scanRangesPerDriverSeq = new HashMap<>();
instanceParam.nodeToPerDriverSeqScanRanges.put(planNodeId, scanRangesPerDriverSeq);
for (int driverSeq = 0; driverSeq < scanRangeParamsPerDriverSeq.size(); ++driverSeq) {
scanRangesPerDriverSeq.put(driverSeq, scanRangeParamsPerDriverSeq.get(driverSeq));
}
}
}
}

Expand Down
3 changes: 2 additions & 1 deletion fe/fe-core/src/main/java/com/starrocks/sql/plan/PlanFragmentBuilder.java
View file
Open in desktop
Expand Up @@ -1295,8 +1295,9 @@ public PlanFragment visitPhysicalHashAggregate(OptExpression optExpr, ExecPlan c
aggregationNode.setHasNullableGenerateChild();
aggregationNode.computeStatistics(optExpr.getStatistics());

if (node.isOnePhaseAgg() && aggregationNode.isColocate()) {
if (node.isOnePhaseAgg() && hasNoExchangeNodes(inputFragment.getPlanRoot())) {
inputFragment.setEnableSharedScan(false);
inputFragment.setAssignScanRangesPerDriverSeq(true);
}

inputFragment.setPlanRoot(aggregationNode);
Expand Down