-
Notifications
You must be signed in to change notification settings - Fork 24.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Transform] Optimize composite agg execution using ordered groupings (#…
…75518) Automatically reorder group_by for composite aggs, ensuring date histogram group by comes first. The order is only changed for execution, the provided config remains unchanged. In case of 2 group_by's of the same order type, the configuration order is respected. Script and runtime field based group_by's are penalized.
- Loading branch information
Hendrik Muhs
committed
Jul 20, 2021
1 parent
957e093
commit 45d1e24
Showing
12 changed files
with
329 additions
and
124 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
77 changes: 77 additions & 0 deletions
77
...rm/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/GroupByOptimizer.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License | ||
| * 2.0; you may not use this file except in compliance with the Elastic License | ||
| * 2.0. | ||
| */ | ||
|
|
||
| package org.elasticsearch.xpack.transform.transforms.pivot; | ||
|
|
||
| import org.elasticsearch.core.Tuple; | ||
| import org.elasticsearch.xpack.core.transform.transforms.pivot.SingleGroupSource; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.Collection; | ||
| import java.util.Comparator; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Map.Entry; | ||
| import java.util.Set; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| public final class GroupByOptimizer { | ||
|
|
||
| private GroupByOptimizer() {} | ||
|
|
||
| /** | ||
| * Returns an ordered collection of group by fields in order to get better performance. | ||
| * | ||
| * The decision is based on the type and whether the input field is a indexed/runtime/script field | ||
| * | ||
| * TODO: take index sorting into account | ||
| * | ||
| * @param groups group by as defined by the user | ||
| * @param runtimeFields set of runtime fields | ||
| * @return collection in order of priority | ||
| */ | ||
| static Collection<Entry<String, SingleGroupSource>> reorderGroups(Map<String, SingleGroupSource> groups, Set<String> runtimeFields) { | ||
| if (groups.size() == 1) { | ||
| return groups.entrySet(); | ||
| } | ||
|
|
||
| List<Tuple<Entry<String, SingleGroupSource>, Integer>> prioritizedGroups = new ArrayList<>(groups.size()); | ||
|
|
||
| // respect the order in the configuration by giving every entry a base priority | ||
| int basePriority = groups.size(); | ||
|
|
||
| for (Entry<String, SingleGroupSource> groupBy : groups.entrySet()) { | ||
| // prefer indexed fields over runtime fields over scripts | ||
| int priority = basePriority-- + (groupBy.getValue().getScriptConfig() == null | ||
| ? runtimeFields.contains(groupBy.getValue().getField()) ? 250 : 500 | ||
| : 0); | ||
|
|
||
| switch (groupBy.getValue().getType()) { | ||
| case DATE_HISTOGRAM: | ||
| priority += 4000; | ||
| break; | ||
| case HISTOGRAM: | ||
| priority += 3000; | ||
| break; | ||
| case TERMS: | ||
| priority += 2000; | ||
| break; | ||
| case GEOTILE_GRID: | ||
| priority += 1000; | ||
| break; | ||
| default: | ||
| assert false : "new group source type misses priority definition"; | ||
| } | ||
|
|
||
| prioritizedGroups.add(new Tuple<>(groupBy, priority)); | ||
| } | ||
|
|
||
| prioritizedGroups.sort(Comparator.comparing(Tuple<Entry<String, SingleGroupSource>, Integer>::v2).reversed()); | ||
|
|
||
| return prioritizedGroups.stream().map(x -> x.v1()).collect(Collectors.toList()); | ||
| } | ||
| } |
Oops, something went wrong.