Skip to content

Optimize sort settings default providers #138711

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

Optimize sort settings default providers #138711

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5c5f1d1
Add MappingParsingBenchmark
jordan-powers Nov 20, 2025
bcfa5ec
Use constant lists for default sort settings
jordan-powers Nov 20, 2025
397cd92
Further optimize
jordan-powers Nov 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions benchmarks/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ dependencies {
api(project(':x-pack:plugin:esql'))
api(project(':x-pack:plugin:esql:compute'))
api(project(':x-pack:plugin:mapper-exponential-histogram'))
api(project(':x-pack:plugin:logsdb'))
implementation project(path: ':libs:native')
implementation project(path: ':libs:simdvec')
implementation project(path: ':libs:exponential-histogram')
Expand Down
171 changes: 171 additions & 0 deletions ...rks/src/main/java/org/elasticsearch/benchmark/indices/common/MappingParsingBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

package org.elasticsearch.benchmark.indices.common;

import org.elasticsearch.TransportVersion;
import org.elasticsearch.cluster.ClusterModule;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.logging.LogConfigurator;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.IndexVersion;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
import org.elasticsearch.index.mapper.MapperMetrics;
import org.elasticsearch.index.mapper.MapperRegistry;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptCompiler;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.xcontent.NamedXContentRegistry;
import org.elasticsearch.xcontent.XContentParserConfiguration;
import org.elasticsearch.xpack.logsdb.LogsDBPlugin;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;

@Fork(value = 1)
@Warmup(iterations = 2)
@Measurement(iterations = 5)
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
@State(Scope.Benchmark)
public class MappingParsingBenchmark {
static {
// For Elasticsearch900Lucene101Codec:
LogConfigurator.loadLog4jPlugins();
LogConfigurator.configureESLogging();
LogConfigurator.setNodeName("test");
}

private static final String MAPPING = """
{
"_doc": {
"dynamic": false,
"properties": {
"@timestamp": {
"type": "date"
},
"host": {
"type": "object",
"properties": {
"name": {
"type": "keyword"
}
}
},
"message": {
"type": "pattern_text"
}
}
}
}
\s""";

@Param("1024")
private int numIndices;

private List<MapperService> mapperServices;
private CompressedXContent compressedMapping;

private Random random = new Random();
private static final String CHARS = "abcdefghijklmnopqrstuvwxyz1234567890";

private String randomIndexName() {
StringBuilder b = new StringBuilder();
for (int i = 0; i < 10; i++) {
b.append(CHARS.charAt(random.nextInt(CHARS.length())));
}
return b.toString();
}

@Setup
public void setUp() throws IOException {
Settings settings = Settings.builder()
.put("index.number_of_replicas", 0)
.put("index.number_of_shards", 1)
.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
.put("index.mode", "logsdb")
.put("index.logsdb.sort_on_host_name", true)
.put("index.logsdb.sort_on_message_template", true)
.build();

LogsDBPlugin logsDBPlugin = new LogsDBPlugin(settings);

Set<Setting<?>> definedSettings = new HashSet<>(IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
definedSettings.addAll(logsDBPlugin.getSettings().stream().filter(Setting::hasIndexScope).toList());
IndexScopedSettings indexScopedSettings = new IndexScopedSettings(Settings.EMPTY, definedSettings);

mapperServices = new ArrayList<>(numIndices);
for (int i = 0; i < numIndices; i++) {
IndexMetadata meta = IndexMetadata.builder(randomIndexName()).settings(settings).build();
IndexSettings indexSettings = new IndexSettings(meta, settings, indexScopedSettings);
MapperRegistry mapperRegistry = new IndicesModule(List.of(logsDBPlugin)).getMapperRegistry();
SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP);
MapperService mapperService = new MapperService(
() -> TransportVersion.current(),
indexSettings,
IndexAnalyzers.of(Map.of()),
XContentParserConfiguration.EMPTY.withRegistry(new NamedXContentRegistry(ClusterModule.getNamedXWriteables()))
.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE),
similarityService,
mapperRegistry,
() -> {
throw new UnsupportedOperationException();
},
new ProvidedIdFieldMapper(() -> true),
new ScriptCompiler() {
@Override
public <T> T compile(Script script, ScriptContext<T> scriptContext) {
throw new UnsupportedOperationException();
}
},
bitsetFilterCache::getBitSetProducer,
MapperMetrics.NOOP
);

mapperServices.add(mapperService);
}

compressedMapping = new CompressedXContent(MAPPING);
}

@Benchmark
public void mappingParsingBenchmark() {
for (MapperService service : mapperServices) {
service.merge("_doc", compressedMapping, MapperService.MergeReason.MAPPING_UPDATE);
}
}
}
161 changes: 100 additions & 61 deletions server/src/main/java/org/elasticsearch/index/IndexSortConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
import org.elasticsearch.search.sort.SortOrder;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Locale;
Expand Down Expand Up @@ -107,31 +107,68 @@ public final class IndexSortConfig {
);

public static class IndexSortConfigDefaults {
public static final FieldSortSpec[] TIME_SERIES_SORT, HOSTNAME_TIMESTAMP_BWC_SORT;
public record SortDefault(List<String> fields, List<String> order, List<String> mode, List<String> missing) {
public SortDefault {
assert fields.size() == order.size();
assert fields.size() == mode.size();
assert fields.size() == missing.size();
}
}

private static final FieldSortSpec HOSTNAME_SPEC, MESSAGE_PATTERN_SPEC, TIMESTAMP_SPEC;
public static final SortDefault NO_SORT, TIME_SERIES_SORT, TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_BWC_SORT,
MESSAGE_PATTERN_TIMESTAMP_SORT, HOSTNAME_MESSAGE_PATTERN_TIMESTAMP_SORT;

static {
TIMESTAMP_SPEC = new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH);
TIMESTAMP_SPEC.order = SortOrder.DESC;
TIME_SERIES_SORT = new FieldSortSpec[] { new FieldSortSpec(TimeSeriesIdFieldMapper.NAME), TIMESTAMP_SPEC };
NO_SORT = new SortDefault(Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList());

HOSTNAME_SPEC = new FieldSortSpec(IndexMode.HOST_NAME);
HOSTNAME_SPEC.order = SortOrder.ASC;
HOSTNAME_SPEC.missingValue = "_last";
HOSTNAME_SPEC.mode = MultiValueMode.MIN;
TIME_SERIES_SORT = new SortDefault(
List.of(TimeSeriesIdFieldMapper.NAME, DataStreamTimestampFieldMapper.DEFAULT_PATH),
List.of("asc", "desc"),
List.of("min", "max"),
List.of("_last", "_last")
);

MESSAGE_PATTERN_SPEC = new FieldSortSpec("message.template_id");
TIMESTAMP_SORT = new SortDefault(
List.of(DataStreamTimestampFieldMapper.DEFAULT_PATH),
List.of("desc"),
List.of("max"),
List.of("_last")
);

HOSTNAME_TIMESTAMP_SORT = new SortDefault(
List.of(IndexMode.HOST_NAME, DataStreamTimestampFieldMapper.DEFAULT_PATH),
List.of("asc", "desc"),
List.of("min", "max"),
List.of("_last", "_last")
);

MESSAGE_PATTERN_TIMESTAMP_SORT = new SortDefault(
List.of("message.template_id", DataStreamTimestampFieldMapper.DEFAULT_PATH),
List.of("asc", "desc"),
List.of("min", "max"),
List.of("_last", "_last")
);

HOSTNAME_MESSAGE_PATTERN_TIMESTAMP_SORT = new SortDefault(
List.of(IndexMode.HOST_NAME, "message.template_id", DataStreamTimestampFieldMapper.DEFAULT_PATH),
List.of("asc", "asc", "desc"),
List.of("min", "min", "max"),
List.of("_last", "_last", "_last")
);

// Older indexes use ascending ordering for host name and timestamp.
HOSTNAME_TIMESTAMP_BWC_SORT = new FieldSortSpec[] {
new FieldSortSpec(IndexMode.HOST_NAME),
new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH) };
HOSTNAME_TIMESTAMP_BWC_SORT = new SortDefault(
List.of(IndexMode.HOST_NAME, DataStreamTimestampFieldMapper.DEFAULT_PATH),
List.of("asc", "asc"),
List.of("min", "min"),
List.of("_last", "_last")
);

}

public static FieldSortSpec[] getDefaultSortSpecs(Settings settings) {
static SortDefault getSortDefault(Settings settings) {
if (settings.isEmpty()) {
return new FieldSortSpec[0];
return NO_SORT;
}

// Can't use IndexSettings.MODE.get(settings) here because the validation logic for IndexSettings.MODE uses the default value
Expand All @@ -152,71 +189,73 @@ public static FieldSortSpec[] getDefaultSortSpecs(Settings settings) {
IndexVersions.UPGRADE_TO_LUCENE_10_0_0
)) {

List<FieldSortSpec> sortSpecs = new ArrayList<>(3);
if (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(settings)) {
sortSpecs.add(HOSTNAME_SPEC);
}
if (IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.get(settings)) {
sortSpecs.add(MESSAGE_PATTERN_SPEC);
boolean sortOnHostName = settings.getAsBoolean(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), false);
boolean sortOnMessageTemplate = settings.getAsBoolean(IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.getKey(), false);
if (sortOnHostName && sortOnMessageTemplate) {
return HOSTNAME_MESSAGE_PATTERN_TIMESTAMP_SORT;
} else if (sortOnHostName) {
return HOSTNAME_TIMESTAMP_SORT;
} else if (sortOnMessageTemplate) {
return MESSAGE_PATTERN_TIMESTAMP_SORT;
} else {
return TIMESTAMP_SORT;
}
sortSpecs.add(TIMESTAMP_SPEC);

return sortSpecs.toArray(FieldSortSpec[]::new);
} else {
return HOSTNAME_TIMESTAMP_BWC_SORT;
}
}

return new FieldSortSpec[0];
}

public static FieldSortSpec[] getSortSpecs(Settings settings) {
if (INDEX_SORT_FIELD_SETTING.exists(settings) == false) {
return IndexSortConfigDefaults.getDefaultSortSpecs(settings);
}

List<String> fields = INDEX_SORT_FIELD_SETTING.get(settings);
FieldSortSpec[] sortSpecs = fields.stream().map(FieldSortSpec::new).toArray(FieldSortSpec[]::new);

// Need to populate `order` because the default value of `mode` depends on it
if (INDEX_SORT_ORDER_SETTING.exists(settings)) {
List<SortOrder> orders = INDEX_SORT_ORDER_SETTING.get(settings);
for (int i = 0; i < sortSpecs.length; i++) {
sortSpecs[i].order = orders.get(i);
}
}

return sortSpecs;
return NO_SORT;
}

public static List<String> getDefaultSortFields(Settings settings) {
return Arrays.stream(getDefaultSortSpecs(settings)).map(sortSpec -> sortSpec.field).toList();
return getSortDefault(settings).fields();
}

public static List<String> getDefaultSortOrder(Settings settings) {
return Arrays.stream(getSortSpecs(settings))
.map(sortSpec -> sortSpec.order != null ? sortSpec.order : SortOrder.ASC)
.map(Enum::toString)
.toList();
if (settings.hasValue(INDEX_SORT_FIELD_SETTING.getKey()) == false) {
return getSortDefault(settings).order();
}

List<String> sortFields = settings.getAsList(INDEX_SORT_FIELD_SETTING.getKey());
List<String> order = new ArrayList<>(sortFields.size());
for (int i = 0; i < sortFields.size(); ++i) {
order.add("asc");
}
return order;
}

public static List<String> getDefaultSortMode(Settings settings) {
return Arrays.stream(getSortSpecs(settings)).map(sortSpec -> {
if (sortSpec.mode != null) {
return sortSpec.mode;
} else if (sortSpec.order == SortOrder.DESC) {
return MultiValueMode.MAX;
if (settings.hasValue(INDEX_SORT_FIELD_SETTING.getKey()) == false) {
return getSortDefault(settings).mode();
}

List<String> sortFields = settings.getAsList(INDEX_SORT_FIELD_SETTING.getKey());
List<String> sortOrder = settings.getAsList(INDEX_SORT_ORDER_SETTING.getKey(), null);

List<String> mode = new ArrayList<>(sortFields.size());
for (int i = 0; i < sortFields.size(); ++i) {
if (sortOrder != null && sortOrder.get(i).equals(SortOrder.DESC.toString())) {
mode.add("max");
} else {
return MultiValueMode.MIN;
mode.add("min");
}
}).map(order -> order.toString().toLowerCase(Locale.ROOT)).toList();
}
return mode;
}

public static List<String> getDefaultSortMissing(Settings settings) {
// _last is the default per IndexFieldData.XFieldComparatorSource.Nested#sortMissingLast
return Arrays.stream(getSortSpecs(settings))
.map(sortSpec -> sortSpec.missingValue != null ? sortSpec.missingValue : "_last")
.toList();
if (settings.hasValue(INDEX_SORT_FIELD_SETTING.getKey()) == false) {
return getSortDefault(settings).missing();
}

List<String> sortFields = settings.getAsList(INDEX_SORT_FIELD_SETTING.getKey());
List<String> missing = new ArrayList<>(sortFields.size());
for (int i = 0; i < sortFields.size(); ++i) {
// _last is the default per IndexFieldData.XFieldComparatorSource.Nested#sortMissingLast
missing.add("_last");
}
return missing;
}
}

Expand Down
Loading