Skip to content

Commit

Permalink
Use faster and smaller maps in MappingLookup (#86380)
Browse files Browse the repository at this point in the history
It's in the title. These data structures are a huge (if not the largest) contributor to the idle-state
heap consumption of large data nodes at the moment, lets make them a little more compact and as
a side effect also faster.
  • Loading branch information
original-brownbear committed May 3, 2022
1 parent 3e8f729 commit 0eda627
Show file tree
Hide file tree
Showing 3 changed files with 35 additions and 12 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.xcontent.XContentFactory;
import org.elasticsearch.xcontent.XContentType;
import org.hamcrest.Matchers;

import java.io.IOException;
import java.util.Collection;
Expand Down Expand Up @@ -348,7 +349,8 @@ public void testMultipleJoinFields() throws Exception {
})));
assertThat(
exc.getMessage(),
equalTo("Only one [parent-join] field can be defined per index, got [join_field, another_join_field]")
Matchers.either(equalTo("Only one [parent-join] field can be defined per index, got [join_field, another_join_field]"))
.or(equalTo("Only one [parent-join] field can be defined per index, got [another_join_field, join_field]"))
);
}

Expand All @@ -366,7 +368,8 @@ public void testMultipleJoinFields() throws Exception {
);
assertThat(
exc.getMessage(),
equalTo("Only one [parent-join] field can be defined per index, got [join_field, another_join_field]")
Matchers.either(equalTo("Only one [parent-join] field can be defined per index, got [join_field, another_join_field]"))
.or(equalTo("Only one [parent-join] field can be defined per index, got [another_join_field, join_field]"))
);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@
* An immutable container for looking up {@link MappedFieldType}s by their name.
*/
final class FieldTypeLookup {
private final Map<String, MappedFieldType> fullNameToFieldType = new HashMap<>();
private final Map<String, DynamicFieldType> dynamicFieldTypes = new HashMap<>();
private final Map<String, MappedFieldType> fullNameToFieldType;
private final Map<String, DynamicFieldType> dynamicFieldTypes;

/**
* A map from field name to all fields whose content has been copied into it
Expand All @@ -33,7 +33,7 @@ final class FieldTypeLookup {
*
* For convenience, the set of copied fields includes the field itself.
*/
private final Map<String, Set<String>> fieldToCopiedFields = new HashMap<>();
private final Map<String, Set<String>> fieldToCopiedFields;

private final int maxParentPathDots;

Expand All @@ -43,6 +43,9 @@ final class FieldTypeLookup {
Collection<RuntimeField> runtimeFields
) {

final Map<String, MappedFieldType> fullNameToFieldType = new HashMap<>();
final Map<String, DynamicFieldType> dynamicFieldTypes = new HashMap<>();
final Map<String, Set<String>> fieldToCopiedFields = new HashMap<>();
for (FieldMapper fieldMapper : fieldMappers) {
String fieldName = fieldMapper.name();
MappedFieldType fieldType = fieldMapper.fieldType();
Expand Down Expand Up @@ -71,6 +74,9 @@ final class FieldTypeLookup {
String aliasName = fieldAliasMapper.name();
String path = fieldAliasMapper.path();
MappedFieldType fieldType = fullNameToFieldType.get(path);
if (fieldType == null) {
continue;
}
fullNameToFieldType.put(aliasName, fieldType);
if (fieldType instanceof DynamicFieldType) {
dynamicFieldTypes.put(aliasName, (DynamicFieldType) fieldType);
Expand All @@ -81,6 +87,12 @@ final class FieldTypeLookup {
// this will override concrete fields with runtime fields that have the same name
fullNameToFieldType.put(fieldType.name(), fieldType);
}
// make all fields into compact+fast immutable maps
this.fullNameToFieldType = Map.copyOf(fullNameToFieldType);
this.dynamicFieldTypes = Map.copyOf(dynamicFieldTypes);
// make values into more compact immutable sets to save memory
fieldToCopiedFields.entrySet().forEach(e -> e.setValue(Set.copyOf(e.getValue())));
this.fieldToCopiedFields = Map.copyOf(fieldToCopiedFields);
}

private static int dotCount(String path) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,10 @@ private CacheKey() {}
private final NestedLookup nestedLookup;
private final FieldTypeLookup fieldTypeLookup;
private final FieldTypeLookup indexTimeLookup; // for index-time scripts, a lookup that does not include runtime fields
private final Map<String, NamedAnalyzer> indexAnalyzersMap = new HashMap<>();
private final List<FieldMapper> indexTimeScriptMappers = new ArrayList<>();
private final Map<String, NamedAnalyzer> indexAnalyzersMap;
private final List<FieldMapper> indexTimeScriptMappers;
private final Mapping mapping;
private final Set<String> completionFields = new HashSet<>();
private final Set<String> completionFields;

/**
* Creates a new {@link MappingLookup} instance by parsing the provided mapping and extracting its field definitions.
Expand Down Expand Up @@ -142,6 +142,9 @@ private MappingLookup(
}
this.nestedLookup = NestedLookup.build(nestedMappers);

final Map<String, NamedAnalyzer> indexAnalyzersMap = new HashMap<>();
final Set<String> completionFields = new HashSet<>();
final List<FieldMapper> indexTimeScriptMappers = new ArrayList<>();
for (FieldMapper mapper : mappers) {
if (objects.containsKey(mapper.name())) {
throw new MapperParsingException("Field [" + mapper.name() + "] is defined both as an object and a field");
Expand Down Expand Up @@ -169,8 +172,12 @@ private MappingLookup(

this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers, mapping.getRoot().runtimeFields());
this.indexTimeLookup = new FieldTypeLookup(mappers, aliasMappers, Collections.emptyList());
this.fieldMappers = Collections.unmodifiableMap(fieldMappers);
this.objectMappers = Collections.unmodifiableMap(objects);
// make all fields into compact+fast immutable maps
this.fieldMappers = Map.copyOf(fieldMappers);
this.objectMappers = Map.copyOf(objects);
this.indexAnalyzersMap = Map.copyOf(indexAnalyzersMap);
this.completionFields = Set.copyOf(completionFields);
this.indexTimeScriptMappers = List.copyOf(indexTimeScriptMappers);

mapping.getRoot()
.runtimeFields()
Expand Down Expand Up @@ -203,8 +210,9 @@ List<FieldMapper> indexTimeScriptMappers() {
}

public NamedAnalyzer indexAnalyzer(String field, Function<String, NamedAnalyzer> unmappedFieldAnalyzer) {
if (this.indexAnalyzersMap.containsKey(field)) {
return this.indexAnalyzersMap.get(field);
final NamedAnalyzer analyzer = indexAnalyzersMap.get(field);
if (analyzer != null) {
return analyzer;
}
return unmappedFieldAnalyzer.apply(field);
}
Expand Down

0 comments on commit 0eda627

Please sign in to comment.