Skip to content

Commit

Permalink
Speed up parsing IndexMetadata from XContent (#86297)
Browse files Browse the repository at this point in the history 
Cleaning up this code and more importantly helping out the master
node with the performance of parsing these a little as we need
to parse these during snapshot deletes which gets very expensive
in case of large index per snapshot counts.
  • Loading branch information
@original-brownbear
original-brownbear committed May 2, 2022
1 parent 4b286d7 commit 3872a1e
Show file tree
Hide file tree
Showing 2 changed files with 121 additions and 134 deletions.
194 changes: 90 additions & 104 deletions server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@

package org.elasticsearch.cluster.metadata;

import org.elasticsearch.Assertions;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.rollover.RolloverInfo;
import org.elasticsearch.action.support.ActiveShardCount;
Expand Down Expand Up @@ -1841,136 +1840,123 @@ public static IndexMetadata fromXContent(XContentParser parser) throws IOExcepti
XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser);
Builder builder = new Builder(parser.currentName());

String currentFieldName = null;
String currentFieldName;
XContentParser.Token token = parser.nextToken();
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser);
boolean mappingVersion = false;
boolean settingsVersion = false;
boolean aliasesVersion = false;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if (KEY_SETTINGS.equals(currentFieldName)) {
builder.settings(Settings.fromXContent(parser));
} else if (KEY_MAPPINGS.equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
String mappingType = currentFieldName;
Map<String, Object> mappingSource = MapBuilder.<String, Object>newMapBuilder()
.put(mappingType, parser.mapOrdered())
.map();
builder.putMapping(new MappingMetadata(mappingType, mappingSource));
} else {
throw new IllegalArgumentException("Unexpected token: " + token);
while ((currentFieldName = parser.nextFieldName()) != null) {
token = parser.nextToken();
if (token == XContentParser.Token.START_OBJECT) {
switch (currentFieldName) {
case KEY_SETTINGS:
builder.settings(Settings.fromXContent(parser));
break;
case KEY_MAPPINGS:
while ((currentFieldName = parser.nextFieldName()) != null) {
token = parser.nextToken();
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser);
builder.putMapping(new MappingMetadata(currentFieldName, Map.of(currentFieldName, parser.mapOrdered())));
}
}
} else if (KEY_ALIASES.equals(currentFieldName)) {
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
builder.putAlias(AliasMetadata.Builder.fromXContent(parser));
}
} else if (KEY_IN_SYNC_ALLOCATIONS.equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_ARRAY) {
String shardId = currentFieldName;
break;
case KEY_ALIASES:
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
builder.putAlias(AliasMetadata.Builder.fromXContent(parser));
}
break;
case KEY_IN_SYNC_ALLOCATIONS:
while ((currentFieldName = parser.nextFieldName()) != null) {
token = parser.nextToken();
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, token, parser);
final int shardId = Integer.parseInt(currentFieldName);
Set<String> allocationIds = new HashSet<>();
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
if (token == XContentParser.Token.VALUE_STRING) {
allocationIds.add(parser.text());
}
}
builder.putInSyncAllocationIds(Integer.valueOf(shardId), allocationIds);
} else {
throw new IllegalArgumentException("Unexpected token: " + token);
builder.putInSyncAllocationIds(shardId, allocationIds);
}
}
} else if (KEY_ROLLOVER_INFOS.equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
break;
case KEY_ROLLOVER_INFOS:
while ((currentFieldName = parser.nextFieldName()) != null) {
token = parser.nextToken();
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser);
builder.putRolloverInfo(RolloverInfo.parse(parser, currentFieldName));
} else {
throw new IllegalArgumentException("Unexpected token: " + token);
}
}
} else if ("warmers".equals(currentFieldName)) {
// TODO: do this in 6.0:
// throw new IllegalArgumentException("Warmers are not supported anymore - are you upgrading from 1.x?");
// ignore: warmers have been removed in 5.0 and are
// simply ignored when upgrading from 2.x
assert Version.CURRENT.major <= 5;
parser.skipChildren();
} else if (KEY_TIMESTAMP_RANGE.equals(currentFieldName)) {
builder.timestampRange(IndexLongFieldRange.fromXContent(parser));
} else {
// assume it's custom index metadata
builder.putCustom(currentFieldName, parser.mapStrings());
break;
case "warmers":
// TODO: do this in 6.0:
// throw new IllegalArgumentException("Warmers are not supported anymore - are you upgrading from 1.x?");
// ignore: warmers have been removed in 5.0 and are
// simply ignored when upgrading from 2.x
assert Version.CURRENT.major <= 5;
parser.skipChildren();
break;
case KEY_TIMESTAMP_RANGE:
builder.timestampRange(IndexLongFieldRange.fromXContent(parser));
break;
default:
// assume it's custom index metadata
builder.putCustom(currentFieldName, parser.mapStrings());
break;
}
} else if (token == XContentParser.Token.START_ARRAY) {
if (KEY_MAPPINGS.equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
builder.putMapping(new MappingMetadata(new CompressedXContent(parser.binaryValue())));
} else {
Map<String, Object> mapping = parser.mapOrdered();
if (mapping.size() == 1) {
String mappingType = mapping.keySet().iterator().next();
builder.putMapping(new MappingMetadata(mappingType, mapping));
switch (currentFieldName) {
case KEY_MAPPINGS:
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
builder.putMapping(new MappingMetadata(new CompressedXContent(parser.binaryValue())));
} else {
Map<String, Object> mapping = parser.mapOrdered();
if (mapping.size() == 1) {
String mappingType = mapping.keySet().iterator().next();
builder.putMapping(new MappingMetadata(mappingType, mapping));
}
}
}
}
} else if (KEY_PRIMARY_TERMS.equals(currentFieldName)) {
ArrayList<Long> list = new ArrayList<>();
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
XContentParserUtils.ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
list.add(parser.longValue());
}
builder.primaryTerms(list.stream().mapToLong(i -> i).toArray());
} else {
throw new IllegalArgumentException("Unexpected field for an array " + currentFieldName);
break;
case KEY_PRIMARY_TERMS:
ArrayList<Long> list = new ArrayList<>();
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
XContentParserUtils.ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
list.add(parser.longValue());
}
builder.primaryTerms(list.stream().mapToLong(i -> i).toArray());
break;
default:
throw new IllegalArgumentException("Unexpected field for an array " + currentFieldName);
}
} else if (token.isValue()) {
if (KEY_STATE.equals(currentFieldName)) {
builder.state(State.fromString(parser.text()));
} else if (KEY_VERSION.equals(currentFieldName)) {
builder.version(parser.longValue());
} else if (KEY_MAPPING_VERSION.equals(currentFieldName)) {
mappingVersion = true;
builder.mappingVersion(parser.longValue());
} else if (KEY_SETTINGS_VERSION.equals(currentFieldName)) {
settingsVersion = true;
builder.settingsVersion(parser.longValue());
} else if (KEY_ALIASES_VERSION.equals(currentFieldName)) {
aliasesVersion = true;
builder.aliasesVersion(parser.longValue());
} else if (KEY_ROUTING_NUM_SHARDS.equals(currentFieldName)) {
builder.setRoutingNumShards(parser.intValue());
} else if (KEY_SYSTEM.equals(currentFieldName)) {
builder.system(parser.booleanValue());
} else {
throw new IllegalArgumentException("Unexpected field [" + currentFieldName + "]");
switch (currentFieldName) {
case KEY_STATE -> builder.state(State.fromString(parser.text()));
case KEY_VERSION -> builder.version(parser.longValue());
case KEY_MAPPING_VERSION -> {
mappingVersion = true;
builder.mappingVersion(parser.longValue());
}
case KEY_SETTINGS_VERSION -> {
settingsVersion = true;
builder.settingsVersion(parser.longValue());
}
case KEY_ALIASES_VERSION -> {
aliasesVersion = true;
builder.aliasesVersion(parser.longValue());
}
case KEY_ROUTING_NUM_SHARDS -> builder.setRoutingNumShards(parser.intValue());
case KEY_SYSTEM -> builder.system(parser.booleanValue());
default -> throw new IllegalArgumentException("Unexpected field [" + currentFieldName + "]");
}
} else {
throw new IllegalArgumentException("Unexpected token " + token);
}
}
XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser);
if (Assertions.ENABLED) {
assert mappingVersion : "mapping version should be present for indices created on or after 6.5.0";
}
if (Assertions.ENABLED) {
assert settingsVersion : "settings version should be present for indices created on or after 6.5.0";
}

Version indexVersion = indexCreatedVersion(builder.settings);
if (Assertions.ENABLED && indexVersion.onOrAfter(Version.V_7_2_0)) {
assert aliasesVersion : "aliases version should be present for indices created on or after 7.2.0";
}
assert mappingVersion : "mapping version should be present for indices created on or after 6.5.0";
assert settingsVersion : "settings version should be present for indices created on or after 6.5.0";
assert indexCreatedVersion(builder.settings).before(Version.V_7_2_0) || aliasesVersion
: "aliases version should be present for indices created on or after 7.2.0";
return builder.build();
}

Expand Down
61 changes: 31 additions & 30 deletions server/src/main/java/org/elasticsearch/common/settings/Settings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -715,46 +715,47 @@ private static Settings fromXContent(XContentParser parser, boolean allowNullVal
private static void fromXContent(XContentParser parser, StringBuilder keyBuilder, Settings.Builder builder, boolean allowNullValues)
throws IOException {
final int length = keyBuilder.length();
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
if (parser.currentToken() == XContentParser.Token.FIELD_NAME) {
keyBuilder.setLength(length);
keyBuilder.append(parser.currentName());
} else if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
keyBuilder.append('.');
fromXContent(parser, keyBuilder, builder, allowNullValues);
} else if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
List<String> list = new ArrayList<>();
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
list.add(parser.text());
} else if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
list.add(parser.text()); // just use the string representation here
} else if (parser.currentToken() == XContentParser.Token.VALUE_BOOLEAN) {
list.add(String.valueOf(parser.text()));
} else {
throw new IllegalStateException("only value lists are allowed in serialized settings");
String currentFieldName;
while ((currentFieldName = parser.nextFieldName()) != null) {
keyBuilder.setLength(length);
keyBuilder.append(currentFieldName);
XContentParser.Token token = parser.nextToken();
switch (token) {
case START_OBJECT -> {
keyBuilder.append('.');
fromXContent(parser, keyBuilder, builder, allowNullValues);
}
case START_ARRAY -> {
List<String> list = new ArrayList<>();
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
switch (token) {
// just use the string representation here
case VALUE_STRING, VALUE_NUMBER, VALUE_BOOLEAN -> list.add(parser.text());
default -> throw new IllegalStateException("only value lists are allowed in serialized settings");
}
}
String key = keyBuilder.toString();
validateValue(key, list, parser, allowNullValues);
builder.putList(key, list);
}
String key = keyBuilder.toString();
validateValue(key, list, parser, allowNullValues);
builder.putList(key, list);
} else if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
String key = keyBuilder.toString();
validateValue(key, null, parser, allowNullValues);
builder.putNull(key);
} else if (parser.currentToken() == XContentParser.Token.VALUE_STRING
|| parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
case VALUE_NULL -> {
String key = keyBuilder.toString();
validateValue(key, null, parser, allowNullValues);
builder.putNull(key);
}
case VALUE_STRING, VALUE_NUMBER -> {
String key = keyBuilder.toString();
String value = parser.text();
validateValue(key, value, parser, allowNullValues);
builder.put(key, value);
} else if (parser.currentToken() == XContentParser.Token.VALUE_BOOLEAN) {
}
case VALUE_BOOLEAN -> {
String key = keyBuilder.toString();
validateValue(key, parser.text(), parser, allowNullValues);
builder.put(key, parser.booleanValue());
} else {
XContentParserUtils.throwUnknownToken(parser.currentToken(), parser);
}
default -> XContentParserUtils.throwUnknownToken(parser.currentToken(), parser);
}
}
}

Expand Down

0 comments on commit 3872a1e

Please sign in to comment.