Skip to content

Commit

Permalink
IndexTemplateMetaData.mappings() should return a single object (#52961)
Browse files Browse the repository at this point in the history 
Currently it returns a map, but that map can only ever hold one set of mappings so
it makes sense to change the signature here.

Relates to #41059
  • Loading branch information
@romseygeek
romseygeek committed Mar 3, 2020
1 parent 9a2a59b commit a76df69
Show file tree
Hide file tree
Showing 9 changed files with 53 additions and 94 deletions.
8 changes: 5 additions & 3 deletions ...-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesResponseTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,7 @@ public void testParsingFromEsResponse() throws IOException {
assertThat(result.order(), equalTo(esIMD.order()));
assertThat(result.version(), equalTo(esIMD.version()));

assertThat(esIMD.mappings().size(), equalTo(1));
BytesArray mappingSource = new BytesArray(esIMD.mappings().valuesIt().next().uncompressed());
BytesArray mappingSource = new BytesArray(esIMD.mappings().uncompressed());
Map<String, Object> expectedMapping =
XContentHelper.convertToMap(mappingSource, true, xContentBuilder.contentType()).v2();
assertThat(result.mappings().sourceAsMap(), equalTo(expectedMapping.get("_doc")));
Expand Down Expand Up @@ -205,7 +204,10 @@ static void toXContent(GetIndexTemplatesResponse response, XContentBuilder build
serverTemplateBuilder.order(clientITMD.order());
serverTemplateBuilder.version(clientITMD.version());
if (clientITMD.mappings() != null) {
serverTemplateBuilder.putMapping(MapperService.SINGLE_MAPPING_NAME, clientITMD.mappings().source());
// The client-side mappings never include a wrapping type, but server-side mappings
// for index templates still do so we need to wrap things here
String mappings = "{\"" + MapperService.SINGLE_MAPPING_NAME + "\": " + clientITMD.mappings().source().string() + "}";
serverTemplateBuilder.putMapping(MapperService.SINGLE_MAPPING_NAME, mappings);
}
serverIndexTemplates.add(serverTemplateBuilder.build());

Expand Down
26 changes: 1 addition & 25 deletions server/src/main/java/org/elasticsearch/cluster/ClusterState.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
Expand Down Expand Up @@ -430,30 +429,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.startObject("templates");
for (ObjectCursor<IndexTemplateMetaData> cursor : metaData().templates().values()) {
IndexTemplateMetaData templateMetaData = cursor.value;
builder.startObject(templateMetaData.name());

builder.field("index_patterns", templateMetaData.patterns());
builder.field("order", templateMetaData.order());

builder.startObject("settings");
Settings settings = templateMetaData.settings();
settings.toXContent(builder, params);
builder.endObject();

builder.startObject("mappings");
for (ObjectObjectCursor<String, CompressedXContent> cursor1 : templateMetaData.mappings()) {
Map<String, Object> mapping = XContentHelper.convertToMap(new BytesArray(cursor1.value.uncompressed()), false).v2();
if (mapping.size() == 1 && mapping.containsKey(cursor1.key)) {
// the type name is the root value, reduce it
mapping = (Map<String, Object>) mapping.get(cursor1.key);
}
builder.field(cursor1.key);
builder.map(mapping);
}
builder.endObject();


builder.endObject();
IndexTemplateMetaData.Builder.toXContentWithTypes(templateMetaData, builder, params);
}
builder.endObject();

Expand Down
79 changes: 35 additions & 44 deletions server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,10 @@
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
Expand Down Expand Up @@ -132,12 +134,15 @@ public Settings settings() {
return this.settings;
}

public ImmutableOpenMap<String, CompressedXContent> mappings() {
return this.mappings;
public CompressedXContent mappings() {
if (this.mappings.isEmpty()) {
return null;
}
return this.mappings.iterator().next().value;
}

public ImmutableOpenMap<String, CompressedXContent> getMappings() {
return this.mappings;
public CompressedXContent getMappings() {
return this.mappings();
}

public ImmutableOpenMap<String, AliasMetaData> aliases() {
Expand Down Expand Up @@ -219,6 +224,19 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeOptionalVInt(version);
}

@Override
public String toString() {
try {
XContentBuilder builder = JsonXContent.contentBuilder();
builder.startObject();
IndexTemplateMetaData.Builder.toXContentWithTypes(this, builder, ToXContent.EMPTY_PARAMS);
builder.endObject();
return Strings.toString(builder);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}

public static class Builder {

private static final Set<String> VALID_FIELDS = Sets.newHashSet(
Expand Down Expand Up @@ -251,7 +269,7 @@ public Builder(IndexTemplateMetaData indexTemplateMetaData) {
patterns(indexTemplateMetaData.patterns());
settings(indexTemplateMetaData.settings());

mappings = ImmutableOpenMap.builder(indexTemplateMetaData.mappings());
mappings = ImmutableOpenMap.builder(indexTemplateMetaData.mappings);
aliases = ImmutableOpenMap.builder(indexTemplateMetaData.aliases());
}

Expand Down Expand Up @@ -371,41 +389,18 @@ private static void toInnerXContent(IndexTemplateMetaData indexTemplateMetaData,
indexTemplateMetaData.settings().toXContent(builder, params);
builder.endObject();

if (params.paramAsBoolean("reduce_mappings", false)) {
// The parameter include_type_name is only ever used in the REST API, where reduce_mappings is
// always set to true. We therefore only check for include_type_name in this branch.
if (includeTypeName == false) {
Map<String, Object> documentMapping = null;
for (ObjectObjectCursor<String, CompressedXContent> cursor : indexTemplateMetaData.mappings()) {
assert documentMapping == null;
byte[] mappingSource = cursor.value.uncompressed();
Map<String, Object> mapping = XContentHelper.convertToMap(new BytesArray(mappingSource), true).v2();
documentMapping = reduceMapping(cursor.key, mapping);
}
includeTypeName &= (params.paramAsBoolean("reduce_mappings", false) == false);

if (documentMapping != null) {
builder.field("mappings", documentMapping);
} else {
builder.startObject("mappings").endObject();
}
} else {
builder.startObject("mappings");
for (ObjectObjectCursor<String, CompressedXContent> cursor : indexTemplateMetaData.mappings()) {
byte[] mappingSource = cursor.value.uncompressed();
Map<String, Object> mapping = XContentHelper.convertToMap(new BytesArray(mappingSource), true).v2();
mapping = reduceMapping(cursor.key, mapping);
builder.field(cursor.key);
builder.map(mapping);
}
builder.endObject();
CompressedXContent m = indexTemplateMetaData.mappings();
if (m != null) {
Map<String, Object> documentMapping = XContentHelper.convertToMap(new BytesArray(m.uncompressed()), true).v2();
if (includeTypeName == false) {
documentMapping = reduceMapping(documentMapping);
}
builder.field("mappings");
builder.map(documentMapping);
} else {
builder.startArray("mappings");
for (ObjectObjectCursor<String, CompressedXContent> cursor : indexTemplateMetaData.mappings()) {
byte[] data = cursor.value.uncompressed();
builder.map(XContentHelper.convertToMap(new BytesArray(data), true).v2());
}
builder.endArray();
builder.startObject("mappings").endObject();
}

builder.startObject("aliases");
Expand All @@ -416,13 +411,9 @@ private static void toInnerXContent(IndexTemplateMetaData indexTemplateMetaData,
}

@SuppressWarnings("unchecked")
private static Map<String, Object> reduceMapping(String type, Map<String, Object> mapping) {
if (mapping.size() == 1 && mapping.containsKey(type)) {
// the type name is the root value, reduce it
return (Map<String, Object>) mapping.get(type);
} else {
return mapping;
}
private static Map<String, Object> reduceMapping(Map<String, Object> mapping) {
assert mapping.keySet().size() == 1;
return (Map<String, Object>) mapping.values().iterator().next();
}

public static IndexTemplateMetaData fromXContent(XContentParser parser, String templateName) throws IOException {
Expand Down
12 changes: 4 additions & 8 deletions server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -414,15 +414,11 @@ static Map<String, Object> parseMappings(String mappingsJson, List<IndexTemplate
Map<String, Object> mappings = MapperService.parseMapping(xContentRegistry, mappingsJson);
// apply templates, merging the mappings into the request mapping if exists
for (IndexTemplateMetaData template : templates) {
for (ObjectObjectCursor<String, CompressedXContent> cursor : template.mappings()) {
String mappingString = cursor.value.string();
// Templates are wrapped with their _type names, which for pre-8x templates may not
// be _doc. For now, we unwrap them based on the _type name, and then re-wrap with
// _doc
// TODO in 9x these will all have a _type of _doc so no re-wrapping will be necessary
Map<String, Object> templateMapping = MapperService.parseMapping(xContentRegistry, mappingString);
CompressedXContent mapping = template.mappings();
if (mapping != null) {
Map<String, Object> templateMapping = MapperService.parseMapping(xContentRegistry, mapping.string());
assert templateMapping.size() == 1 : templateMapping;
assert cursor.key.equals(templateMapping.keySet().iterator().next()) : cursor.key + " != " + templateMapping;
// pre-8x templates may have a wrapper type other than _doc, so we re-wrap things here
templateMapping = Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME,
templateMapping.values().iterator().next());
if (mappings.isEmpty()) {
Expand Down
4 changes: 1 addition & 3 deletions server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,12 @@
import java.util.Arrays;
import java.util.Collections;

import static java.util.Collections.singletonMap;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.Matchers.contains;

public class IndexTemplateMetaDataTests extends ESTestCase {

public void testIndexTemplateMetaDataXContentRoundTrip() throws Exception {
ToXContent.Params params = new ToXContent.MapParams(singletonMap("reduce_mappings", "true"));

String template = "{\"index_patterns\" : [ \".test-*\" ],\"order\" : 1000," +
"\"settings\" : {\"number_of_shards\" : 1,\"number_of_replicas\" : 0}," +
Expand All @@ -62,7 +60,7 @@ public void testIndexTemplateMetaDataXContentRoundTrip() throws Exception {
final BytesReference templateBytesRoundTrip;
try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent)) {
builder.startObject();
IndexTemplateMetaData.Builder.toXContentWithTypes(indexTemplateMetaData, builder, params);
IndexTemplateMetaData.Builder.toXContentWithTypes(indexTemplateMetaData, builder, ToXContent.EMPTY_PARAMS);
builder.endObject();
templateBytesRoundTrip = BytesReference.bytes(builder);
}
Expand Down
8 changes: 3 additions & 5 deletions x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
Expand Down Expand Up @@ -161,14 +160,13 @@ public static boolean checkTemplateExistsAndVersionMatches(
if (templateMeta == null) {
return false;
}
ImmutableOpenMap<String, CompressedXContent> mappings = templateMeta.getMappings();
CompressedXContent mappings = templateMeta.getMappings();
// check all mappings contain correct version in _meta
// we have to parse the source here which is annoying
for (Object typeMapping : mappings.values().toArray()) {
CompressedXContent typeMappingXContent = (CompressedXContent) typeMapping;
if (mappings != null) {
try {
Map<String, Object> typeMappingMap = convertToMap(
new BytesArray(typeMappingXContent.uncompressed()), false,
new BytesArray(mappings.uncompressed()), false,
XContentType.JSON).v2();
// should always contain one entry with key = typename
assert (typeMappingMap.size() == 1);
Expand Down
4 changes: 1 addition & 3 deletions x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.indices.SystemIndexDescriptor;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.plugins.SystemIndexPlugin;
Expand Down Expand Up @@ -56,8 +55,7 @@ public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDat
templates.keySet().removeIf(OLD_LOGSTASH_INDEX_NAME::equals);
TemplateUtils.loadTemplateIntoMap("/" + LOGSTASH_TEMPLATE_FILE_NAME + ".json", templates, LOGSTASH_INDEX_TEMPLATE_NAME,
Version.CURRENT.toString(), TEMPLATE_VERSION_VARIABLE, LogManager.getLogger(Logstash.class));
//internal representation of typeless templates requires the default "_doc" type, which is also required for internal templates
assert templates.get(LOGSTASH_INDEX_TEMPLATE_NAME).mappings().get(MapperService.SINGLE_MAPPING_NAME) != null;
assert templates.get(LOGSTASH_INDEX_TEMPLATE_NAME).mappings() != null;
return templates;
};
}
Expand Down
2 changes: 1 addition & 1 deletion ...va/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ private void assertTemplateNotUpdated() {
final String name = MonitoringTemplateUtils.templateName(system.getSystem());

for (IndexTemplateMetaData template : client().admin().indices().prepareGetTemplates(name).get().getIndexTemplates()) {
final String docMapping = template.getMappings().get("_doc").toString();
final String docMapping = template.getMappings().toString();

assertThat(docMapping, notNullValue());
assertThat(docMapping, containsString("test"));
Expand Down
4 changes: 2 additions & 2 deletions ...m/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -362,7 +362,7 @@ protected static void installLatestVersionedIndexTemplateIfRequired(
// Installing the template involves communication with the master node, so it's more expensive but much rarer
try {
IndexTemplateMetaData indexTemplateMetaData = getIndexTemplateMetaData();
BytesReference jsonMappings = new BytesArray(indexTemplateMetaData.mappings().get(SINGLE_MAPPING_NAME).uncompressed());
BytesReference jsonMappings = new BytesArray(indexTemplateMetaData.mappings().uncompressed());
PutIndexTemplateRequest request = new PutIndexTemplateRequest(TransformInternalIndexConstants.LATEST_INDEX_VERSIONED_NAME)
.patterns(indexTemplateMetaData.patterns())
.version(indexTemplateMetaData.version())
Expand Down Expand Up @@ -396,7 +396,7 @@ protected static void installLatestAuditIndexTemplateIfRequired(
// Installing the template involves communication with the master node, so it's more expensive but much rarer
try {
IndexTemplateMetaData indexTemplateMetaData = getAuditIndexTemplateMetaData();
BytesReference jsonMappings = new BytesArray(indexTemplateMetaData.mappings().get(SINGLE_MAPPING_NAME).uncompressed());
BytesReference jsonMappings = new BytesArray(indexTemplateMetaData.mappings().uncompressed());
PutIndexTemplateRequest request = new PutIndexTemplateRequest(TransformInternalIndexConstants.AUDIT_INDEX).patterns(
indexTemplateMetaData.patterns()
)
Expand Down

0 comments on commit a76df69

Please sign in to comment.