diff --git a/docs/changelog/78997.yaml b/docs/changelog/78997.yaml new file mode 100644 index 0000000000000..b937ca9997f0c --- /dev/null +++ b/docs/changelog/78997.yaml @@ -0,0 +1,5 @@ +pr: 78997 +summary: Add 'flatten' parameter to object mappers +area: Mapping +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 7a919c21e5dfd..5f1199304b4da 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -23,7 +23,8 @@ public class DocumentMapper { * @return the newly created document mapper */ public static DocumentMapper createEmpty(MapperService mapperService) { - RootObjectMapper root = new RootObjectMapper.Builder(MapperService.SINGLE_MAPPING_NAME).build(MapperBuilderContext.ROOT); + RootObjectMapper root = new RootObjectMapper.Builder(MapperService.SINGLE_MAPPING_NAME, false) + .build(MapperBuilderContext.ROOT); MetadataFieldMapper[] metadata = mapperService.getMetadataMappers().values().toArray(new MetadataFieldMapper[0]); Mapping mapping = new Mapping(root, metadata, null); return new DocumentMapper(mapperService.documentParser(), mapping); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 24b81241592b8..6be219863b71d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -17,11 +17,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -29,11 +25,14 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentType; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; @@ -102,8 +101,7 @@ public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingL context.reorderParentAndGetDocs(), context.sourceToParse().source(), context.sourceToParse().getXContentType(), - createDynamicUpdate(mappingLookup, - context.getDynamicMappers(), context.getDynamicRuntimeFields()) + createDynamicUpdate(context) ); } @@ -250,171 +248,21 @@ private static String[] splitAndValidatePath(String fullFieldPath) { } } - /** - * Creates a Mapping containing any dynamically added fields, or returns null if there were no dynamic mappings. - */ - static Mapping createDynamicUpdate(MappingLookup mappingLookup, - List dynamicMappers, - List dynamicRuntimeFields) { - if (dynamicMappers.isEmpty() && dynamicRuntimeFields.isEmpty()) { + static Mapping createDynamicUpdate(DocumentParserContext context) { + if (context.getDynamicMappers().isEmpty() && context.getDynamicRuntimeFields().isEmpty()) { return null; } - RootObjectMapper root; - if (dynamicMappers.isEmpty() == false) { - root = createDynamicUpdate(mappingLookup, dynamicMappers); - root.fixRedundantIncludes(); - } else { - root = mappingLookup.getMapping().getRoot().copyAndReset(); - } - root.addRuntimeFields(dynamicRuntimeFields); - return mappingLookup.getMapping().mappingUpdate(root); - } - - private static RootObjectMapper createDynamicUpdate(MappingLookup mappingLookup, - List dynamicMappers) { - - // We build a mapping by first sorting the mappers, so that all mappers containing a common prefix - // will be processed in a contiguous block. When the prefix is no longer seen, we pop the extra elements - // off the stack, merging them upwards into the existing mappers. - dynamicMappers.sort(Comparator.comparing(Mapper::name)); - Iterator dynamicMapperItr = dynamicMappers.iterator(); - List parentMappers = new ArrayList<>(); - Mapper firstUpdate = dynamicMapperItr.next(); - parentMappers.add(createUpdate(mappingLookup.getMapping().getRoot(), splitAndValidatePath(firstUpdate.name()), 0, firstUpdate)); - Mapper previousMapper = null; - while (dynamicMapperItr.hasNext()) { - Mapper newMapper = dynamicMapperItr.next(); - if (previousMapper != null && newMapper.name().equals(previousMapper.name())) { - // We can see the same mapper more than once, for example, if we had foo.bar and foo.baz, where - // foo did not yet exist. This will create 2 copies in dynamic mappings, which should be identical. - // Here we just skip over the duplicates, but we merge them to ensure there are no conflicts. - newMapper.merge(previousMapper); - continue; - } - previousMapper = newMapper; - String[] nameParts = splitAndValidatePath(newMapper.name()); - - // We first need the stack to only contain mappers in common with the previously processed mapper - // For example, if the first mapper processed was a.b.c, and we now have a.d, the stack will contain - // a.b, and we want to merge b back into the stack so it just contains a - int i = removeUncommonMappers(parentMappers, nameParts); - - // Then we need to add back mappers that may already exist within the stack, but are not on it. - // For example, if we processed a.b, followed by an object mapper a.c.d, and now are adding a.c.d.e - // then the stack will only have a on it because we will have already merged a.c.d into the stack. - // So we need to pull a.c, followed by a.c.d, onto the stack so e can be added to the end. - i = expandCommonMappers(parentMappers, nameParts, i); - - // If there are still parents of the new mapper which are not on the stack, we need to pull them - // from the existing mappings. In order to maintain the invariant that the stack only contains - // fields which are updated, we cannot simply add the existing mappers to the stack, since they - // may have other subfields which will not be updated. Instead, we pull the mapper from the existing - // mappings, and build an update with only the new mapper and its parents. This then becomes our - // "new mapper", and can be added to the stack. - if (i < nameParts.length - 1) { - newMapper = createExistingMapperUpdate(parentMappers, nameParts, i, mappingLookup, newMapper); - } - - if (newMapper instanceof ObjectMapper) { - parentMappers.add((ObjectMapper) newMapper); - } else { - addToLastMapper(parentMappers, newMapper, true); - } + RootObjectMapper.Builder rootBuilder = context.updateRoot(); + for (Mapper mapper : context.getDynamicMappers()) { + splitAndValidatePath(mapper.name()); + rootBuilder.addDynamic(mapper.name(), null, mapper, context); } - popMappers(parentMappers, 1, true); - assert parentMappers.size() == 1; - return (RootObjectMapper) parentMappers.get(0); - } - - private static void popMappers(List parentMappers, int keepBefore, boolean merge) { - assert keepBefore >= 1; // never remove the root mapper - // pop off parent mappers not needed by the current mapper, - // merging them backwards since they are immutable - for (int i = parentMappers.size() - 1; i >= keepBefore; --i) { - addToLastMapper(parentMappers, parentMappers.remove(i), merge); + for (RuntimeField runtimeField : context.getDynamicRuntimeFields()) { + rootBuilder.addRuntimeField(runtimeField); } - } - - /** - * Adds a mapper as an update into the last mapper. If merge is true, the new mapper - * will be merged in with other child mappers of the last parent, otherwise it will be a new update. - */ - private static void addToLastMapper(List parentMappers, Mapper mapper, boolean merge) { - assert parentMappers.size() >= 1; - int lastIndex = parentMappers.size() - 1; - ObjectMapper withNewMapper = parentMappers.get(lastIndex).mappingUpdate(mapper); - if (merge) { - withNewMapper = parentMappers.get(lastIndex).merge(withNewMapper); - } - parentMappers.set(lastIndex, withNewMapper); - } - - /** - * Removes mappers that exist on the stack, but are not part of the path of the current nameParts, - * Returns the next unprocessed index from nameParts. - */ - private static int removeUncommonMappers(List parentMappers, String[] nameParts) { - int keepBefore = 1; - while (keepBefore < parentMappers.size() && - parentMappers.get(keepBefore).simpleName().equals(nameParts[keepBefore - 1])) { - ++keepBefore; - } - popMappers(parentMappers, keepBefore, true); - return keepBefore - 1; - } - - /** - * Adds mappers from the end of the stack that exist as updates within those mappers. - * Returns the next unprocessed index from nameParts. - */ - private static int expandCommonMappers(List parentMappers, String[] nameParts, int i) { - ObjectMapper last = parentMappers.get(parentMappers.size() - 1); - while (i < nameParts.length - 1 && last.getMapper(nameParts[i]) != null) { - Mapper newLast = last.getMapper(nameParts[i]); - assert newLast instanceof ObjectMapper; - last = (ObjectMapper) newLast; - parentMappers.add(last); - ++i; - } - return i; - } - - /** - * Creates an update for intermediate object mappers that are not on the stack, but parents of newMapper. - */ - private static ObjectMapper createExistingMapperUpdate(List parentMappers, String[] nameParts, int i, - MappingLookup mappingLookup, Mapper newMapper) { - String updateParentName = nameParts[i]; - final ObjectMapper lastParent = parentMappers.get(parentMappers.size() - 1); - if (parentMappers.size() > 1) { - // only prefix with parent mapper if the parent mapper isn't the root (which has a fake name) - updateParentName = lastParent.name() + '.' + nameParts[i]; - } - ObjectMapper updateParent = mappingLookup.objectMappers().get(updateParentName); - assert updateParent != null : updateParentName + " doesn't exist"; - return createUpdate(updateParent, nameParts, i + 1, newMapper); - } - - /** - * Build an update for the parent which will contain the given mapper and any intermediate fields. - */ - private static ObjectMapper createUpdate(ObjectMapper parent, String[] nameParts, int i, Mapper mapper) { - List parentMappers = new ArrayList<>(); - ObjectMapper previousIntermediate = parent; - for (; i < nameParts.length - 1; ++i) { - Mapper intermediate = previousIntermediate.getMapper(nameParts[i]); - assert intermediate != null : "Field " + previousIntermediate.name() + " does not have a subfield " + nameParts[i]; - assert intermediate instanceof ObjectMapper; - parentMappers.add((ObjectMapper) intermediate); - previousIntermediate = (ObjectMapper) intermediate; - } - if (parentMappers.isEmpty() == false) { - // add the new mapper to the stack, and pop down to the original parent level - addToLastMapper(parentMappers, mapper, false); - popMappers(parentMappers, 1, false); - mapper = parentMappers.get(0); - } - return parent.mappingUpdate(mapper); + RootObjectMapper root = rootBuilder.build(MapperBuilderContext.ROOT); + root.fixRedundantIncludes(); + return context.mappingLookup().getMapping().mappingUpdate(root); } static void parseObjectOrNested(DocumentParserContext context, ObjectMapper mapper) throws IOException { @@ -679,12 +527,24 @@ private static void parseValue(final DocumentParserContext context, ObjectMapper if (mapper != null) { parseObjectOrField(context, mapper); } else { - currentFieldName = paths[paths.length - 1]; - Tuple parentMapperTuple = getDynamicParentMapper(context, paths, parentMapper); - parentMapper = parentMapperTuple.v2(); - parseDynamicValue(context, parentMapper, currentFieldName, token); - for (int i = 0; i < parentMapperTuple.v1(); i++) { - context.path().remove(); + if (parentMapper.flatten) { + parseDynamicValue(context, parentMapper, currentFieldName, token); + } else { + Tuple parentMapperTuple = getDynamicParentMapper(context, paths, parentMapper); + parentMapper = parentMapperTuple.v2(); + int pathLength = parentMapperTuple.v1(); + // If our dynamic parent mapper is flattened, then we can't just assume that our name + // is the last path part. We instead need to construct it by concatenating all path + // parts from the parent mapper onwards. + StringBuilder compositeFieldName = new StringBuilder(paths[pathLength]); + while (pathLength < paths.length - 1) { + pathLength++; + compositeFieldName.append(".").append(paths[pathLength]); + } + parseDynamicValue(context, parentMapper, compositeFieldName.toString(), token); + for (int i = 0; i < parentMapperTuple.v1(); i++) { + context.path().remove(); + } } } } @@ -813,6 +673,9 @@ private static Tuple getDynamicParentMapper(DocumentParse context.path().add(paths[i]); pathsAdded++; parent = mapper; + if (parent.flatten) { + break; + } } return new Tuple<>(pathsAdded, mapper); } @@ -859,6 +722,12 @@ private static Mapper getMapper(final DocumentParserContext context, return mapper; } + // Is the full name of the mapper a direct child of this object? + mapper = objectMapper.getMapper(fieldPath); + if (mapper != null) { + return mapper; + } + for (int i = 0; i < subfields.length - 1; ++i) { mapper = objectMapper.getMapper(subfields[i]); if (mapper instanceof ObjectMapper == false) { @@ -976,7 +845,14 @@ protected String contentType() { private static class NoOpObjectMapper extends ObjectMapper { NoOpObjectMapper(String name, String fullPath) { - super(name, fullPath, new Explicit<>(true, false), Dynamic.RUNTIME, Collections.emptyMap()); + super( + name, + fullPath, + new Explicit<>(true, false), + false, + Dynamic.RUNTIME, + Collections.emptyMap() + ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 9c4f5f5e82822..fc157af32ac0f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -257,6 +257,13 @@ public final List getDynamicRuntimeFields() { */ public abstract Iterable nonRootDocuments(); + /** + * @return a RootObjectMapper.Builder to be used to construct a dynamic mapping update + */ + public final RootObjectMapper.Builder updateRoot() { + return mappingLookup.getMapping().getRoot().newBuilder(); + } + /** * Return a new context that will be within a copy-to operation. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java index 1820955a475d7..8bf3ea4c02a2b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java @@ -12,10 +12,10 @@ import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; -import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.index.mapper.ObjectMapper.Dynamic; import org.elasticsearch.script.ScriptCompiler; +import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; import java.time.format.DateTimeParseException; @@ -127,7 +127,8 @@ Mapper createDynamicObjectMapper(DocumentParserContext context, String name) { Mapper mapper = createObjectMapperFromTemplate(context, name); return mapper != null ? mapper - : new ObjectMapper.Builder(name).enabled(true).build(MapperBuilderContext.forPath(context.path())); + : new ObjectMapper.Builder(name, false).enabled(true).build(MapperBuilderContext.forPath(context.path()) + ); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index ebb743dca2402..e110db2bcf1ee 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.common.Strings; import org.elasticsearch.xcontent.ToXContentFragment; import java.util.Map; @@ -66,4 +67,8 @@ public final String simpleName() { */ public abstract void validate(MappingLookup mappers); + @Override + public String toString() { + return name() + ":" + Strings.toString(this); + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java index ca6fe392236e3..1d94e0af096e6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java @@ -11,13 +11,13 @@ import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.index.mapper.MapperService.MergeReason; import java.io.IOException; import java.io.UncheckedIOException; @@ -35,7 +35,7 @@ public final class Mapping implements ToXContentFragment { public static final Mapping EMPTY = new Mapping( - new RootObjectMapper.Builder("_doc").build(MapperBuilderContext.ROOT), + new RootObjectMapper.Builder("_doc", false).build(MapperBuilderContext.ROOT), new MetadataFieldMapper[0], null); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java index 2a1a8e073e3d8..dc38b128721c8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java @@ -60,6 +60,21 @@ public MappingParserContext(Function similarityLooku this.idFieldDataEnabled = idFieldDataEnabled; } + protected MappingParserContext(MappingParserContext in) { + this( + in.similarityLookupService, + in.typeParsers, + in.runtimeFieldParsers, + in.indexVersionCreated, + in.searchExecutionContextSupplier, + in.dateFormatter, + in.scriptCompiler, + in.indexAnalyzers, + in.indexSettings, + in.idFieldDataEnabled + ); + } + public IndexAnalyzers getIndexAnalyzers() { return indexAnalyzers; } @@ -133,9 +148,7 @@ MappingParserContext createMultiFieldContext(MappingParserContext in) { private static class MultiFieldParserContext extends MappingParserContext { MultiFieldParserContext(MappingParserContext in) { - super(in.similarityLookupService, in.typeParsers, in.runtimeFieldParsers, in.indexVersionCreated, - in.searchExecutionContextSupplier, in.dateFormatter, in.scriptCompiler, in.indexAnalyzers, in.indexSettings, - in.idFieldDataEnabled); + super(in); } @Override @@ -146,9 +159,7 @@ public boolean isWithinMultiField() { static class DynamicTemplateParserContext extends MappingParserContext { DynamicTemplateParserContext(MappingParserContext in) { - super(in.similarityLookupService, in.typeParsers, in.runtimeFieldParsers, in.indexVersionCreated, - in.searchExecutionContextSupplier, in.dateFormatter, in.scriptCompiler, in.indexAnalyzers, in.indexSettings, - in.idFieldDataEnabled); + super(in); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java index f14918e9da802..f79bb7ab66f53 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java @@ -34,7 +34,7 @@ public static class Builder extends ObjectMapper.Builder { private final Version indexCreatedVersion; public Builder(String name, Version indexCreatedVersion) { - super(name); + super(name, false); this.indexCreatedVersion = indexCreatedVersion; } @@ -94,6 +94,7 @@ protected static void parseNested(String name, Map node, NestedO private Explicit includeInParent; private final String nestedTypePath; private final Query nestedTypeFilter; + private final Version indexVersionCreated; NestedObjectMapper( String name, @@ -101,7 +102,7 @@ protected static void parseNested(String name, Map node, NestedO Map mappers, Builder builder ) { - super(name, fullPath, builder.enabled, builder.dynamic, mappers); + super(name, fullPath, builder.enabled, false, builder.dynamic, mappers); if (builder.indexCreatedVersion.before(Version.V_8_0_0)) { this.nestedTypePath = "__" + fullPath; } else { @@ -110,6 +111,7 @@ protected static void parseNested(String name, Map node, NestedO this.nestedTypeFilter = NestedPathFieldMapper.filter(builder.indexCreatedVersion, nestedTypePath); this.includeInParent = builder.includeInParent; this.includeInRoot = builder.includeInRoot; + this.indexVersionCreated = builder.indexCreatedVersion; } public Query nestedTypeFilter() { @@ -145,6 +147,16 @@ public Map getChildren() { return Collections.unmodifiableMap(this.mappers); } + @Override + public ObjectMapper.Builder newBuilder() { + NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(simpleName(), indexVersionCreated); + builder.enabled = enabled; + builder.dynamic = dynamic; + builder.includeInRoot = includeInRoot; + builder.includeInParent = includeInParent; + return builder; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(simpleName()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 524758243ca58..0c53d94fd7295 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -13,10 +13,10 @@ import org.elasticsearch.common.collect.CopyOnWriteHashMap; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.xcontent.ToXContent; -import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.mapper.MapperService.MergeReason; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.ArrayList; @@ -28,12 +28,15 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.function.BiFunction; public class ObjectMapper extends Mapper implements Cloneable { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(ObjectMapper.class); public static final String CONTENT_TYPE = "object"; + public static final String FLATTEN = "flatten"; + public static class Defaults { public static final boolean ENABLED = true; } @@ -62,13 +65,15 @@ DynamicFieldsBuilder getDynamicFieldsBuilder() { public static class Builder extends Mapper.Builder { protected Explicit enabled = new Explicit<>(true, false); + protected final boolean flatten; protected Dynamic dynamic; protected final List mappersBuilders = new ArrayList<>(); - public Builder(String name) { + public Builder(String name, boolean flatten) { super(name); + this.flatten = flatten; } public Builder enabled(boolean enabled) { @@ -86,6 +91,54 @@ public Builder add(Mapper.Builder builder) { return this; } + /** + * Adds a dynamically created Mapper to this builder. + * + * @param name the name of the Mapper, including object prefixes + * @param prefix the object prefix of this mapper + * @param mapper the mapper to add + * @param context the DocumentParserContext in which the mapper has been built + */ + public void addDynamic(String name, String prefix, Mapper mapper, DocumentParserContext context) { + // If we're a flattened mapper, or if the mapper to add has no dots and is therefore + // a leaf mapper, we just add it here + if (flatten || name.contains(".") == false) { + mappersBuilders.add(new Mapper.Builder(name) { + @Override + public Mapper build(MapperBuilderContext context) { + return mapper; + } + }); + } + // otherwise we strip off the first object path of the mapper name, load or create + // the relevant object mapper, and then recurse down into it, passing the remainder + // of the mapper name. So for a mapper 'foo.bar.baz', we locate 'foo' and then + // call addDynamic on it with the name 'bar.baz'. + else { + int firstDotIndex = name.indexOf("."); + String childName = name.substring(0, firstDotIndex); + String fullChildName = prefix == null ? childName : prefix + "." + childName; + ObjectMapper.Builder childBuilder = findChild(childName, fullChildName, context); + childBuilder.addDynamic(name.substring(firstDotIndex + 1), fullChildName, mapper, context); + mappersBuilders.add(childBuilder); + } + } + + private ObjectMapper.Builder findChild(String childName, String fullChildName, DocumentParserContext context) { + // does the child mapper already exist? if so, use that + ObjectMapper child = context.mappingLookup().objectMappers().get(fullChildName); + if (child != null) { + return child.newBuilder(); + } + // has the child mapper been added as a dynamic update already? + child = context.getObjectMapper(fullChildName); + if (child != null) { + return child.newBuilder(); + } + // create a new child mapper + return new ObjectMapper.Builder(childName, false); + } + protected final Map buildMappers(boolean root, MapperBuilderContext context) { if (root == false) { context = context.createChildContext(name); @@ -93,6 +146,10 @@ protected final Map buildMappers(boolean root, MapperBuilderCont Map mappers = new HashMap<>(); for (Mapper.Builder builder : mappersBuilders) { Mapper mapper = builder.build(context); + if (flatten && mapper instanceof ObjectMapper) { + throw new MapperParsingException( + "Mapper [" + name + "] can only contain flat leaf fields, but [" + mapper.name() + "] is an object"); + } Mapper existing = mappers.get(mapper.simpleName()); if (existing != null) { mapper = existing.merge(mapper); @@ -104,7 +161,7 @@ protected final Map buildMappers(boolean root, MapperBuilderCont @Override public ObjectMapper build(MapperBuilderContext context) { - return new ObjectMapper(name, context.buildFullName(name), enabled, dynamic, buildMappers(false, context)); + return new ObjectMapper(name, context.buildFullName(name), enabled, flatten, dynamic, buildMappers(false, context)); } } @@ -114,15 +171,39 @@ public Mapper.Builder parse(String name, Map node, MappingParserContext parserContext) throws MapperParsingException { - ObjectMapper.Builder builder = new Builder(name); - for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { - Map.Entry entry = iterator.next(); - String fieldName = entry.getKey(); - Object fieldNode = entry.getValue(); - if (parseObjectOrDocumentTypeProperties(fieldName, fieldNode, parserContext, builder)) { - iterator.remove(); + + return parse(name, false, (n, c) -> { + ObjectMapper.Builder builder = new Builder(n, parseFlatten(node)); + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + String fieldName = entry.getKey(); + Object fieldNode = entry.getValue(); + if (parseObjectOrDocumentTypeProperties(fieldName, fieldNode, parserContext, builder)) { + iterator.remove(); + } } + return builder; + }, parserContext); + } + + private static Mapper.Builder parse( + String name, + boolean allowDotsInFieldNames, + BiFunction parser, + MappingParserContext parserContext + ) { + if (allowDotsInFieldNames) { + return parser.apply(name, parserContext); } + String[] fieldNameParts = name.split("\\."); + String realFieldName = fieldNameParts[fieldNameParts.length - 1]; + Mapper.Builder builder = parser.apply(realFieldName, parserContext); + for (int i = fieldNameParts.length - 2; i >= 0; --i) { + ObjectMapper.Builder intermediate = new ObjectMapper.Builder(fieldNameParts[i], false); + intermediate.add(builder); + builder = intermediate; + } + return builder; } @@ -199,16 +280,12 @@ protected static void parseProperties(ObjectMapper.Builder objBuilder, if (typeParser == null) { throw new MapperParsingException("No handler for type [" + type + "] declared on field [" + fieldName + "]"); } - String[] fieldNameParts = fieldName.split("\\."); - String realFieldName = fieldNameParts[fieldNameParts.length - 1]; - Mapper.Builder fieldBuilder = typeParser.parse(realFieldName, propNode, parserContext); - for (int i = fieldNameParts.length - 2; i >= 0; --i) { - ObjectMapper.Builder intermediate - = new ObjectMapper.Builder(fieldNameParts[i]); - intermediate.add(fieldBuilder); - fieldBuilder = intermediate; - } - objBuilder.add(fieldBuilder); + objBuilder.add(parse( + fieldName, + objBuilder.flatten, + (n, c) -> typeParser.parse(n, propNode, c), + parserContext + )); propNode.remove("type"); MappingParser.checkNoRemainingFields(fieldName, propNode); iterator.remove(); @@ -222,23 +299,40 @@ protected static void parseProperties(ObjectMapper.Builder objBuilder, MappingParser.checkNoRemainingFields(propsNode, "DocType mapping definition has unsupported parameters: "); } + + protected static boolean parseFlatten(Map node) { + if (node.containsKey(FLATTEN) == false) { + return false; + } + boolean value = XContentMapValues.nodeBooleanValue(node.get(FLATTEN), FLATTEN); + node.remove(FLATTEN); + return value; + } } private final String fullPath; protected Explicit enabled; - + protected boolean flatten; protected volatile Dynamic dynamic; protected volatile CopyOnWriteHashMap mappers; - ObjectMapper(String name, String fullPath, Explicit enabled, Dynamic dynamic, Map mappers) { + ObjectMapper( + String name, + String fullPath, + Explicit enabled, + boolean flatten, + Dynamic dynamic, + Map mappers + ) { super(name); if (name.isEmpty()) { throw new IllegalArgumentException("name cannot be empty string"); } this.fullPath = fullPath; this.enabled = enabled; + this.flatten = flatten; this.dynamic = dynamic; if (mappers == null) { this.mappers = new CopyOnWriteHashMap<>(); @@ -258,20 +352,14 @@ protected ObjectMapper clone() { return clone; } - ObjectMapper copyAndReset() { - ObjectMapper copy = clone(); - // reset the sub mappers - copy.mappers = new CopyOnWriteHashMap<>(); - return copy; - } - /** - * Build a mapping update with the provided sub mapping update. + * @return a Builder that will produce an empty ObjectMapper with the same configuration as this one */ - final ObjectMapper mappingUpdate(Mapper mapper) { - ObjectMapper mappingUpdate = copyAndReset(); - mappingUpdate.putMapper(mapper); - return mappingUpdate; + public ObjectMapper.Builder newBuilder() { + ObjectMapper.Builder builder = new ObjectMapper.Builder(simpleName(), flatten); + builder.enabled = this.enabled; + builder.dynamic = this.dynamic; + return builder; } @Override @@ -353,6 +441,17 @@ protected void doMerge(final ObjectMapper mergeWith, MergeReason reason) { } } + if (reason == MergeReason.INDEX_TEMPLATE) { + this.flatten = mergeWith.flatten; + } else { + if (this.flatten != mergeWith.flatten) { + throw new MapperParsingException( + "Can't change parameter [flatten] from [" + + this.flatten + "] to [" + mergeWith.flatten + "]" + ); + } + } + for (Mapper mergeWithMapper : mergeWith) { Mapper mergeIntoMapper = mappers.get(mergeWithMapper.simpleName()); @@ -399,7 +498,9 @@ void toXContent(XContentBuilder builder, Params params, ToXContent custom) throw if (isEnabled() != Defaults.ENABLED) { builder.field("enabled", enabled.value()); } - + if (flatten) { + builder.field("flatten", flatten); + } if (custom != null) { custom.toXContent(builder, params); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java index ed2e51660d343..d2d98fe721ac3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java @@ -15,10 +15,10 @@ import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.time.DateFormatter; -import org.elasticsearch.xcontent.ToXContent; -import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.DynamicTemplate.XContentFieldType; import org.elasticsearch.index.mapper.MapperService.MergeReason; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.ArrayList; @@ -67,10 +67,10 @@ public static class Builder extends ObjectMapper.Builder { protected Explicit dynamicDateTimeFormatters = new Explicit<>(Defaults.DYNAMIC_DATE_TIME_FORMATTERS, false); protected Explicit dateDetection = new Explicit<>(Defaults.DATE_DETECTION, false); protected Explicit numericDetection = new Explicit<>(Defaults.NUMERIC_DETECTION, false); - protected Map runtimeFields; + protected final Map runtimeFields = new HashMap<>(); - public Builder(String name) { - super(name); + public Builder(String name, boolean flatten) { + super(name, flatten); } public Builder dynamicDateTimeFormatter(Collection dateTimeFormatters) { @@ -89,18 +89,29 @@ public RootObjectMapper.Builder add(Mapper.Builder builder) { return this; } + public RootObjectMapper.Builder addRuntimeField(RuntimeField runtimeField) { + this.runtimeFields.put(runtimeField.name(), runtimeField); + return this; + } + public RootObjectMapper.Builder setRuntime(Map runtimeFields) { - this.runtimeFields = runtimeFields; + this.runtimeFields.putAll(runtimeFields); return this; } @Override public RootObjectMapper build(MapperBuilderContext context) { - return new RootObjectMapper(name, enabled, dynamic, buildMappers(true, context), + return new RootObjectMapper( + name, + enabled, + flatten, + dynamic, + buildMappers(true, context), runtimeFields == null ? Collections.emptyMap() : runtimeFields, dynamicDateTimeFormatters, dynamicTemplates, - dateDetection, numericDetection); + dateDetection, + numericDetection); } } @@ -140,7 +151,7 @@ static final class TypeParser extends ObjectMapper.TypeParser { @Override public RootObjectMapper.Builder parse(String name, Map node, MappingParserContext parserContext) throws MapperParsingException { - RootObjectMapper.Builder builder = new Builder(name); + RootObjectMapper.Builder builder = new Builder(name, parseFlatten(node)); Iterator> iterator = node.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry entry = iterator.next(); @@ -235,11 +246,19 @@ private boolean processField(RootObjectMapper.Builder builder, private Explicit dynamicTemplates; private Map runtimeFields; - RootObjectMapper(String name, Explicit enabled, Dynamic dynamic, Map mappers, - Map runtimeFields, - Explicit dynamicDateTimeFormatters, Explicit dynamicTemplates, - Explicit dateDetection, Explicit numericDetection) { - super(name, name, enabled, dynamic, mappers); + RootObjectMapper( + String name, + Explicit enabled, + boolean flatten, + Dynamic dynamic, + Map mappers, + Map runtimeFields, + Explicit dynamicDateTimeFormatters, + Explicit dynamicTemplates, + Explicit dateDetection, + Explicit numericDetection + ) { + super(name, name, enabled, flatten, dynamic, mappers); this.runtimeFields = runtimeFields; this.dynamicTemplates = dynamicTemplates; this.dynamicDateTimeFormatters = dynamicDateTimeFormatters; @@ -255,18 +274,11 @@ protected ObjectMapper clone() { } @Override - RootObjectMapper copyAndReset() { - RootObjectMapper copy = (RootObjectMapper) super.copyAndReset(); - // for dynamic updates, no need to carry root-specific options, we just - // set everything to their implicit default value so that they are not - // applied at merge time - copy.dynamicTemplates = new Explicit<>(new DynamicTemplate[0], false); - copy.dynamicDateTimeFormatters = new Explicit<>(Defaults.DYNAMIC_DATE_TIME_FORMATTERS, false); - copy.dateDetection = new Explicit<>(Defaults.DATE_DETECTION, false); - copy.numericDetection = new Explicit<>(Defaults.NUMERIC_DETECTION, false); - //also no need to carry the already defined runtime fields, only new ones need to be added - copy.runtimeFields.clear(); - return copy; + public RootObjectMapper.Builder newBuilder() { + RootObjectMapper.Builder builder = new RootObjectMapper.Builder(name(), flatten); + builder.enabled = enabled; + builder.dynamic = dynamic; + return builder; } /** diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 1b141adebc0b5..b34f047a5731d 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -569,7 +569,7 @@ public void testRolloverClusterStateForDataStream() throws Exception { when(env.sharedDataFile()).thenReturn(null); AllocationService allocationService = mock(AllocationService.class); when(allocationService.reroute(any(ClusterState.class), any(String.class))).then(i -> i.getArguments()[0]); - RootObjectMapper.Builder root = new RootObjectMapper.Builder("_doc"); + RootObjectMapper.Builder root = new RootObjectMapper.Builder("_doc", false); root.add(new DateFieldMapper.Builder(dataStream.getTimeStampField().getName(), DateFieldMapper.Resolution.MILLISECONDS, DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, ScriptCompiler.NONE, true, Version.CURRENT)); MetadataFieldMapper dtfm = getDataStreamTimestampFieldMapper(); diff --git a/server/src/test/java/org/elasticsearch/cluster/action/index/MappingUpdatedActionTests.java b/server/src/test/java/org/elasticsearch/cluster/action/index/MappingUpdatedActionTests.java index ae229c6df6444..4ac592b7b6697 100644 --- a/server/src/test/java/org/elasticsearch/cluster/action/index/MappingUpdatedActionTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/action/index/MappingUpdatedActionTests.java @@ -143,7 +143,7 @@ public void testSendUpdateMappingUsingAutoPutMappingAction() { new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)); mua.setClient(client); - RootObjectMapper rootObjectMapper = new RootObjectMapper.Builder("name").build(MapperBuilderContext.ROOT); + RootObjectMapper rootObjectMapper = new RootObjectMapper.Builder("name", false).build(MapperBuilderContext.ROOT); Mapping update = new Mapping(rootObjectMapper, new MetadataFieldMapper[0], Map.of()); mua.sendUpdateMapping(new Index("name", "uuid"), update, ActionListener.wrap(() -> {})); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index 6089dfd478aa4..f756793acd9c1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -17,20 +17,19 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.CompositeFieldScript; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentType; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -585,7 +584,22 @@ DocumentMapper createDummyMapping() throws Exception { } MapperService createMapperService() throws Exception { - return createMapperService(mapping(b -> { + return createMapperService(topMapping(b -> { + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "runtime*"); + b.startObject("runtime").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + b.startObject("properties"); b.startObject("y").field("type", "object").endObject(); b.startObject("x"); { @@ -605,45 +619,37 @@ MapperService createMapperService() throws Exception { b.endObject(); } b.endObject(); + b.endObject(); })); } - // creates an object mapper, which is about 100x harder than it should be.... - private static ObjectMapper createObjectMapper(String name) { - ContentPath path = new ContentPath(0); - String[] nameParts = name.split("\\."); - for (int i = 0; i < nameParts.length - 1; ++i) { - path.add(nameParts[i]); - } - return new ObjectMapper.Builder(nameParts[nameParts.length - 1]).enabled(true).build(MapperBuilderContext.forPath(path)); - } - public void testEmptyMappingUpdate() throws Exception { DocumentMapper docMapper = createDummyMapping(); - assertNull(DocumentParser.createDynamicUpdate(docMapper.mappers(), Collections.emptyList(), Collections.emptyList())); + ParsedDocument doc = docMapper.parse(source(b -> {})); + assertNull(doc.dynamicMappingsUpdate()); } public void testSingleMappingUpdate() throws Exception { DocumentMapper docMapper = createDummyMapping(); - List updates = Collections.singletonList(new MockFieldMapper("foo")); - Mapping mapping = DocumentParser.createDynamicUpdate(docMapper.mappers(), updates, Collections.emptyList()); + ParsedDocument doc = docMapper.parse(source(b -> b.field("foo", 10))); + Mapping mapping = doc.dynamicMappingsUpdate(); assertNotNull(mapping); assertNotNull(mapping.getRoot().getMapper("foo")); } public void testSingleRuntimeFieldMappingUpdate() throws Exception { DocumentMapper docMapper = createDummyMapping(); - List updates = Collections.singletonList(new TestRuntimeField("foo", "any")); - Mapping mapping = DocumentParser.createDynamicUpdate(docMapper.mappers(), Collections.emptyList(), updates); + ParsedDocument doc = docMapper.parse(source(b -> b.field("runtime-field", "10"))); + Mapping mapping = doc.dynamicMappingsUpdate(); assertNotNull(mapping); - assertNull(mapping.getRoot().getMapper("foo")); - assertNotNull(mapping.getRoot().getRuntimeField("foo")); + assertNull(mapping.getRoot().getMapper("runtime-field")); + assertNotNull(mapping.getRoot().getRuntimeField("runtime-field")); } public void testSubfieldMappingUpdate() throws Exception { DocumentMapper docMapper = createDummyMapping(); - List updates = Collections.singletonList(new MockFieldMapper("x.foo")); - Mapping mapping = DocumentParser.createDynamicUpdate(docMapper.mappers(), updates, Collections.emptyList()); + ParsedDocument doc = docMapper.parse(source(b -> b.field("x.foo", 10))); + Mapping mapping = doc.dynamicMappingsUpdate(); assertNotNull(mapping); Mapper xMapper = mapping.getRoot().getMapper("x"); assertNotNull(xMapper); @@ -653,21 +659,22 @@ public void testSubfieldMappingUpdate() throws Exception { } public void testRuntimeSubfieldMappingUpdate() throws Exception { - DocumentMapper docMapper = createDummyMapping(); - List updates = Collections.singletonList(new TestRuntimeField("x.foo", "any")); - Mapping mapping = DocumentParser.createDynamicUpdate(docMapper.mappers(), Collections.emptyList(), updates); + DocumentMapper docMapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "runtime"))); + ParsedDocument doc = docMapper.parse(source(b -> b.field("runtime.foo", 10))); + Mapping mapping = doc.dynamicMappingsUpdate(); assertNotNull(mapping); - Mapper xMapper = mapping.getRoot().getMapper("x"); + Mapper xMapper = mapping.getRoot().getMapper("runtime"); assertNull(xMapper); - assertNotNull(mapping.getRoot().getRuntimeField("x.foo")); + assertNotNull(mapping.getRoot().getRuntimeField("runtime.foo")); } public void testMultipleSubfieldMappingUpdate() throws Exception { DocumentMapper docMapper = createDummyMapping(); - List updates = new ArrayList<>(); - updates.add(new MockFieldMapper("x.foo")); - updates.add(new MockFieldMapper("x.bar")); - Mapping mapping = DocumentParser.createDynamicUpdate(docMapper.mappers(), updates, Collections.emptyList()); + ParsedDocument doc = docMapper.parse(source(b -> { + b.field("x.foo", 10); + b.field("x.bar", 20); + })); + Mapping mapping = doc.dynamicMappingsUpdate(); assertNotNull(mapping); Mapper xMapper = mapping.getRoot().getMapper("x"); assertNotNull(xMapper); @@ -679,8 +686,8 @@ public void testMultipleSubfieldMappingUpdate() throws Exception { public void testDeepSubfieldMappingUpdate() throws Exception { DocumentMapper docMapper = createDummyMapping(); - List updates = Collections.singletonList(new MockFieldMapper("x.subx.foo")); - Mapping mapping = DocumentParser.createDynamicUpdate(docMapper.mappers(), updates, Collections.emptyList()); + ParsedDocument doc = docMapper.parse(source(b -> b.field("x.subx.foo", 10))); + Mapping mapping = doc.dynamicMappingsUpdate(); assertNotNull(mapping); Mapper xMapper = mapping.getRoot().getMapper("x"); assertNotNull(xMapper); @@ -693,10 +700,11 @@ public void testDeepSubfieldMappingUpdate() throws Exception { public void testDeepSubfieldAfterSubfieldMappingUpdate() throws Exception { DocumentMapper docMapper = createDummyMapping(); - List updates = new ArrayList<>(); - updates.add(new MockFieldMapper("x.a")); - updates.add(new MockFieldMapper("x.subx.b")); - Mapping mapping = DocumentParser.createDynamicUpdate(docMapper.mappers(), updates, Collections.emptyList()); + ParsedDocument doc = docMapper.parse(source(b -> { + b.field("x.a", 10); + b.field("x.subx.b", 10); + })); + Mapping mapping = doc.dynamicMappingsUpdate(); assertNotNull(mapping); Mapper xMapper = mapping.getRoot().getMapper("x"); assertNotNull(xMapper); @@ -710,12 +718,15 @@ public void testDeepSubfieldAfterSubfieldMappingUpdate() throws Exception { public void testObjectMappingUpdate() throws Exception { MapperService mapperService = createMapperService(); DocumentMapper docMapper = mapperService.documentMapper(); - List updates = new ArrayList<>(); - updates.add(createObjectMapper("foo")); - updates.add(createObjectMapper("foo.bar")); - updates.add(new MockFieldMapper("foo.bar.baz")); - updates.add(new MockFieldMapper("foo.field")); - Mapping mapping = DocumentParser.createDynamicUpdate(docMapper.mappers(), updates, Collections.emptyList()); + ParsedDocument doc = docMapper.parse(source(b -> { + b.startObject("foo"); + b.startObject("bar"); + b.field("baz", 10); + b.endObject(); + b.field("field", 10); + b.endObject(); + })); + Mapping mapping = doc.dynamicMappingsUpdate(); assertNotNull(mapping); Mapper fooMapper = mapping.getRoot().getMapper("foo"); assertNotNull(fooMapper); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java index 4f91a616d409a..c6291e0c5cf42 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java @@ -277,8 +277,8 @@ public void testIntroduceTwoFields() throws Exception { } public void testObject() throws Exception { - DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); - ParsedDocument doc = mapper.parse(source(b -> { + MapperService mapperService = createMapperService(mapping(b -> {})); + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> { b.startObject("foo"); { b.startObject("bar").field("baz", "foo").endObject(); @@ -287,7 +287,8 @@ public void testObject() throws Exception { })); assertNotNull(doc.dynamicMappingsUpdate()); - assertThat(Strings.toString(doc.dynamicMappingsUpdate()), + merge(mapperService, dynamicMapping(doc.dynamicMappingsUpdate())); + assertThat(Strings.toString(mapperService.documentMapper().mapping()), containsString("{\"foo\":{\"properties\":{\"bar\":{\"properties\":{\"baz\":{\"type\":\"text\"")); } @@ -586,4 +587,19 @@ public void testDynamicRuntimeDotsInFieldNames() throws IOException { Strings.toString(doc.dynamicMappingsUpdate()) ); } + + public void testDynamicConcreteDotsInFieldNames() throws IOException { + MapperService mapperService = createMapperService(topMapping(b -> b.field("flatten", true))); + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> { + b.field("instrument.name", "fred"); + b.field("temp", 20); + b.field("temp.min", 15); + b.field("temp.max", 25); + })); + assertEquals("{\"_doc\":{\"flatten\":true,\"properties\":{" + + "\"instrument.name\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}}," + + "\"temp\":{\"type\":\"long\"},\"temp.max\":{\"type\":\"long\"},\"temp.min\":{\"type\":\"long\"}" + + "}}}", + Strings.toString(doc.dynamicMappingsUpdate())); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java index f9e6924a26a4b..fc22eaa030061 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java @@ -954,4 +954,47 @@ public void testDynamicRuntimeWithDynamicTemplate() throws IOException { Strings.toString(parsedDoc2.dynamicMappingsUpdate()) ); } + + public void testFlattenedObjects() throws IOException { + MapperService mapperService = createMapperService(topMapping(b -> { + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match_mapping_type", "object"); + b.field("match", "metric"); + b.startObject("mapping").field("type", "object").field("flatten", true).endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> { + b.field("foo.bar", 10); + b.field("foo.metric.count", 10); + b.field("foo.metric.count.min", 4); + b.field("foo.metric.count.max", 15); + b.startObject("bar"); + b.startObject("baz").field("count", 4).endObject(); + b.startObject("bill"); + b.startObject("metric"); + b.field("count", 10); + b.field("count.min", 5); + b.field("count.max", 15); + b.endObject(); + b.endObject(); + b.endObject(); + })); + + merge(mapperService, dynamicMapping(doc.dynamicMappingsUpdate())); + + assertThat(mapperService.fieldType("foo.metric.count").typeName(), equalTo("long")); + assertThat(mapperService.fieldType("bar.bill.metric.count").typeName(), equalTo("long")); + assertNotNull(mapperService.mappingLookup().objectMappers().get("bar.bill.metric")); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java index e9e0f4ba1e52b..e6a9d0e4db94f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java @@ -185,6 +185,7 @@ private static FieldMapper createFieldMapper(String parent, String name) { private static ObjectMapper createObjectMapper(String name) { return new ObjectMapper(name, name, new Explicit<>(true, false), + false, ObjectMapper.Dynamic.FALSE, emptyMap()); } @@ -196,7 +197,7 @@ private static MappingLookup createMappingLookup(List fieldMappers, List objectMappers, List fieldAliasMappers, List runtimeFields) { - RootObjectMapper.Builder builder = new RootObjectMapper.Builder("_doc"); + RootObjectMapper.Builder builder = new RootObjectMapper.Builder("_doc", false); Map runtimeFieldTypes = runtimeFields.stream().collect(Collectors.toMap(RuntimeField::name, r -> r)); builder.setRuntime(runtimeFieldTypes); Mapping mapping = new Mapping( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java index 7bab424a7e0a3..c767bbad4992b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java @@ -35,7 +35,7 @@ public class MappingLookupTests extends ESTestCase { private static MappingLookup createMappingLookup(List fieldMappers, List objectMappers, List runtimeFields) { - RootObjectMapper.Builder builder = new RootObjectMapper.Builder("_doc"); + RootObjectMapper.Builder builder = new RootObjectMapper.Builder("_doc", false); Map runtimeFieldTypes = runtimeFields.stream().collect(Collectors.toMap(RuntimeField::name, r -> r)); builder.setRuntime(runtimeFieldTypes); Mapping mapping = new Mapping( @@ -67,8 +67,14 @@ public void testRuntimeFieldLeafOverride() { public void testSubfieldOverride() { MockFieldMapper fieldMapper = new MockFieldMapper("object.subfield"); - ObjectMapper objectMapper = new ObjectMapper("object", "object", new Explicit<>(true, true), - ObjectMapper.Dynamic.TRUE, Collections.singletonMap("object.subfield", fieldMapper)); + ObjectMapper objectMapper = new ObjectMapper( + "object", + "object", + new Explicit<>(true, true), + false, + ObjectMapper.Dynamic.TRUE, + Collections.singletonMap("object.subfield", fieldMapper) + ); MappingLookup mappingLookup = createMappingLookup( Collections.singletonList(fieldMapper), Collections.singletonList(objectMapper), diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java index 209ee41fd07f1..94177267f8b6e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; @@ -48,7 +49,7 @@ private static MappingParser createMappingParser(Settings settings) { () -> metadataMappers, type -> MapperService.SINGLE_MAPPING_NAME); } - public void testFieldNameWithDots() throws Exception { + public void testFieldNameWithDotsDisallowed() throws Exception { XContentBuilder builder = mapping(b -> { b.startObject("foo.bar").field("type", "text").endObject(); b.startObject("foo.baz").field("type", "keyword").endObject(); @@ -62,6 +63,24 @@ public void testFieldNameWithDots() throws Exception { assertNotNull(objectMapper.getMapper("baz")); } + public void testFieldNameWithDotsAllowed() throws Exception { + DocumentMapper docMapper = createDocumentMapper(topMapping(b -> { + b.field("flatten", true); + b.startObject("properties"); + b.startObject("foo.bar").field("type", "text").endObject(); + b.startObject("foo.baz").field("type", "keyword").endObject(); + b.endObject(); + })); + ParsedDocument doc = docMapper.parse(source(b -> { + b.field("foo.bar", "first"); + b.field("foo.baz", "second"); + })); + + assertNull(doc.dynamicMappingsUpdate()); + assertNotNull(doc.rootDoc().getField("foo.bar")); + assertEquals(new BytesRef("second"), doc.rootDoc().getField("foo.baz").binaryValue()); + } + public void testFieldNameWithDeepDots() throws Exception { XContentBuilder builder = mapping(b -> { b.startObject("foo.bar").field("type", "text").endObject(); @@ -82,7 +101,7 @@ public void testFieldNameWithDeepDots() throws Exception { assertNotNull(mappingLookup.objectMappers().get("foo")); } - public void testFieldNameWithDotsConflict() throws IOException { + public void testFieldNameWithDotPrefixDisallowed() throws IOException { XContentBuilder builder = mapping(b -> { b.startObject("foo").field("type", "text").endObject(); b.startObject("foo.baz").field("type", "keyword").endObject(); @@ -92,6 +111,19 @@ public void testFieldNameWithDotsConflict() throws IOException { assertTrue(e.getMessage(), e.getMessage().contains("mapper [foo] cannot be changed from type [text] to [ObjectMapper]")); } + public void testFieldNameWithDotPrefixAllowed() throws IOException { + XContentBuilder builder = topMapping(b -> { + b.field("flatten", true); + b.startObject("properties"); + b.startObject("foo").field("type", "text").endObject(); + b.startObject("foo.baz").field("type", "keyword").endObject(); + b.endObject(); + }); + MapperService mapperService = createMapperService(builder); + assertEquals("text", mapperService.fieldType("foo").typeName()); + assertEquals("keyword", mapperService.fieldType("foo.baz").typeName()); + } + public void testMultiFieldsWithFieldAlias() throws IOException { XContentBuilder builder = mapping(b -> { b.startObject("field"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java index e026b0cec1661..0062994b7a9b0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java @@ -67,7 +67,7 @@ public void testMergeDisabledField() { // GIVEN a mapping with "foo" field disabled Map mappers = new HashMap<>(); //the field is disabled, and we are not trying to re-enable it, hence merge should work - mappers.put("disabled", new ObjectMapper.Builder("disabled").build(MapperBuilderContext.ROOT)); + mappers.put("disabled", new ObjectMapper.Builder("disabled", false).build(MapperBuilderContext.ROOT)); RootObjectMapper mergeWith = createRootObjectMapper("type1", true, Collections.unmodifiableMap(mappers)); RootObjectMapper merged = (RootObjectMapper)rootObjectMapper.merge(mergeWith); @@ -99,10 +99,10 @@ public void testMergeEnabledForRootMapper() { public void testMergeDisabledRootMapper() { String type = MapperService.SINGLE_MAPPING_NAME; final RootObjectMapper rootObjectMapper = - (RootObjectMapper) new RootObjectMapper.Builder(type).enabled(false).build(MapperBuilderContext.ROOT); + (RootObjectMapper) new RootObjectMapper.Builder(type, false).enabled(false).build(MapperBuilderContext.ROOT); //the root is disabled, and we are not trying to re-enable it, but we do want to be able to add runtime fields final RootObjectMapper mergeWith = - new RootObjectMapper.Builder(type) + new RootObjectMapper.Builder(type, false) .setRuntime(Collections.singletonMap("test", new TestRuntimeField("test", "long"))) .build(MapperBuilderContext.ROOT); @@ -133,7 +133,7 @@ public void testMergeNested() { private static RootObjectMapper createRootObjectMapper(String name, boolean enabled, Map mappers) { final RootObjectMapper rootObjectMapper - = (RootObjectMapper) new RootObjectMapper.Builder(name).enabled(enabled).build(MapperBuilderContext.ROOT); + = (RootObjectMapper) new RootObjectMapper.Builder(name, false).enabled(enabled).build(MapperBuilderContext.ROOT); mappers.values().forEach(rootObjectMapper::putMapper); @@ -141,7 +141,7 @@ private static RootObjectMapper createRootObjectMapper(String name, boolean enab } private static ObjectMapper createObjectMapper(String name, boolean enabled, Map mappers) { - final ObjectMapper mapper = new ObjectMapper.Builder(name).enabled(enabled).build(MapperBuilderContext.ROOT); + final ObjectMapper mapper = new ObjectMapper.Builder(name, false).enabled(enabled).build(MapperBuilderContext.ROOT); mappers.values().forEach(mapper::putMapper); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java index 2d53b5566974e..7536b4471bd5a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java @@ -40,7 +40,7 @@ public void testDifferentInnerObjectTokenFailure() throws Exception { " \"value\":\"value\"\n" + " }"), XContentType.JSON))); - assertTrue(e.getMessage(), e.getMessage().contains("cannot be changed from type")); + assertThat(e.getMessage(), containsString("can't merge a non object mapping [object.array.object] with an object mapping")); } public void testEmptyArrayProperties() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java index 9d920c4aebe69..86ca6fbfb3413 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java @@ -17,6 +17,7 @@ import java.util.Arrays; import java.util.Collections; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; public class RootObjectMapperTests extends MapperServiceTestCase { @@ -136,6 +137,15 @@ public void testIllegalFormatField() throws Exception { } } + public void testAllowDotsInLeafFields() throws Exception { + MapperService mapperService = createMapperService(topMapping(b -> b.field("flatten", true))); + Exception e = expectThrows( + MapperParsingException.class, + () -> merge(mapperService, topMapping(b -> b.field("flatten", false))) + ); + assertThat(e.getMessage(), containsString("Can't change parameter [flatten] from [true] to [false]")); + } + public void testRuntimeSection() throws IOException { String mapping = Strings.toString(runtimeMapping(builder -> { builder.startObject("field1").field("type", "double").endObject(); diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 3f2e6e3fbddd4..ef075bcfc3240 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -325,7 +325,7 @@ public void testFielddataLookupOneFieldManyReferences() throws IOException { private static MappingLookup createMappingLookup(List concreteFields, List runtimeFields) { List mappers = concreteFields.stream().map(MockFieldMapper::new).collect(Collectors.toList()); - RootObjectMapper.Builder builder = new RootObjectMapper.Builder("_doc"); + RootObjectMapper.Builder builder = new RootObjectMapper.Builder("_doc", false); Map runtimeFieldTypes = runtimeFields.stream().collect(Collectors.toMap(RuntimeField::name, r -> r)); builder.setRuntime(runtimeFieldTypes); Mapping mapping = new Mapping(