diff --git a/dev-tools/create-bwc-index.py b/dev-tools/create-bwc-index.py index 6cdf9b36fa305..1dd295fe1dbc4 100644 --- a/dev-tools/create-bwc-index.py +++ b/dev-tools/create-bwc-index.py @@ -226,6 +226,11 @@ def generate_index(client, version, index_name): } } } + mappings['auto_boost'] = { + '_all': { + 'auto_boost': True + } + } client.indices.create(index=index_name, body={ 'settings': { diff --git a/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java b/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java index 70e2b5039b6f6..92e4048e962ac 100644 --- a/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java +++ b/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java @@ -19,11 +19,14 @@ package org.elasticsearch.common.lucene.all; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.payloads.AveragePayloadFunction; import org.apache.lucene.search.payloads.PayloadTermQuery; import org.apache.lucene.search.similarities.Similarity; @@ -119,4 +122,22 @@ public boolean equals(Object obj) { return true; } + @Override + public Query rewrite(IndexReader reader) throws IOException { + boolean hasPayloads = false; + for (LeafReaderContext context : reader.leaves()) { + final Terms terms = context.reader().terms(term.field()); + if (terms.hasPayloads()) { + hasPayloads = true; + break; + } + } + if (hasPayloads == false) { + TermQuery rewritten = new TermQuery(term); + rewritten.setBoost(getBoost()); + return rewritten; + } + return this; + } + } diff --git a/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index f1693fec30392..112e68e49bd04 100644 --- a/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.collect.Sets; + import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; @@ -33,6 +34,7 @@ import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Preconditions; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.compress.CompressedString; @@ -70,6 +72,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -438,10 +441,11 @@ public ParsedDocument parse(SourceToParse source, @Nullable ParseListener listen ParseContext.InternalParseContext context = cache.get(); if (source.type() != null && !source.type().equals(this.type)) { - throw new MapperParsingException("Type mismatch, provide type [" + source.type() + "] but mapper is of type [" + this.type + "]", context.mappingsModified()); + throw new MapperParsingException("Type mismatch, provide type [" + source.type() + "] but mapper is of type [" + this.type + "]"); } source.type(this.type); + boolean mappingsModified = false; XContentParser parser = source.parser(); try { if (parser == null) { @@ -456,7 +460,7 @@ public ParsedDocument parse(SourceToParse source, @Nullable ParseListener listen int countDownTokens = 0; XContentParser.Token token = parser.nextToken(); if (token != XContentParser.Token.START_OBJECT) { - throw new MapperParsingException("Malformed content, must start with an object", context.mappingsModified()); + throw new MapperParsingException("Malformed content, must start with an object"); } boolean emptyDoc = false; token = parser.nextToken(); @@ -464,7 +468,7 @@ public ParsedDocument parse(SourceToParse source, @Nullable ParseListener listen // empty doc, we can handle it... emptyDoc = true; } else if (token != XContentParser.Token.FIELD_NAME) { - throw new MapperParsingException("Malformed content, after first object, either the type field or the actual properties should exist", context.mappingsModified()); + throw new MapperParsingException("Malformed content, after first object, either the type field or the actual properties should exist"); } for (RootMapper rootMapper : rootMappersOrdered) { @@ -472,7 +476,31 @@ public ParsedDocument parse(SourceToParse source, @Nullable ParseListener listen } if (!emptyDoc) { - rootObjectMapper.parse(context); + Mapper update = rootObjectMapper.parse(context); + for (RootObjectMapper mapper : context.updates()) { + if (update == null) { + update = mapper; + } else { + MapperUtils.merge(update, mapper); + } + } + if (update != null) { + // TODO: validate the mapping update on the master node + // lock to avoid concurrency issues with mapping updates coming from the API + synchronized(this) { + // simulate on the first time to check if the mapping update is applicable + MergeContext mergeContext = newMmergeContext(new MergeFlags().simulate(true)); + rootObjectMapper.merge(update, mergeContext); + if (mergeContext.hasConflicts()) { + throw new MapperParsingException("Could not apply generated dynamic mappings: " + Arrays.toString(mergeContext.buildConflicts())); + } else { + // then apply it for real + mappingsModified = true; + mergeContext = newMmergeContext(new MergeFlags().simulate(false)); + rootObjectMapper.merge(update, mergeContext); + } + } + } } for (int i = 0; i < countDownTokens; i++) { @@ -490,10 +518,10 @@ public ParsedDocument parse(SourceToParse source, @Nullable ParseListener listen // Throw a more meaningful message if the document is empty. if (source.source() != null && source.source().length() == 0) { - throw new MapperParsingException("failed to parse, document is empty", context.mappingsModified()); + throw new MapperParsingException("failed to parse, document is empty"); } - throw new MapperParsingException("failed to parse", e, context.mappingsModified()); + throw new MapperParsingException("failed to parse", e); } finally { // only close the parser when its not provided externally if (source.parser() == null && parser != null) { @@ -521,7 +549,7 @@ public ParsedDocument parse(SourceToParse source, @Nullable ParseListener listen } ParsedDocument doc = new ParsedDocument(context.uid(), context.version(), context.id(), context.type(), source.routing(), source.timestamp(), source.ttl(), context.docs(), - context.source(), context.mappingsModified()).parent(source.parent()); + context.source(), mappingsModified).parent(source.parent()); // reset the context to free up memory context.reset(null, null, null, null); return doc; @@ -637,8 +665,41 @@ public void traverse(ObjectMapperListener listener) { rootObjectMapper.traverse(listener); } + private MergeContext newMmergeContext(MergeFlags mergeFlags) { + return new MergeContext(mergeFlags) { + + List conflicts = new ArrayList<>(); + + @Override + public void addFieldMappers(List> fieldMappers) { + DocumentMapper.this.addFieldMappers(fieldMappers); + } + + @Override + public void addObjectMappers(Collection objectMappers) { + DocumentMapper.this.addObjectMappers(objectMappers); + } + + @Override + public void addConflict(String mergeFailure) { + conflicts.add(mergeFailure); + } + + @Override + public boolean hasConflicts() { + return conflicts.isEmpty() == false; + } + + @Override + public String[] buildConflicts() { + return conflicts.toArray(Strings.EMPTY_ARRAY); + } + + }; + } + public synchronized MergeResult merge(DocumentMapper mergeWith, MergeFlags mergeFlags) { - MergeContext mergeContext = new MergeContext(this, mergeFlags); + final MergeContext mergeContext = newMmergeContext(mergeFlags); assert rootMappers.size() == mergeWith.rootMappers.size(); rootObjectMapper.merge(mergeWith.rootObjectMapper, mergeContext); diff --git a/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 151940b6c30e3..ae2f6acbf2d5a 100644 --- a/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -125,7 +125,12 @@ public Version indexVersionCreated() { String name(); - void parse(ParseContext context) throws IOException; + /** + * Parse using the provided {@link ParseContext} and return a mapping + * update if dynamic mappings modified the mappings, or {@code null} if + * mappings were not modified. + */ + Mapper parse(ParseContext context) throws IOException; void merge(Mapper mergeWith, MergeContext mergeContext) throws MergeMappingException; diff --git a/src/main/java/org/elasticsearch/index/mapper/MapperParsingException.java b/src/main/java/org/elasticsearch/index/mapper/MapperParsingException.java index 9cac8a26ef985..25a7e1f92333f 100644 --- a/src/main/java/org/elasticsearch/index/mapper/MapperParsingException.java +++ b/src/main/java/org/elasticsearch/index/mapper/MapperParsingException.java @@ -28,28 +28,10 @@ public class MapperParsingException extends MapperException { public MapperParsingException(String message) { super(message); - mappingsModified = false; - } - - public boolean isMappingsModified() { - return mappingsModified; - } - - private boolean mappingsModified = false; - - public MapperParsingException(String message, boolean mappingsModified) { - super(message); - this.mappingsModified = mappingsModified; - } - - public MapperParsingException(String message, Throwable cause, boolean mappingsModified) { - super(message, cause); - this.mappingsModified = mappingsModified; } public MapperParsingException(String message, Throwable cause) { super(message, cause); - this.mappingsModified = false; } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/MapperUtils.java b/src/main/java/org/elasticsearch/index/mapper/MapperUtils.java new file mode 100644 index 0000000000000..be4915b8392c7 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/mapper/MapperUtils.java @@ -0,0 +1,82 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.ElasticsearchIllegalStateException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.index.mapper.object.ObjectMapper; +import org.elasticsearch.index.mapper.object.RootObjectMapper; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +public enum MapperUtils { + ; + + /** + * Parse the given {@code context} with the given {@code mapper} and apply + * the potential mapping update in-place. This method is useful when + * composing mapping updates. + */ + public static M parseAndMergeUpdate(M mapper, ParseContext context) throws IOException { + final Mapper update = mapper.parse(context); + if (update != null) { + merge(mapper, update); + } + return mapper; + } + + /** + * Merge {@code mergeWith} into {@code mergeTo}. Note: this method only + * merges mappings, not lookup structures. Conflicts are returned as exceptions. + */ + public static void merge(Mapper mergeInto, Mapper mergeWith) { + MergeContext ctx = new MergeContext(new DocumentMapper.MergeFlags().simulate(false)) { + + @Override + public boolean hasConflicts() { + return false; + } + + @Override + public String[] buildConflicts() { + return Strings.EMPTY_ARRAY; + } + + @Override + public void addObjectMappers(Collection objectMappers) { + // no-op + } + + @Override + public void addFieldMappers(List> fieldMappers) { + // no-op + } + + @Override + public void addConflict(String mergeFailure) { + throw new ElasticsearchIllegalStateException("Merging dynamic updates triggered a conflict: " + mergeFailure); + } + }; + mergeInto.merge(mergeWith, ctx); + } + +} diff --git a/src/main/java/org/elasticsearch/index/mapper/MergeContext.java b/src/main/java/org/elasticsearch/index/mapper/MergeContext.java index 4c250c242f1fe..f8ddb8375175e 100644 --- a/src/main/java/org/elasticsearch/index/mapper/MergeContext.java +++ b/src/main/java/org/elasticsearch/index/mapper/MergeContext.java @@ -19,41 +19,33 @@ package org.elasticsearch.index.mapper; -import com.google.common.collect.Lists; +import org.elasticsearch.index.mapper.object.ObjectMapper; +import java.util.Collection; import java.util.List; /** * */ -public class MergeContext { +public abstract class MergeContext { - private final DocumentMapper documentMapper; private final DocumentMapper.MergeFlags mergeFlags; - private final List mergeConflicts = Lists.newArrayList(); - public MergeContext(DocumentMapper documentMapper, DocumentMapper.MergeFlags mergeFlags) { - this.documentMapper = documentMapper; + public MergeContext(DocumentMapper.MergeFlags mergeFlags) { this.mergeFlags = mergeFlags; } - public DocumentMapper docMapper() { - return documentMapper; - } + public abstract void addFieldMappers(List> fieldMappers); + + public abstract void addObjectMappers(Collection objectMappers); public DocumentMapper.MergeFlags mergeFlags() { return mergeFlags; } - public void addConflict(String mergeFailure) { - mergeConflicts.add(mergeFailure); - } + public abstract void addConflict(String mergeFailure); - public boolean hasConflicts() { - return !mergeConflicts.isEmpty(); - } + public abstract boolean hasConflicts(); - public String[] buildConflicts() { - return mergeConflicts.toArray(new String[mergeConflicts.size()]); - } + public abstract String[] buildConflicts(); } diff --git a/src/main/java/org/elasticsearch/index/mapper/ParseContext.java b/src/main/java/org/elasticsearch/index/mapper/ParseContext.java index 9f50a7df045ce..7cf3d97938b26 100644 --- a/src/main/java/org/elasticsearch/index/mapper/ParseContext.java +++ b/src/main/java/org/elasticsearch/index/mapper/ParseContext.java @@ -22,7 +22,7 @@ import com.carrotsearch.hppc.ObjectObjectMap; import com.carrotsearch.hppc.ObjectObjectOpenHashMap; import com.google.common.collect.Lists; -import org.apache.lucene.analysis.Analyzer; + import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; @@ -38,7 +38,11 @@ import org.elasticsearch.index.mapper.DocumentMapper.ParseListener; import org.elasticsearch.index.mapper.object.RootObjectMapper; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; /** * @@ -194,31 +198,6 @@ public DocumentMapperParser docMapperParser() { return in.docMapperParser(); } - @Override - public boolean mappingsModified() { - return in.mappingsModified(); - } - - @Override - public void setMappingsModified() { - in.setMappingsModified(); - } - - @Override - public void setWithinNewMapper() { - in.setWithinNewMapper(); - } - - @Override - public void clearWithinNewMapper() { - in.clearWithinNewMapper(); - } - - @Override - public boolean isWithinNewMapper() { - return in.isWithinNewMapper(); - } - @Override public boolean isWithinCopyTo() { return in.isWithinCopyTo(); @@ -379,6 +358,15 @@ public StringBuilder stringBuilder() { return in.stringBuilder(); } + @Override + public void addRootObjectUpdate(RootObjectMapper update) { + in.addRootObjectUpdate(update); + } + + @Override + public List updates() { + return in.updates(); + } } public static class InternalParseContext extends ParseContext { @@ -414,12 +402,13 @@ public static class InternalParseContext extends ParseContext { private Map ignoredValues = new HashMap<>(); private boolean mappingsModified = false; - private boolean withinNewMapper = false; private AllEntries allEntries = new AllEntries(); private float docBoost = 1.0f; + private final List rootMapperDynamicUpdates = new ArrayList<>(); + public InternalParseContext(String index, @Nullable Settings indexSettings, DocumentMapperParser docMapperParser, DocumentMapper docMapper, ContentPath path) { this.index = index; this.indexSettings = indexSettings; @@ -444,11 +433,11 @@ public void reset(XContentParser parser, Document document, SourceToParse source this.source = source == null ? null : sourceToParse.source(); this.path.reset(); this.mappingsModified = false; - this.withinNewMapper = false; this.listener = listener == null ? DocumentMapper.ParseListener.EMPTY : listener; this.allEntries = new AllEntries(); this.ignoredValues.clear(); this.docBoost = 1.0f; + this.rootMapperDynamicUpdates.clear(); } @Override @@ -461,31 +450,6 @@ public DocumentMapperParser docMapperParser() { return this.docMapperParser; } - @Override - public boolean mappingsModified() { - return this.mappingsModified; - } - - @Override - public void setMappingsModified() { - this.mappingsModified = true; - } - - @Override - public void setWithinNewMapper() { - this.withinNewMapper = true; - } - - @Override - public void clearWithinNewMapper() { - this.withinNewMapper = false; - } - - @Override - public boolean isWithinNewMapper() { - return withinNewMapper; - } - @Override public String index() { return this.index; @@ -638,22 +602,22 @@ public StringBuilder stringBuilder() { stringBuilder.setLength(0); return this.stringBuilder; } + + @Override + public void addRootObjectUpdate(RootObjectMapper mapper) { + rootMapperDynamicUpdates.add(mapper); + } + + @Override + public List updates() { + return rootMapperDynamicUpdates; + } } public abstract boolean flyweight(); public abstract DocumentMapperParser docMapperParser(); - public abstract boolean mappingsModified(); - - public abstract void setMappingsModified(); - - public abstract void setWithinNewMapper(); - - public abstract void clearWithinNewMapper(); - - public abstract boolean isWithinNewMapper(); - /** * Return a new context that will be within a copy-to operation. */ @@ -854,4 +818,15 @@ public final T parseExternalValue(Class clazz) { */ public abstract StringBuilder stringBuilder(); + /** + * Add a dynamic update to the root object mapper. + * TODO: can we nuke it, it is only needed for copy_to + */ + public abstract void addRootObjectUpdate(RootObjectMapper update); + + /** + * Get dynamic updates to the root object mapper. + * TODO: can we nuke it, it is only needed for copy_to + */ + public abstract List updates(); } diff --git a/src/main/java/org/elasticsearch/index/mapper/StrictDynamicMappingException.java b/src/main/java/org/elasticsearch/index/mapper/StrictDynamicMappingException.java index 098d5abf08621..f675396369b5a 100644 --- a/src/main/java/org/elasticsearch/index/mapper/StrictDynamicMappingException.java +++ b/src/main/java/org/elasticsearch/index/mapper/StrictDynamicMappingException.java @@ -24,8 +24,8 @@ */ public class StrictDynamicMappingException extends MapperParsingException { - public StrictDynamicMappingException(String path, String fieldName, boolean mappingsModified) { - super("mapping set to strict, dynamic introduction of [" + fieldName + "] within [" + path + "] is not allowed", mappingsModified); + public StrictDynamicMappingException(String path, String fieldName) { + super("mapping set to strict, dynamic introduction of [" + fieldName + "] within [" + path + "] is not allowed"); } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java index b68be28e2f131..2e7328c290748 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java @@ -44,6 +44,7 @@ import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchIllegalArgumentException; +import org.elasticsearch.ElasticsearchIllegalStateException; import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -70,6 +71,7 @@ import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.mapper.internal.AllFieldMapper; import org.elasticsearch.index.mapper.object.ObjectMapper; +import org.elasticsearch.index.mapper.object.RootObjectMapper; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.index.search.FieldDataTermsFilter; import org.elasticsearch.index.similarity.SimilarityLookupService; @@ -81,7 +83,6 @@ import java.util.Comparator; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.TreeMap; /** @@ -434,7 +435,7 @@ public CopyTo copyTo() { } @Override - public void parse(ParseContext context) throws IOException { + public Mapper parse(ParseContext context) throws IOException { final List fields = new ArrayList<>(2); try { parseCreateField(context, fields); @@ -447,12 +448,13 @@ public void parse(ParseContext context) throws IOException { } } } catch (Exception e) { - throw new MapperParsingException("failed to parse [" + names.fullName() + "]", e, context.mappingsModified()); + throw new MapperParsingException("failed to parse [" + names.fullName() + "]", e); } multiFields.parse(this, context); if (copyTo != null) { copyTo.parse(context); } + return null; } /** @@ -968,7 +970,7 @@ public void merge(Mapper mergeWith, MergeContext mergeContext) throws MergeMappi // first add all field mappers if (newFieldMappers != null) { - mergeContext.docMapper().addFieldMappers(newFieldMappers); + mergeContext.addFieldMappers(newFieldMappers); } // now publish mappers if (newMappersBuilder != null) { @@ -1089,54 +1091,41 @@ public void parse(String field, ParseContext context) throws IOException { // The path of the dest field might be completely different from the current one so we need to reset it context = context.overridePath(new ContentPath(0)); + ObjectMapper mapper = context.root(); + String objectPath = ""; + String fieldPath = field; int posDot = field.lastIndexOf('.'); if (posDot > 0) { - // Compound name - String objectPath = field.substring(0, posDot); - String fieldPath = field.substring(posDot + 1); - ObjectMapper mapper = context.docMapper().objectMappers().get(objectPath); - if (mapper == null) { - //TODO: Create an object dynamically? - throw new MapperParsingException("attempt to copy value to non-existing object [" + field + "]", context.mappingsModified()); - } - + objectPath = field.substring(0, posDot); context.path().add(objectPath); - - // We might be in dynamically created field already, so need to clean withinNewMapper flag - // and then restore it, so we wouldn't miss new mappers created from copy_to fields - boolean origWithinNewMapper = context.isWithinNewMapper(); - context.clearWithinNewMapper(); - - try { - mapper.parseDynamicValue(context, fieldPath, context.parser().currentToken()); - } finally { - if (origWithinNewMapper) { - context.setWithinNewMapper(); - } else { - context.clearWithinNewMapper(); - } + mapper = context.docMapper().objectMappers().get(objectPath); + fieldPath = field.substring(posDot + 1); + } + if (mapper == null) { + //TODO: Create an object dynamically? + throw new MapperParsingException("attempt to copy value to non-existing object [" + field + "]"); + } + ObjectMapper update = mapper.parseDynamicValue(context, fieldPath, context.parser().currentToken()); + assert update != null; // we are parsing a dynamic value so we necessarily created a new mapping + + // propagate the update to the root + while (objectPath.length() > 0) { + String parentPath = ""; + ObjectMapper parent = context.root(); + posDot = objectPath.lastIndexOf('.'); + if (posDot > 0) { + parentPath = objectPath.substring(0, posDot); + parent = context.docMapper().objectMappers().get(parentPath); } - - } else { - // We might be in dynamically created field already, so need to clean withinNewMapper flag - // and then restore it, so we wouldn't miss new mappers created from copy_to fields - boolean origWithinNewMapper = context.isWithinNewMapper(); - context.clearWithinNewMapper(); - try { - context.docMapper().root().parseDynamicValue(context, field, context.parser().currentToken()); - } finally { - if (origWithinNewMapper) { - context.setWithinNewMapper(); - } else { - context.clearWithinNewMapper(); - } + if (parent == null) { + throw new ElasticsearchIllegalStateException("[" + objectPath + "] has no parent for path [" + parentPath + "]"); } - + update = parent.mappingUpdate(update); + objectPath = parentPath; } + context.addRootObjectUpdate((RootObjectMapper) update); } } - - } /** diff --git a/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java index 03792a11bd3b8..7e037bd533c40 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java @@ -266,7 +266,7 @@ public synchronized PostingsFormat postingsFormat(PostingsFormat in) { } @Override - public void parse(ParseContext context) throws IOException { + public Mapper parse(ParseContext context) throws IOException { XContentParser parser = context.parser(); XContentParser.Token token = parser.currentToken(); @@ -382,6 +382,7 @@ public void parse(ParseContext context) throws IOException { context.doc().add(getCompletionField(ctx, input, suggestPayload)); } } + return null; } private void checkWeight(long weight) { diff --git a/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java index 8d2460bffddeb..f7a39b2c95295 100644 --- a/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java @@ -515,7 +515,7 @@ protected void parseCreateField(ParseContext context, List fields) throws } @Override - public void parse(ParseContext context) throws IOException { + public Mapper parse(ParseContext context) throws IOException { ContentPath.Type origPathType = context.path().pathType(); context.path().pathType(pathType); context.path().add(name()); @@ -565,6 +565,7 @@ public void parse(ParseContext context) throws IOException { context.path().remove(); context.path().pathType(origPathType); + return null; } private void parseGeohashField(ParseContext context, String geohash) throws IOException { diff --git a/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java index a96a84550f73d..896185f39f6fa 100644 --- a/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java @@ -237,19 +237,19 @@ public FieldDataType defaultFieldDataType() { } @Override - public void parse(ParseContext context) throws IOException { + public Mapper parse(ParseContext context) throws IOException { try { Shape shape = context.parseExternalValue(Shape.class); if (shape == null) { ShapeBuilder shapeBuilder = ShapeBuilder.parse(context.parser(), this); if (shapeBuilder == null) { - return; + return null; } shape = shapeBuilder.build(); } Field[] fields = defaultStrategy.createIndexableFields(shape); if (fields == null || fields.length == 0) { - return; + return null; } for (Field field : fields) { if (!customBoost()) { @@ -262,6 +262,7 @@ public void parse(ParseContext context) throws IOException { } catch (Exception e) { throw new MapperParsingException("failed to parse [" + names.fullName() + "]", e); } + return null; } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java index 98e64ccddaebb..7fd2d5859ea7a 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java @@ -25,7 +25,6 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; @@ -33,9 +32,7 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.all.AllField; import org.elasticsearch.common.lucene.all.AllTermQuery; -import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.loader.SettingsLoader; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldDataType; @@ -97,9 +94,6 @@ public static class Builder extends AbstractFieldMapper.Builder node, ParserContext if (fieldName.equals("enabled")) { builder.enabled(nodeBooleanValue(fieldNode) ? EnabledAttributeMapper.ENABLED : EnabledAttributeMapper.DISABLED); iterator.remove(); - } else if (fieldName.equals("auto_boost")) { - builder.autoBoost = nodeBooleanValue(fieldNode); + } else if (fieldName.equals("auto_boost") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) { + // Old 1.x setting which is now ignored iterator.remove(); } } @@ -165,24 +159,17 @@ public Mapper.Builder parse(String name, Map node, ParserContext private EnabledAttributeMapper enabledState; - // The autoBoost flag is automatically set based on indexed docs on the mappings - // if a doc is indexed with a specific boost value and part of _all, it is automatically - // set to true. This allows to optimize (automatically, which we like) for the common case - // where fields don't usually have boost associated with them, and we don't need to use the - // special SpanTermQuery to look at payloads - private volatile boolean autoBoost; public AllFieldMapper(Settings indexSettings) { - this(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE), null, null, Defaults.ENABLED, false, null, null, null, indexSettings); + this(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE), null, null, Defaults.ENABLED, null, null, null, indexSettings); } protected AllFieldMapper(String name, FieldType fieldType, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer, - EnabledAttributeMapper enabled, boolean autoBoost, SimilarityProvider similarity, Loading normsLoading, + EnabledAttributeMapper enabled, SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings) { super(new Names(name, name, name, name), 1.0f, fieldType, false, indexAnalyzer, searchAnalyzer, similarity, normsLoading, fieldDataSettings, indexSettings); this.enabledState = enabled; - this.autoBoost = autoBoost; } @@ -202,13 +189,7 @@ public FieldDataType defaultFieldDataType() { @Override public Query queryStringTermQuery(Term term) { - if (!autoBoost) { - return new TermQuery(term); - } - if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) { - return new AllTermQuery(term); - } - return new TermQuery(term); + return new AllTermQuery(term); } @Override @@ -226,8 +207,9 @@ public void postParse(ParseContext context) throws IOException { } @Override - public void parse(ParseContext context) throws IOException { + public Mapper parse(ParseContext context) throws IOException { // we parse in post parse + return null; } @Override @@ -242,14 +224,6 @@ protected void parseCreateField(ParseContext context, List fields) throws } // reset the entries context.allEntries().reset(); - - // if the autoBoost flag is not set, and we indexed a doc with custom boost, make - // sure to update the flag, and notify mappings on change - if (!autoBoost && context.allEntries().customBoost()) { - autoBoost = true; - context.setMappingsModified(); - } - Analyzer analyzer = findAnalyzer(context); fields.add(new AllField(names.indexName(), context.allEntries(), analyzer, fieldType)); } @@ -305,9 +279,6 @@ private void innerToXContent(XContentBuilder builder, boolean includeDefaults) t if (includeDefaults || enabledState != Defaults.ENABLED) { builder.field("enabled", enabledState.enabled); } - if (includeDefaults || autoBoost != false) { - builder.field("auto_boost", autoBoost); - } if (includeDefaults || fieldType.stored() != Defaults.FIELD_TYPE.stored()) { builder.field("store", fieldType.stored()); } diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java index b38c2186d66aa..33633d3c06d09 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java @@ -184,8 +184,9 @@ public void postParse(ParseContext context) throws IOException { } @Override - public void parse(ParseContext context) throws IOException { + public Mapper parse(ParseContext context) throws IOException { // we parse in post parse + return null; } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java index 816bb4557c716..6556aa20bff7b 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java @@ -307,16 +307,11 @@ public void preParse(ParseContext context) throws IOException { @Override public void postParse(ParseContext context) throws IOException { if (context.id() == null && !context.sourceToParse().flyweight()) { - throw new MapperParsingException("No id found while parsing the content source", context.mappingsModified()); + throw new MapperParsingException("No id found while parsing the content source"); } // it either get built in the preParse phase, or get parsed... } - @Override - public void parse(ParseContext context) throws IOException { - super.parse(context); - } - @Override public boolean includeInObject() { return true; @@ -329,7 +324,7 @@ protected void parseCreateField(ParseContext context, List fields) throws // we are in the parse Phase String id = parser.text(); if (context.id() != null && !context.id().equals(id)) { - throw new MapperParsingException("Provided id [" + context.id() + "] does not match the content one [" + id + "]", context.mappingsModified()); + throw new MapperParsingException("Provided id [" + context.id() + "] does not match the content one [" + id + "]"); } context.id(id); } // else we are in the pre/post parse phase diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java index 7bbf155276d68..a530102e67372 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java @@ -166,8 +166,8 @@ public void postParse(ParseContext context) throws IOException { } @Override - public void parse(ParseContext context) throws IOException { - + public Mapper parse(ParseContext context) throws IOException { + return null; } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java index 1d883196189cb..8aee69f8ba384 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java @@ -182,10 +182,11 @@ public void postParse(ParseContext context) throws IOException { } @Override - public void parse(ParseContext context) throws IOException { + public Mapper parse(ParseContext context) throws IOException { // no need ot parse here, we either get the routing in the sourceToParse // or we don't have routing, if we get it in sourceToParse, we process it in preParse // which will always be called + return null; } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java index b6f269d75e18b..bd954a8b75625 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java @@ -134,8 +134,9 @@ public void postParse(ParseContext context) throws IOException { } @Override - public void parse(ParseContext context) throws IOException { + public Mapper parse(ParseContext context) throws IOException { // nothing to do here, we call the parent in postParse + return null; } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java index 21544a9e8594d..3814ba41ee6a2 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java @@ -251,8 +251,9 @@ public void postParse(ParseContext context) throws IOException { } @Override - public void parse(ParseContext context) throws IOException { + public Mapper parse(ParseContext context) throws IOException { // nothing to do here, we will call it in pre parse + return null; } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java index 4f2c57e3e9450..10a141697555b 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java @@ -175,7 +175,7 @@ public void postParse(ParseContext context) throws IOException { } @Override - public void parse(ParseContext context) throws IOException, MapperParsingException { + public Mapper parse(ParseContext context) throws IOException, MapperParsingException { if (context.sourceToParse().ttl() < 0) { // no ttl has been provided externally long ttl; if (context.parser().currentToken() == XContentParser.Token.VALUE_STRING) { @@ -188,6 +188,7 @@ public void parse(ParseContext context) throws IOException, MapperParsingExcepti } context.sourceToParse().ttl(ttl); } + return null; } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java index f8fa3984ef5a4..18075ff3866e8 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java @@ -273,8 +273,9 @@ public void postParse(ParseContext context) throws IOException { } @Override - public void parse(ParseContext context) throws IOException { + public Mapper parse(ParseContext context) throws IOException { // nothing to do here, we call the parent in preParse + return null; } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java index df72d57b0638c..c93a1545aec47 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java @@ -158,8 +158,9 @@ public void postParse(ParseContext context) throws IOException { } @Override - public void parse(ParseContext context) throws IOException { + public Mapper parse(ParseContext context) throws IOException { // we parse in pre parse + return null; } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java index 49e6242d0dfbb..d84835d9f3da8 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java @@ -167,8 +167,9 @@ public void postParse(ParseContext context) throws IOException { } @Override - public void parse(ParseContext context) throws IOException { + public Mapper parse(ParseContext context) throws IOException { // nothing to do here, we either do it in post parse, or in pre parse. + return null; } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java index 9822e4ede05b5..4ae3eaa415a87 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java @@ -113,8 +113,9 @@ protected void parseCreateField(ParseContext context, List fields) throws } @Override - public void parse(ParseContext context) throws IOException { + public Mapper parse(ParseContext context) throws IOException { // _version added in preparse + return null; } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java b/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java index 653f9f0804ac4..dfc4e64b3429a 100644 --- a/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.mapper.object; import com.google.common.collect.Iterables; + import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; @@ -38,6 +39,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.FieldMapperListener; @@ -45,6 +47,7 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilders; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperUtils; import org.elasticsearch.index.mapper.MergeContext; import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.ObjectMapperListener; @@ -84,7 +87,7 @@ /** * */ -public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll { +public class ObjectMapper implements Mapper, AllFieldMapper.IncludeInAll, Cloneable { public static final String CONTENT_TYPE = "object"; public static final String NESTED_CONTENT_TYPE = "nested"; @@ -370,8 +373,6 @@ protected Builder createBuilder(String name) { private volatile CopyOnWriteHashMap mappers; - private final Object mutex = new Object(); - ObjectMapper(String name, String fullPath, boolean enabled, Nested nested, Dynamic dynamic, ContentPath.Type pathType, Map mappers) { this.name = name; this.fullPath = fullPath; @@ -389,6 +390,28 @@ protected Builder createBuilder(String name) { this.nestedTypeFilter = new TermFilter(new Term(TypeFieldMapper.NAME, nestedTypePathAsBytes)); } + @Override + protected ObjectMapper clone() { + ObjectMapper clone; + try { + clone = (ObjectMapper) super.clone(); + } catch (CloneNotSupportedException e) { + throw new RuntimeException(); + } + return clone; + } + + /** + * Build a mapping update with the provided sub mapping update. + */ + public ObjectMapper mappingUpdate(Mapper mapper) { + ObjectMapper mappingUpdate = clone(); + // reset the sub mappers + mappingUpdate.mappers = new CopyOnWriteHashMap<>(); + mappingUpdate.putMapper(mapper); + return mappingUpdate; + } + @Override public String name() { return this.name; @@ -440,14 +463,16 @@ public Filter nestedTypeFilter() { return this.nestedTypeFilter; } - public ObjectMapper putMapper(Mapper mapper) { + /** + * Put a new mapper. + * NOTE: this method must be called under the current {@link DocumentMapper} + * lock if concurrent updates are expected. + */ + public void putMapper(Mapper mapper) { if (mapper instanceof AllFieldMapper.IncludeInAll) { ((AllFieldMapper.IncludeInAll) mapper).includeInAllIfNotSet(includeInAll); } - synchronized (mutex) { - mappers = mappers.copyAndPut(mapper.name(), mapper); - } - return this; + mappers = mappers.copyAndPut(mapper.name(), mapper); } @Override @@ -482,10 +507,10 @@ protected boolean allowValue() { } @Override - public void parse(ParseContext context) throws IOException { + public ObjectMapper parse(ParseContext context) throws IOException { if (!enabled) { context.parser().skipChildren(); - return; + return null; } XContentParser parser = context.parser(); @@ -493,13 +518,13 @@ public void parse(ParseContext context) throws IOException { XContentParser.Token token = parser.currentToken(); if (token == XContentParser.Token.VALUE_NULL) { // the object is null ("obj1" : null), simply bail - return; + return null; } if (token.isValue() && !allowValue()) { // if we are parsing an object but it is just a value, its only allowed on root level parsers with there // is a field name with the same name as the type - throw new MapperParsingException("object mapping for [" + name + "] tried to parse field [" + currentFieldName + "] as object, but found a concrete value", context.mappingsModified()); + throw new MapperParsingException("object mapping for [" + name + "] tried to parse field [" + currentFieldName + "] as object, but found a concrete value"); } if (nested.isNested()) { @@ -533,21 +558,30 @@ public void parse(ParseContext context) throws IOException { token = parser.nextToken(); } + ObjectMapper update = null; while (token != XContentParser.Token.END_OBJECT) { + ObjectMapper newUpdate = null; if (token == XContentParser.Token.START_OBJECT) { - serializeObject(context, currentFieldName); + newUpdate = serializeObject(context, currentFieldName); } else if (token == XContentParser.Token.START_ARRAY) { - serializeArray(context, currentFieldName); + newUpdate = serializeArray(context, currentFieldName); } else if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.VALUE_NULL) { serializeNullValue(context, currentFieldName); } else if (token == null) { - throw new MapperParsingException("object mapping for [" + name + "] tried to parse field [" + currentFieldName + "] as object, but got EOF, has a concrete value been provided to it?", context.mappingsModified()); + throw new MapperParsingException("object mapping for [" + name + "] tried to parse field [" + currentFieldName + "] as object, but got EOF, has a concrete value been provided to it?"); } else if (token.isValue()) { - serializeValue(context, currentFieldName, token); + newUpdate = serializeValue(context, currentFieldName, token); } token = parser.nextToken(); + if (newUpdate != null) { + if (update == null) { + update = newUpdate; + } else { + MapperUtils.merge(update, newUpdate); + } + } } // restore the enable path flag context.path().pathType(origPathType); @@ -577,6 +611,7 @@ public void parse(ParseContext context) throws IOException { } } } + return update; } private void serializeNullValue(ParseContext context, String lastFieldName) throws IOException { @@ -585,54 +620,51 @@ private void serializeNullValue(ParseContext context, String lastFieldName) thro if (mapper != null) { if (mapper instanceof FieldMapper) { if (!((FieldMapper) mapper).supportsNullValue()) { - throw new MapperParsingException("no object mapping found for null value in [" + lastFieldName + "]", context.mappingsModified()); + throw new MapperParsingException("no object mapping found for null value in [" + lastFieldName + "]"); } } mapper.parse(context); } else if (dynamic == Dynamic.STRICT) { - throw new StrictDynamicMappingException(fullPath, lastFieldName, context.mappingsModified()); + throw new StrictDynamicMappingException(fullPath, lastFieldName); } } - private void serializeObject(final ParseContext context, String currentFieldName) throws IOException { + private ObjectMapper serializeObject(final ParseContext context, String currentFieldName) throws IOException { if (currentFieldName == null) { - throw new MapperParsingException("object mapping [" + name + "] trying to serialize an object with no field associated with it, current value [" + context.parser().textOrNull() + "]", context.mappingsModified()); + throw new MapperParsingException("object mapping [" + name + "] trying to serialize an object with no field associated with it, current value [" + context.parser().textOrNull() + "]"); } context.path().add(currentFieldName); + ObjectMapper update = null; Mapper objectMapper = mappers.get(currentFieldName); if (objectMapper != null) { - objectMapper.parse(context); + final Mapper subUpdate = objectMapper.parse(context); + if (subUpdate != null) { + // propagate mapping update + update = mappingUpdate(subUpdate); + } } else { Dynamic dynamic = this.dynamic; if (dynamic == null) { dynamic = context.root().dynamic(); } if (dynamic == Dynamic.STRICT) { - throw new StrictDynamicMappingException(fullPath, currentFieldName, context.mappingsModified()); + throw new StrictDynamicMappingException(fullPath, currentFieldName); } else if (dynamic == Dynamic.TRUE) { - // we sync here just so we won't add it twice. Its not the end of the world - // to sync here since next operations will get it before - synchronized (mutex) { - objectMapper = mappers.get(currentFieldName); - if (objectMapper == null) { - // remove the current field name from path, since template search and the object builder add it as well... - context.path().remove(); - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "object"); - if (builder == null) { - builder = MapperBuilders.object(currentFieldName).enabled(true).pathType(pathType); - // if this is a non root object, then explicitly set the dynamic behavior if set - if (!(this instanceof RootObjectMapper) && this.dynamic != Defaults.DYNAMIC) { - ((Builder) builder).dynamic(this.dynamic); - } - } - BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path()); - objectMapper = builder.build(builderContext); - putDynamicMapper(context, currentFieldName, objectMapper); - } else { - objectMapper.parse(context); + // remove the current field name from path, since template search and the object builder add it as well... + context.path().remove(); + Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "object"); + if (builder == null) { + builder = MapperBuilders.object(currentFieldName).enabled(true).pathType(pathType); + // if this is a non root object, then explicitly set the dynamic behavior if set + if (!(this instanceof RootObjectMapper) && this.dynamic != Defaults.DYNAMIC) { + ((Builder) builder).dynamic(this.dynamic); } } + BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path()); + objectMapper = builder.build(builderContext); + context.path().add(currentFieldName); + update = mappingUpdate(MapperUtils.parseAndMergeUpdate(objectMapper, context)); } else { // not dynamic, read everything up to end object context.parser().skipChildren(); @@ -640,9 +672,10 @@ private void serializeObject(final ParseContext context, String currentFieldName } context.path().remove(); + return update; } - private void serializeArray(ParseContext context, String lastFieldName) throws IOException { + private ObjectMapper serializeArray(ParseContext context, String lastFieldName) throws IOException { String arrayFieldName = lastFieldName; Mapper mapper = mappers.get(lastFieldName); if (mapper != null) { @@ -650,9 +683,15 @@ private void serializeArray(ParseContext context, String lastFieldName) throws I // expects an array, if so we pass the context straight to the mapper and if not // we serialize the array components if (mapper instanceof ArrayValueMapperParser) { - mapper.parse(context); + final Mapper subUpdate = mapper.parse(context); + if (subUpdate != null) { + // propagate the mapping update + return mappingUpdate(subUpdate); + } else { + return null; + } } else { - serializeNonDynamicArray(context, lastFieldName, arrayFieldName); + return serializeNonDynamicArray(context, lastFieldName, arrayFieldName); } } else { @@ -661,278 +700,217 @@ private void serializeArray(ParseContext context, String lastFieldName) throws I dynamic = context.root().dynamic(); } if (dynamic == Dynamic.STRICT) { - throw new StrictDynamicMappingException(fullPath, arrayFieldName, context.mappingsModified()); + throw new StrictDynamicMappingException(fullPath, arrayFieldName); } else if (dynamic == Dynamic.TRUE) { - // we sync here just so we won't add it twice. Its not the end of the world - // to sync here since next operations will get it before - synchronized (mutex) { - mapper = mappers.get(arrayFieldName); - if (mapper == null) { - Mapper.Builder builder = context.root().findTemplateBuilder(context, arrayFieldName, "object"); - if (builder == null) { - serializeNonDynamicArray(context, lastFieldName, arrayFieldName); - return; - } - BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path()); - mapper = builder.build(builderContext); - if (mapper != null && mapper instanceof ArrayValueMapperParser) { - putDynamicMapper(context, arrayFieldName, mapper); - } else { - serializeNonDynamicArray(context, lastFieldName, arrayFieldName); - } - } else { - - serializeNonDynamicArray(context, lastFieldName, arrayFieldName); - } + Mapper.Builder builder = context.root().findTemplateBuilder(context, arrayFieldName, "object"); + if (builder == null) { + return serializeNonDynamicArray(context, lastFieldName, arrayFieldName); + } + BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path()); + mapper = builder.build(builderContext); + if (mapper != null && mapper instanceof ArrayValueMapperParser) { + context.path().add(arrayFieldName); + mapper = MapperUtils.parseAndMergeUpdate(mapper, context); + return mappingUpdate(mapper); + } else { + return serializeNonDynamicArray(context, lastFieldName, arrayFieldName); } } else { - - serializeNonDynamicArray(context, lastFieldName, arrayFieldName); + return serializeNonDynamicArray(context, lastFieldName, arrayFieldName); } } } - private void putDynamicMapper(ParseContext context, String arrayFieldName, Mapper mapper) throws IOException { - // ...now re add it - context.path().add(arrayFieldName); - context.setMappingsModified(); - - if (context.isWithinNewMapper()) { - // within a new mapper, no need to traverse, - // just parse - mapper.parse(context); - } else { - // create a context of new mapper, so we batch - // aggregate all the changes within - // this object mapper once, and traverse all of - // them to add them in a single go - context.setWithinNewMapper(); - try { - mapper.parse(context); - FieldMapperListener.Aggregator newFields = new FieldMapperListener.Aggregator(); - ObjectMapperListener.Aggregator newObjects = new ObjectMapperListener.Aggregator(); - mapper.traverse(newFields); - mapper.traverse(newObjects); - // callback on adding those fields! - context.docMapper().addFieldMappers(newFields.mappers); - context.docMapper().addObjectMappers(newObjects.mappers); - } finally { - context.clearWithinNewMapper(); - } - } - - // only put after we traversed and did the - // callbacks, so other parsing won't see it only - // after we - // properly traversed it and adding the mappers - putMapper(mapper); - } - - private void serializeNonDynamicArray(ParseContext context, String lastFieldName, String arrayFieldName) throws IOException { + private ObjectMapper serializeNonDynamicArray(ParseContext context, String lastFieldName, String arrayFieldName) throws IOException { XContentParser parser = context.parser(); XContentParser.Token token; while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token == XContentParser.Token.START_OBJECT) { - serializeObject(context, lastFieldName); + return serializeObject(context, lastFieldName); } else if (token == XContentParser.Token.START_ARRAY) { - serializeArray(context, lastFieldName); + return serializeArray(context, lastFieldName); } else if (token == XContentParser.Token.FIELD_NAME) { lastFieldName = parser.currentName(); } else if (token == XContentParser.Token.VALUE_NULL) { serializeNullValue(context, lastFieldName); } else if (token == null) { - throw new MapperParsingException("object mapping for [" + name + "] with array for [" + arrayFieldName + "] tried to parse as array, but got EOF, is there a mismatch in types for the same field?", context.mappingsModified()); + throw new MapperParsingException("object mapping for [" + name + "] with array for [" + arrayFieldName + "] tried to parse as array, but got EOF, is there a mismatch in types for the same field?"); } else { - serializeValue(context, lastFieldName, token); + return serializeValue(context, lastFieldName, token); } } + return null; } - private void serializeValue(final ParseContext context, String currentFieldName, XContentParser.Token token) throws IOException { + private ObjectMapper serializeValue(final ParseContext context, String currentFieldName, XContentParser.Token token) throws IOException { if (currentFieldName == null) { - throw new MapperParsingException("object mapping [" + name + "] trying to serialize a value with no field associated with it, current value [" + context.parser().textOrNull() + "]", context.mappingsModified()); + throw new MapperParsingException("object mapping [" + name + "] trying to serialize a value with no field associated with it, current value [" + context.parser().textOrNull() + "]"); } Mapper mapper = mappers.get(currentFieldName); if (mapper != null) { - mapper.parse(context); + Mapper subUpdate = mapper.parse(context); + if (subUpdate == null) { + return null; + } + return mappingUpdate(subUpdate); } else { - parseDynamicValue(context, currentFieldName, token); + return parseDynamicValue(context, currentFieldName, token); } } - public void parseDynamicValue(final ParseContext context, String currentFieldName, XContentParser.Token token) throws IOException { + public ObjectMapper parseDynamicValue(final ParseContext context, String currentFieldName, XContentParser.Token token) throws IOException { Dynamic dynamic = this.dynamic; if (dynamic == null) { dynamic = context.root().dynamic(); } if (dynamic == Dynamic.STRICT) { - throw new StrictDynamicMappingException(fullPath, currentFieldName, context.mappingsModified()); + throw new StrictDynamicMappingException(fullPath, currentFieldName); } if (dynamic == Dynamic.FALSE) { - return; - } - // we sync here since we don't want to add this field twice to the document mapper - // its not the end of the world, since we add it to the mappers once we create it - // so next time we won't even get here for this field - synchronized (mutex) { - Mapper mapper = mappers.get(currentFieldName); - if (mapper == null) { - BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path()); - if (token == XContentParser.Token.VALUE_STRING) { - boolean resolved = false; - - // do a quick test to see if its fits a dynamic template, if so, use it. - // we need to do it here so we can handle things like attachment templates, where calling - // text (to see if its a date) causes the binary value to be cleared - if (!resolved) { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "string", null); - if (builder != null) { - mapper = builder.build(builderContext); - resolved = true; - } - } + return null; + } + Mapper mapper = null; + BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path()); + if (token == XContentParser.Token.VALUE_STRING) { + boolean resolved = false; + + // do a quick test to see if its fits a dynamic template, if so, use it. + // we need to do it here so we can handle things like attachment templates, where calling + // text (to see if its a date) causes the binary value to be cleared + if (!resolved) { + Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "string", null); + if (builder != null) { + mapper = builder.build(builderContext); + resolved = true; + } + } - if (!resolved && context.root().dateDetection()) { - String text = context.parser().text(); - // a safe check since "1" gets parsed as well - if (Strings.countOccurrencesOf(text, ":") > 1 || Strings.countOccurrencesOf(text, "-") > 1 || Strings.countOccurrencesOf(text, "/") > 1) { - for (FormatDateTimeFormatter dateTimeFormatter : context.root().dynamicDateTimeFormatters()) { - try { - dateTimeFormatter.parser().parseMillis(text); - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "date"); - if (builder == null) { - builder = dateField(currentFieldName).dateTimeFormatter(dateTimeFormatter); - } - mapper = builder.build(builderContext); - resolved = true; - break; - } catch (Exception e) { - // failure to parse this, continue - } - } - } - } - if (!resolved && context.root().numericDetection()) { - String text = context.parser().text(); + if (!resolved && context.root().dateDetection()) { + String text = context.parser().text(); + // a safe check since "1" gets parsed as well + if (Strings.countOccurrencesOf(text, ":") > 1 || Strings.countOccurrencesOf(text, "-") > 1 || Strings.countOccurrencesOf(text, "/") > 1) { + for (FormatDateTimeFormatter dateTimeFormatter : context.root().dynamicDateTimeFormatters()) { try { - Long.parseLong(text); - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long"); + dateTimeFormatter.parser().parseMillis(text); + Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "date"); if (builder == null) { - builder = longField(currentFieldName); + builder = dateField(currentFieldName).dateTimeFormatter(dateTimeFormatter); } mapper = builder.build(builderContext); resolved = true; + break; } catch (Exception e) { - // not a long number - } - if (!resolved) { - try { - Double.parseDouble(text); - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double"); - if (builder == null) { - builder = doubleField(currentFieldName); - } - mapper = builder.build(builderContext); - resolved = true; - } catch (Exception e) { - // not a long number - } + // failure to parse this, continue } } - if (!resolved) { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "string"); - if (builder == null) { - builder = stringField(currentFieldName); - } - mapper = builder.build(builderContext); + } + } + if (!resolved && context.root().numericDetection()) { + String text = context.parser().text(); + try { + Long.parseLong(text); + Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long"); + if (builder == null) { + builder = longField(currentFieldName); } - } else if (token == XContentParser.Token.VALUE_NUMBER) { - XContentParser.NumberType numberType = context.parser().numberType(); - if (numberType == XContentParser.NumberType.INT) { - if (context.parser().estimatedNumberType()) { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long"); - if (builder == null) { - builder = longField(currentFieldName); - } - mapper = builder.build(builderContext); - } else { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "integer"); - if (builder == null) { - builder = integerField(currentFieldName); - } - mapper = builder.build(builderContext); - } - } else if (numberType == XContentParser.NumberType.LONG) { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long"); - if (builder == null) { - builder = longField(currentFieldName); - } - mapper = builder.build(builderContext); - } else if (numberType == XContentParser.NumberType.FLOAT) { - if (context.parser().estimatedNumberType()) { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double"); - if (builder == null) { - builder = doubleField(currentFieldName); - } - mapper = builder.build(builderContext); - } else { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "float"); - if (builder == null) { - builder = floatField(currentFieldName); - } - mapper = builder.build(builderContext); - } - } else if (numberType == XContentParser.NumberType.DOUBLE) { + mapper = builder.build(builderContext); + resolved = true; + } catch (Exception e) { + // not a long number + } + if (!resolved) { + try { + Double.parseDouble(text); Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double"); if (builder == null) { builder = doubleField(currentFieldName); } mapper = builder.build(builderContext); + resolved = true; + } catch (Exception e) { + // not a long number } - } else if (token == XContentParser.Token.VALUE_BOOLEAN) { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "boolean"); + } + } + if (!resolved) { + Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "string"); + if (builder == null) { + builder = stringField(currentFieldName); + } + mapper = builder.build(builderContext); + } + } else if (token == XContentParser.Token.VALUE_NUMBER) { + XContentParser.NumberType numberType = context.parser().numberType(); + if (numberType == XContentParser.NumberType.INT) { + if (context.parser().estimatedNumberType()) { + Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long"); if (builder == null) { - builder = booleanField(currentFieldName); + builder = longField(currentFieldName); } mapper = builder.build(builderContext); - } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "binary"); + } else { + Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "integer"); if (builder == null) { - builder = binaryField(currentFieldName); + builder = integerField(currentFieldName); } mapper = builder.build(builderContext); - } else { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, null); - if (builder != null) { - mapper = builder.build(builderContext); - } else { - // TODO how do we identify dynamically that its a binary value? - throw new ElasticsearchIllegalStateException("Can't handle serializing a dynamic type with content token [" + token + "] and field name [" + currentFieldName + "]"); - } } - - if (context.isWithinNewMapper()) { - mapper.parse(context); + } else if (numberType == XContentParser.NumberType.LONG) { + Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long"); + if (builder == null) { + builder = longField(currentFieldName); + } + mapper = builder.build(builderContext); + } else if (numberType == XContentParser.NumberType.FLOAT) { + if (context.parser().estimatedNumberType()) { + Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double"); + if (builder == null) { + builder = doubleField(currentFieldName); + } + mapper = builder.build(builderContext); } else { - context.setWithinNewMapper(); - try { - mapper.parse(context); - FieldMapperListener.Aggregator newFields = new FieldMapperListener.Aggregator(); - mapper.traverse(newFields); - context.docMapper().addFieldMappers(newFields.mappers); - } finally { - context.clearWithinNewMapper(); + Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "float"); + if (builder == null) { + builder = floatField(currentFieldName); } + mapper = builder.build(builderContext); } - - // only put after we traversed and did the callbacks, so other parsing won't see it only after we - // properly traversed it and adding the mappers - putMapper(mapper); - context.setMappingsModified(); + } else if (numberType == XContentParser.NumberType.DOUBLE) { + Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double"); + if (builder == null) { + builder = doubleField(currentFieldName); + } + mapper = builder.build(builderContext); + } + } else if (token == XContentParser.Token.VALUE_BOOLEAN) { + Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "boolean"); + if (builder == null) { + builder = booleanField(currentFieldName); + } + mapper = builder.build(builderContext); + } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { + Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "binary"); + if (builder == null) { + builder = binaryField(currentFieldName); + } + mapper = builder.build(builderContext); + } else { + Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, null); + if (builder != null) { + mapper = builder.build(builderContext); } else { - mapper.parse(context); + // TODO how do we identify dynamically that its a binary value? + throw new ElasticsearchIllegalStateException("Can't handle serializing a dynamic type with content token [" + token + "] and field name [" + currentFieldName + "]"); } } + + mapper = MapperUtils.parseAndMergeUpdate(mapper, context); + + ObjectMapper update = null; + if (mapper != null) { + update = mappingUpdate(mapper); + } + return update; } @Override @@ -966,33 +944,30 @@ public void merge(final Mapper mergeWith, final MergeContext mergeContext) throw List mappersToPut = new ArrayList<>(); FieldMapperListener.Aggregator newFieldMappers = new FieldMapperListener.Aggregator(); ObjectMapperListener.Aggregator newObjectMappers = new ObjectMapperListener.Aggregator(); - synchronized (mutex) { - for (Mapper mapper : mergeWithObject.mappers.values()) { - Mapper mergeWithMapper = mapper; - Mapper mergeIntoMapper = mappers.get(mergeWithMapper.name()); - if (mergeIntoMapper == null) { - // no mapping, simply add it if not simulating - if (!mergeContext.mergeFlags().simulate()) { - mappersToPut.add(mergeWithMapper); - mergeWithMapper.traverse(newFieldMappers); - mergeWithMapper.traverse(newObjectMappers); - } - } else { - mergeIntoMapper.merge(mergeWithMapper, mergeContext); + for (Mapper mapper : mergeWithObject.mappers.values()) { + Mapper mergeWithMapper = mapper; + Mapper mergeIntoMapper = mappers.get(mergeWithMapper.name()); + if (mergeIntoMapper == null) { + // no mapping, simply add it if not simulating + if (!mergeContext.mergeFlags().simulate()) { + mappersToPut.add(mergeWithMapper); + mergeWithMapper.traverse(newFieldMappers); + mergeWithMapper.traverse(newObjectMappers); } - } - if (!newFieldMappers.mappers.isEmpty()) { - mergeContext.docMapper().addFieldMappers(newFieldMappers.mappers); - } - if (!newObjectMappers.mappers.isEmpty()) { - mergeContext.docMapper().addObjectMappers(newObjectMappers.mappers); - } - // and the mappers only after the administration have been done, so it will not be visible to parser (which first try to read with no lock) - for (Mapper mapper : mappersToPut) { - putMapper(mapper); + } else { + mergeIntoMapper.merge(mergeWithMapper, mergeContext); } } - + if (!newFieldMappers.mappers.isEmpty()) { + mergeContext.addFieldMappers(newFieldMappers.mappers); + } + if (!newObjectMappers.mappers.isEmpty()) { + mergeContext.addObjectMappers(newObjectMappers.mappers); + } + // add the mappers only after the administration have been done, so it will not be visible to parser (which first try to read with no lock) + for (Mapper mapper : mappersToPut) { + putMapper(mapper); + } } protected void doMerge(ObjectMapper mergeWith, MergeContext mergeContext) { diff --git a/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java b/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java index fb188ff0f8f06..56d2b96429c24 100644 --- a/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java @@ -21,6 +21,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; + import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.joda.FormatDateTimeFormatter; @@ -206,6 +207,14 @@ protected boolean processField(ObjectMapper.Builder builder, String fieldName, O this.numericDetection = numericDetection; } + @Override + public ObjectMapper mappingUpdate(Mapper mapper) { + RootObjectMapper update = (RootObjectMapper) super.mappingUpdate(mapper); + // dynamic templates are irrelevant for dynamic mappings updates + update.dynamicTemplates = new DynamicTemplate[0]; + return update; + } + public boolean dateDetection() { return this.dateDetection; } @@ -231,7 +240,7 @@ public Mapper.Builder findTemplateBuilder(ParseContext context, String name, Str String mappingType = dynamicTemplate.mappingType(dynamicType); Mapper.TypeParser typeParser = parserContext.typeParser(mappingType); if (typeParser == null) { - throw new MapperParsingException("failed to find type parsed [" + mappingType + "] for [" + name + "]", context.mappingsModified()); + throw new MapperParsingException("failed to find type parsed [" + mappingType + "] for [" + name + "]"); } return typeParser.parse(name, dynamicTemplate.mappingForName(name, dynamicType), parserContext); } diff --git a/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/src/main/java/org/elasticsearch/index/shard/IndexShard.java index c05236d87af15..9e20a73b26be1 100644 --- a/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -457,7 +457,7 @@ static Engine.Create prepareCreate(Tuple docMapper, Sou ParsedDocument doc = docMapper.v1().parse(source).setMappingsModified(docMapper); return new Engine.Create(docMapper.v1(), docMapper.v1().uidMapper().term(doc.uid().stringValue()), doc, version, versionType, origin, startTime, canHaveDuplicates, autoGeneratedId); } catch (Throwable t) { - if (docMapper.v2() || (t instanceof MapperParsingException && ((MapperParsingException)t).isMappingsModified())) { + if (docMapper.v2()) { throw new WriteFailureException(t, docMapper.v1().type()); } else { throw t; @@ -493,7 +493,7 @@ static Engine.Index prepareIndex(Tuple docMapper, Sourc ParsedDocument doc = docMapper.v1().parse(source).setMappingsModified(docMapper); return new Engine.Index(docMapper.v1(), docMapper.v1().uidMapper().term(doc.uid().stringValue()), doc, version, versionType, origin, startTime, canHaveDuplicates); } catch (Throwable t) { - if (docMapper.v2() || (t instanceof MapperParsingException && ((MapperParsingException) t).isMappingsModified())) { + if (docMapper.v2()) { throw new WriteFailureException(t, docMapper.v1().type()); } else { throw t; diff --git a/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java b/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java index e234c5141e4cc..01d7846740f83 100644 --- a/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java @@ -37,23 +37,38 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.mapper.*; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.engine.Engine.Searcher; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.mapper.internal.IndexFieldMapper; import org.elasticsearch.index.mapper.internal.SizeFieldMapper; import org.elasticsearch.index.mapper.internal.SourceFieldMapper; -import org.elasticsearch.index.mapper.internal.TypeFieldMapper; import org.elasticsearch.test.ElasticsearchSingleNodeTest; import org.hamcrest.Matchers; import org.junit.Test; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static org.elasticsearch.common.io.Streams.copyToBytesFromClasspath; import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.hamcrest.Matchers.*; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; /** * @@ -82,7 +97,8 @@ public void testSimpleAllMappers() throws Exception { @Test public void testAllMappersNoBoost() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/noboost-mapping.json"); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + IndexService index = createIndex("test"); + DocumentMapper docMapper = index.mapperService().documentMapperParser().parse(mapping); byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json"); Document doc = docMapper.parse(new BytesArray(json)).rootDoc(); AllField field = (AllField) doc.getField("_all"); @@ -93,7 +109,6 @@ public void testAllMappersNoBoost() throws Exception { assertThat(allEntries.fields().contains("simple1"), equalTo(true)); FieldMapper mapper = docMapper.mappers().smartNameFieldMapper("_all"); assertThat(field.fieldType().omitNorms(), equalTo(false)); - assertThat(mapper.queryStringTermQuery(new Term("_all", "foobar")), Matchers.instanceOf(TermQuery.class)); } @Test @@ -110,7 +125,7 @@ public void testAllMappersTermQuery() throws Exception { assertThat(allEntries.fields().contains("simple1"), equalTo(true)); FieldMapper mapper = docMapper.mappers().smartNameFieldMapper("_all"); assertThat(field.fieldType().omitNorms(), equalTo(false)); - assertThat(mapper.queryStringTermQuery(new Term("_all", "foobar")), Matchers.instanceOf(TermQuery.class)); + assertThat(mapper.queryStringTermQuery(new Term("_all", "foobar")), Matchers.instanceOf(AllTermQuery.class)); } @@ -223,7 +238,6 @@ public void testRandom() throws Exception { boolean omitNorms = false; boolean stored = false; boolean enabled = true; - boolean autoBoost = false; boolean tv_stored = false; boolean tv_payloads = false; boolean tv_offsets = false; @@ -249,9 +263,6 @@ public void testRandom() throws Exception { if (randomBoolean()) { booleanOptionList.add(new Tuple<>("enabled", enabled = randomBoolean())); } - if (randomBoolean()) { - booleanOptionList.add(new Tuple<>("auto_boost", autoBoost = randomBoolean())); - } if (randomBoolean()) { booleanOptionList.add(new Tuple<>("store_term_vector_offsets", tv_offsets = randomBoolean())); } @@ -312,14 +323,6 @@ public void testRandom() throws Exception { } else { assertThat(field, nullValue()); } - - Term term = new Term("foo", "bar"); - Query query = builtDocMapper.allFieldMapper().queryStringTermQuery(term); - if (autoBoost) { - assertThat(query, equalTo((Query)new AllTermQuery(term))); - } else { - assertThat(query, equalTo((Query)new TermQuery(term))); - } if (similarity == null || similarity.equals("TF/IDF")) { assertThat(builtDocMapper.allFieldMapper().similarity(), nullValue()); } else { @@ -458,4 +461,19 @@ public void testDocValuesNotAllowed() throws IOException { assertThat(e.getDetailedMessage(), containsString("[_all] is always tokenized and cannot have doc values")); } } + + public void testAutoBoost() throws Exception { + for (boolean boost : new boolean[] {false, true}) { + String index = "test_" + boost; + IndexService indexService = createIndex(index, client().admin().indices().prepareCreate(index).addMapping("type", "foo", "type=string" + (boost ? ",boost=2" : ""))); + client().prepareIndex(index, "type").setSource("foo", "bar").get(); + client().admin().indices().prepareRefresh(index).get(); + Query query = indexService.mapperService().documentMapper("type").allFieldMapper().termQuery("bar", null); + try (Searcher searcher = indexService.shard(0).acquireSearcher("tests")) { + query = searcher.searcher().rewrite(query); + final Class expected = boost ? AllTermQuery.class : TermQuery.class; + assertThat(query, Matchers.instanceOf(expected)); + } + } + } } diff --git a/src/test/java/org/elasticsearch/index/mapper/dynamic/DynamicMappingTests.java b/src/test/java/org/elasticsearch/index/mapper/dynamic/DynamicMappingTests.java index e5189c4027bed..199c30d029a31 100644 --- a/src/test/java/org/elasticsearch/index/mapper/dynamic/DynamicMappingTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/dynamic/DynamicMappingTests.java @@ -19,15 +19,31 @@ package org.elasticsearch.index.mapper.dynamic; import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableMap; + +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.mapper.*; +import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.FieldMappers; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.mapper.StrictDynamicMappingException; import org.elasticsearch.test.ElasticsearchSingleNodeTest; -import org.junit.Test; import java.io.IOException; import java.util.LinkedHashMap; @@ -39,7 +55,6 @@ public class DynamicMappingTests extends ElasticsearchSingleNodeTest { - @Test public void testDynamicTrue() throws IOException { String mapping = jsonBuilder().startObject().startObject("type") .field("dynamic", "true") @@ -60,7 +75,6 @@ public void testDynamicTrue() throws IOException { assertThat(doc.rootDoc().get("field2"), equalTo("value2")); } - @Test public void testDynamicFalse() throws IOException { String mapping = jsonBuilder().startObject().startObject("type") .field("dynamic", "false") @@ -82,7 +96,6 @@ public void testDynamicFalse() throws IOException { } - @Test public void testDynamicStrict() throws IOException { String mapping = jsonBuilder().startObject().startObject("type") .field("dynamic", "strict") @@ -116,7 +129,6 @@ public void testDynamicStrict() throws IOException { } } - @Test public void testDynamicFalseWithInnerObjectButDynamicSetOnRoot() throws IOException { String mapping = jsonBuilder().startObject().startObject("type") .field("dynamic", "false") @@ -140,7 +152,6 @@ public void testDynamicFalseWithInnerObjectButDynamicSetOnRoot() throws IOExcept assertThat(doc.rootDoc().get("obj1.field2"), nullValue()); } - @Test public void testDynamicStrictWithInnerObjectButDynamicSetOnRoot() throws IOException { String mapping = jsonBuilder().startObject().startObject("type") .field("dynamic", "strict") @@ -173,7 +184,6 @@ public void testDynamicMappingOnEmptyString() throws Exception { assertTrue(mappers != null && mappers.isEmpty() == false); } - @Test public void testIndexingFailureDoesStillCreateType() throws IOException, InterruptedException { XContentBuilder mapping = jsonBuilder().startObject().startObject("_default_") .field("dynamic", "strict") @@ -202,7 +212,6 @@ public boolean apply(java.lang.Object input) { } - @Test public void testTypeCreatedProperly() throws IOException, InterruptedException { XContentBuilder mapping = jsonBuilder().startObject().startObject("_default_") .field("dynamic", "strict") @@ -243,7 +252,6 @@ public boolean apply(java.lang.Object input) { assertNotNull(getMappingsResponse.getMappings().get("test").get("type")); } - @Test public void testFieldsCreatedWithPartialParsing() throws IOException, InterruptedException { XContentBuilder mapping = jsonBuilder().startObject().startObject("doc") .startObject("properties") @@ -304,4 +312,178 @@ public boolean apply(java.lang.Object input) { } })); } + + private String serialize(ToXContent mapper) throws Exception { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + mapper.toXContent(builder, new ToXContent.MapParams(ImmutableMap.of())); + return builder.endObject().string(); + } + + private Mapper parse(DocumentMapper mapper, DocumentMapperParser parser, XContentBuilder builder) throws Exception { + Settings settings = ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); + ParseContext.InternalParseContext ctx = new ParseContext.InternalParseContext("test", settings, parser, mapper, new ContentPath(0)); + SourceToParse source = SourceToParse.source(builder.bytes()); + ctx.reset(XContentHelper.createParser(source.source()), new ParseContext.Document(), source, null); + assertEquals(XContentParser.Token.START_OBJECT, ctx.parser().nextToken()); + ctx.parser().nextToken(); + return mapper.root().parse(ctx); + } + + public void testDynamicMappingsNotNeeded() throws Exception { + IndexService indexService = createIndex("test"); + DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("foo").field("type", "string").endObject().endObject() + .endObject().string(); + + DocumentMapper mapper = parser.parse(mapping); + Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar").endObject()); + // foo is already defined in the mappings + assertNull(update); + } + + public void testField() throws Exception { + IndexService indexService = createIndex("test"); + DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").endObject().endObject() + .endObject().string(); + + DocumentMapper mapper = parser.parse(mapping); + assertEquals(mapping, serialize(mapper)); + + Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar").endObject()); + assertNotNull(update); + // original mapping not modified + assertEquals(mapping, serialize(mapper)); + // but we have an update + assertEquals("{\"type\":{\"properties\":{\"foo\":{\"type\":\"string\"}}}}", serialize(update)); + } + + public void testIncremental() throws Exception { + IndexService indexService = createIndex("test"); + DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); + // Make sure that mapping updates are incremental, this is important for performance otherwise + // every new field introduction runs in linear time with the total number of fields + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("foo").field("type", "string").endObject().endObject() + .endObject().string(); + + DocumentMapper mapper = parser.parse(mapping); + assertEquals(mapping, serialize(mapper)); + + Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar").field("bar", "baz").endObject()); + assertNotNull(update); + // original mapping not modified + assertEquals(mapping, serialize(mapper)); + // but we have an update + assertEquals(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + // foo is NOT in the update + .startObject("bar").field("type", "string").endObject() + .endObject().endObject().string(), serialize(update)); + } + + public void testIntroduceTwoFields() throws Exception { + IndexService indexService = createIndex("test"); + DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").endObject().endObject() + .endObject().string(); + + DocumentMapper mapper = parser.parse(mapping); + assertEquals(mapping, serialize(mapper)); + + Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar").field("bar", "baz").endObject()); + assertNotNull(update); + // original mapping not modified + assertEquals(mapping, serialize(mapper)); + // but we have an update + assertEquals(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("bar").field("type", "string").endObject() + .startObject("foo").field("type", "string").endObject() + .endObject().endObject().string(), serialize(update)); + } + + public void testObject() throws Exception { + IndexService indexService = createIndex("test"); + DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").endObject().endObject() + .endObject().string(); + + DocumentMapper mapper = parser.parse(mapping); + assertEquals(mapping, serialize(mapper)); + + Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().startObject("foo").startObject("bar").field("baz", "foo").endObject().endObject().endObject()); + assertNotNull(update); + // original mapping not modified + assertEquals(mapping, serialize(mapper)); + // but we have an update + assertEquals(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("foo").startObject("properties").startObject("bar").startObject("properties").startObject("baz").field("type", "string").endObject().endObject().endObject().endObject().endObject() + .endObject().endObject().endObject().string(), serialize(update)); + } + + public void testArray() throws Exception { + IndexService indexService = createIndex("test"); + DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").endObject().endObject() + .endObject().string(); + + DocumentMapper mapper = parser.parse(mapping); + assertEquals(mapping, serialize(mapper)); + + Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().startArray("foo").value("bar").value("baz").endArray().endObject()); + assertNotNull(update); + // original mapping not modified + assertEquals(mapping, serialize(mapper)); + // but we have an update + assertEquals(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("foo").field("type", "string").endObject() + .endObject().endObject().endObject().string(), serialize(update)); + } + + public void testInnerDynamicMapping() throws Exception { + IndexService indexService = createIndex("test"); + DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") + .startObject("foo").field("type", "object").endObject() + .endObject().endObject().endObject().string(); + + DocumentMapper mapper = parser.parse(mapping); + assertEquals(mapping, serialize(mapper)); + + Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().startObject("foo").startObject("bar").field("baz", "foo").endObject().endObject().endObject()); + assertNotNull(update); + // original mapping not modified + assertEquals(mapping, serialize(mapper)); + // but we have an update + assertEquals(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("foo").startObject("properties").startObject("bar").startObject("properties").startObject("baz").field("type", "string").endObject().endObject().endObject().endObject().endObject() + .endObject().endObject().endObject().string(), serialize(update)); + } + + public void testComplexArray() throws Exception { + IndexService indexService = createIndex("test"); + DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").endObject().endObject() + .endObject().string(); + + DocumentMapper mapper = parser.parse(mapping); + assertEquals(mapping, serialize(mapper)); + + Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().startArray("foo") + .startObject().field("bar", "baz").endObject() + .startObject().field("baz", 3).endObject() + .endArray().endObject()); + assertEquals(mapping, serialize(mapper)); + assertEquals(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("foo").startObject("properties") + .startObject("bar").field("type", "string").endObject() + .startObject("baz").field("type", "long").endObject() + .endObject().endObject() + .endObject().endObject().endObject().string(), serialize(update)); + } } diff --git a/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java b/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java index 26653c423a511..8df9f1e7d2a12 100755 --- a/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java +++ b/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java @@ -185,7 +185,7 @@ public FieldDataType defaultFieldDataType() { } @Override - public void parse(ParseContext context) throws IOException { + public Mapper parse(ParseContext context) throws IOException { byte[] bytes = "Hello world".getBytes(Charset.defaultCharset()); binMapper.parse(context.createExternalValueContext(bytes)); @@ -210,6 +210,7 @@ public void parse(ParseContext context) throws IOException { if (copyTo != null) { copyTo.parse(context); } + return null; } @Override diff --git a/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalRootMapper.java b/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalRootMapper.java index cb9596917c92c..4ec787accb875 100644 --- a/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalRootMapper.java +++ b/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalRootMapper.java @@ -39,7 +39,8 @@ public String name() { } @Override - public void parse(ParseContext context) throws IOException { + public Mapper parse(ParseContext context) throws IOException { + return null; } @Override diff --git a/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterTests.java b/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterTests.java index 3c690d97616d7..496cb58a692f2 100644 --- a/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterTests.java @@ -68,7 +68,7 @@ public void test_all_conflicts() throws Exception { "[_all] has different store_term_vector_payloads values", "[_all] has different analyzer", "[_all] has different similarity"}; - // auto_boost and fielddata and search_analyzer should not report conflict + // fielddata and search_analyzer should not report conflict testConflict(mapping, mappingUpdate, errorMessage); } diff --git a/src/test/java/org/elasticsearch/index/mapper/update/all_mapping_create_index.json b/src/test/java/org/elasticsearch/index/mapper/update/all_mapping_create_index.json index bda6b7bf6844e..2b9c42d50b2a5 100644 --- a/src/test/java/org/elasticsearch/index/mapper/update/all_mapping_create_index.json +++ b/src/test/java/org/elasticsearch/index/mapper/update/all_mapping_create_index.json @@ -2,7 +2,6 @@ "mappings": { "type": { "_all": { - "auto_boost": true, "store": true, "store_term_vectors": true, "store_term_vector_offsets": true, @@ -29,4 +28,4 @@ } } } -} \ No newline at end of file +} diff --git a/src/test/java/org/elasticsearch/index/mapper/update/all_mapping_update_with_conflicts.json b/src/test/java/org/elasticsearch/index/mapper/update/all_mapping_update_with_conflicts.json index 893804006d478..252aafefb08a9 100644 --- a/src/test/java/org/elasticsearch/index/mapper/update/all_mapping_update_with_conflicts.json +++ b/src/test/java/org/elasticsearch/index/mapper/update/all_mapping_update_with_conflicts.json @@ -1,7 +1,6 @@ { "type": { "_all": { - "auto_boost": false, "store": false, "enabled": false, "store_term_vectors": false, @@ -17,4 +16,4 @@ } } } -} \ No newline at end of file +} diff --git a/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java b/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java index b35ef5e696fcf..37f369fe8b9e2 100644 --- a/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java +++ b/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java @@ -372,11 +372,6 @@ private void randomIndexTemplate() throws IOException { .field("enabled", randomBoolean()) .endObject(); } - if (randomBoolean()) { - mappings.startObject(AllFieldMapper.NAME) - .field("auto_boost", true) - .endObject(); - } if (randomBoolean()) { mappings.startObject(SourceFieldMapper.NAME) .field("compress", randomBoolean()) diff --git a/src/test/resources/org/elasticsearch/bwcompat/index-1.5.0.zip b/src/test/resources/org/elasticsearch/bwcompat/index-1.5.0.zip index 598b7cc79b55a..4cb9dadb0f81c 100644 Binary files a/src/test/resources/org/elasticsearch/bwcompat/index-1.5.0.zip and b/src/test/resources/org/elasticsearch/bwcompat/index-1.5.0.zip differ diff --git a/src/test/resources/org/elasticsearch/bwcompat/repo-1.5.0.zip b/src/test/resources/org/elasticsearch/bwcompat/repo-1.5.0.zip index f8688d9038678..a07309a598d0e 100644 Binary files a/src/test/resources/org/elasticsearch/bwcompat/repo-1.5.0.zip and b/src/test/resources/org/elasticsearch/bwcompat/repo-1.5.0.zip differ