From 2bf0a718b1055352b2179d41709da87f047842f2 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 25 Jul 2014 16:59:42 +0200 Subject: [PATCH] Add multi_field support for Mapper externalValue (plugins) In context of mapper attachment and other mapper plugins, when dealing with multi fields, sub fields never get the `externalValue` although it was set. Here is a full script which reproduce the issue when used with mapper attachment plugin: ``` DELETE /test PUT /test { "mappings": { "test": { "properties": { "f": { "type": "attachment", "fields": { "f": { "analyzer": "english", "fields": { "no_stemming": { "type": "string", "store": "yes", "analyzer": "standard" } } } } } } } } } PUT /test/test/1 { "f": "VGhlIHF1aWNrIGJyb3duIGZveGVz" } GET /test/_search { "query": { "match": { "f": "quick" } } } GET /test/_search { "query": { "match": { "f.no_stemming": "quick" } } } GET /test/test/1?fields=f.no_stemming ``` Related to https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/57 Closes #5402. (cherry picked from commit 11eced0) --- .../index/mapper/DocumentMapper.java | 8 +- .../index/mapper/ParseContext.java | 714 +++++++++++++----- .../mapper/core/AbstractFieldMapper.java | 7 +- .../index/mapper/core/Murmur3FieldMapper.java | 3 +- .../index/mapper/geo/GeoPointFieldMapper.java | 9 +- .../mapper/externalvalues/ExternalMapper.java | 149 ++-- .../ExternalValuesMapperIntegrationTests.java | 65 +- .../externalvalues/RegisterExternalTypes.java | 6 +- .../SimpleExternalMappingTests.java | 177 +++++ 9 files changed, 868 insertions(+), 270 deletions(-) create mode 100644 src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java diff --git a/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 7b1d6fc7f0139..097e7cb4fae77 100644 --- a/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -248,10 +248,10 @@ public DocumentMapper build(DocumentMapperParser docMapperParser) { } - private CloseableThreadLocal cache = new CloseableThreadLocal() { + private CloseableThreadLocal cache = new CloseableThreadLocal() { @Override - protected ParseContext initialValue() { - return new ParseContext(index, indexSettings, docMapperParser, DocumentMapper.this, new ContentPath(0)); + protected ParseContext.InternalParseContext initialValue() { + return new ParseContext.InternalParseContext(index, indexSettings, docMapperParser, DocumentMapper.this, new ContentPath(0)); } }; @@ -484,7 +484,7 @@ public ParsedDocument parse(SourceToParse source) throws MapperParsingException } public ParsedDocument parse(SourceToParse source, @Nullable ParseListener listener) throws MapperParsingException { - ParseContext context = cache.get(); + ParseContext.InternalParseContext context = cache.get(); if (source.type() != null && !source.type().equals(this.type)) { throw new MapperParsingException("Type mismatch, provide type [" + source.type() + "] but mapper is of type [" + this.type + "]"); diff --git a/src/main/java/org/elasticsearch/index/mapper/ParseContext.java b/src/main/java/org/elasticsearch/index/mapper/ParseContext.java index b9da8af2c20f2..fcbe9dd89dfbc 100644 --- a/src/main/java/org/elasticsearch/index/mapper/ParseContext.java +++ b/src/main/java/org/elasticsearch/index/mapper/ParseContext.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.analysis.AnalysisService; +import org.elasticsearch.index.mapper.DocumentMapper.ParseListener; import org.elasticsearch.index.mapper.object.RootObjectMapper; import java.util.*; @@ -41,7 +42,7 @@ /** * */ -public class ParseContext { +public abstract class ParseContext { /** Fork of {@link org.apache.lucene.document.Document} with additional functionality. */ public static class Document implements Iterable { @@ -121,242 +122,572 @@ public BytesRef getBinaryValue(String name) { } - private final DocumentMapper docMapper; + private static class FilterParseContext extends ParseContext { - private final DocumentMapperParser docMapperParser; + private final ParseContext in; - private final ContentPath path; + private FilterParseContext(ParseContext in) { + this.in = in; + } - private XContentParser parser; + @Override + public boolean flyweight() { + return in.flyweight(); + } - private Document document; + @Override + public DocumentMapperParser docMapperParser() { + return in.docMapperParser(); + } - private List documents = Lists.newArrayList(); + @Override + public boolean mappingsModified() { + return in.mappingsModified(); + } - private Analyzer analyzer; + @Override + public void setMappingsModified() { + in.setMappingsModified(); + } - private final String index; + @Override + public void setWithinNewMapper() { + in.setWithinNewMapper(); + } - @Nullable - private final Settings indexSettings; + @Override + public void clearWithinNewMapper() { + in.clearWithinNewMapper(); + } - private SourceToParse sourceToParse; - private BytesReference source; + @Override + public boolean isWithinNewMapper() { + return in.isWithinNewMapper(); + } - private String id; + @Override + public boolean isWithinCopyTo() { + return in.isWithinCopyTo(); + } - private DocumentMapper.ParseListener listener; + @Override + public boolean isWithinMultiFields() { + return in.isWithinMultiFields(); + } - private Field uid, version; + @Override + public String index() { + return in.index(); + } - private StringBuilder stringBuilder = new StringBuilder(); + @Override + public Settings indexSettings() { + return in.indexSettings(); + } - private Map ignoredValues = new HashMap<>(); + @Override + public String type() { + return in.type(); + } - private boolean mappingsModified = false; - private boolean withinNewMapper = false; - private boolean withinCopyTo = false; - private boolean withinMultiFields = false; + @Override + public SourceToParse sourceToParse() { + return in.sourceToParse(); + } - private boolean externalValueSet; + @Override + public BytesReference source() { + return in.source(); + } - private Object externalValue; + @Override + public void source(BytesReference source) { + in.source(source); + } - private AllEntries allEntries = new AllEntries(); + @Override + public ContentPath path() { + return in.path(); + } - private float docBoost = 1.0f; + @Override + public XContentParser parser() { + return in.parser(); + } - public ParseContext(String index, @Nullable Settings indexSettings, DocumentMapperParser docMapperParser, DocumentMapper docMapper, ContentPath path) { - this.index = index; - this.indexSettings = indexSettings; - this.docMapper = docMapper; - this.docMapperParser = docMapperParser; - this.path = path; - } + @Override + public ParseListener listener() { + return in.listener(); + } - public void reset(XContentParser parser, Document document, SourceToParse source, DocumentMapper.ParseListener listener) { - this.parser = parser; - this.document = document; - if (document != null) { - this.documents = Lists.newArrayList(); - this.documents.add(document); - } else { - this.documents = null; - } - this.analyzer = null; - this.uid = null; - this.version = null; - this.id = null; - this.sourceToParse = source; - this.source = source == null ? null : sourceToParse.source(); - this.path.reset(); - this.mappingsModified = false; - this.withinNewMapper = false; - this.listener = listener == null ? DocumentMapper.ParseListener.EMPTY : listener; - this.allEntries = new AllEntries(); - this.ignoredValues.clear(); - this.docBoost = 1.0f; - } + @Override + public Document rootDoc() { + return in.rootDoc(); + } - public boolean flyweight() { - return sourceToParse.flyweight(); - } + @Override + public List docs() { + return in.docs(); + } - public DocumentMapperParser docMapperParser() { - return this.docMapperParser; - } + @Override + public Document doc() { + return in.doc(); + } - public boolean mappingsModified() { - return this.mappingsModified; - } + @Override + public void addDoc(Document doc) { + in.addDoc(doc); + } - public void setMappingsModified() { - this.mappingsModified = true; - } + @Override + public Document switchDoc(Document doc) { + return in.switchDoc(doc); + } - public void setWithinNewMapper() { - this.withinNewMapper = true; - } + @Override + public RootObjectMapper root() { + return in.root(); + } - public void clearWithinNewMapper() { - this.withinNewMapper = false; - } + @Override + public DocumentMapper docMapper() { + return in.docMapper(); + } + + @Override + public AnalysisService analysisService() { + return in.analysisService(); + } + + @Override + public String id() { + return in.id(); + } + + @Override + public void ignoredValue(String indexName, String value) { + in.ignoredValue(indexName, value); + } + + @Override + public String ignoredValue(String indexName) { + return in.ignoredValue(indexName); + } + + @Override + public void id(String id) { + in.id(id); + } + + @Override + public Field uid() { + return in.uid(); + } + + @Override + public void uid(Field uid) { + in.uid(uid); + } + + @Override + public Field version() { + return in.version(); + } + + @Override + public void version(Field version) { + in.version(version); + } + + @Override + public AllEntries allEntries() { + return in.allEntries(); + } + + @Override + public Analyzer analyzer() { + return in.analyzer(); + } + + @Override + public void analyzer(Analyzer analyzer) { + in.analyzer(analyzer); + } + + @Override + public boolean externalValueSet() { + return in.externalValueSet(); + } + + @Override + public Object externalValue() { + return in.externalValue(); + } + + @Override + public float docBoost() { + return in.docBoost(); + } + + @Override + public void docBoost(float docBoost) { + in.docBoost(docBoost); + } + + @Override + public StringBuilder stringBuilder() { + return in.stringBuilder(); + } - public boolean isWithinNewMapper() { - return withinNewMapper; } - public void setWithinCopyTo() { - this.withinCopyTo = true; + public static class InternalParseContext extends ParseContext { + + private final DocumentMapper docMapper; + + private final DocumentMapperParser docMapperParser; + + private final ContentPath path; + + private XContentParser parser; + + private Document document; + + private List documents = Lists.newArrayList(); + + private Analyzer analyzer; + + private final String index; + + @Nullable + private final Settings indexSettings; + + private SourceToParse sourceToParse; + private BytesReference source; + + private String id; + + private DocumentMapper.ParseListener listener; + + private Field uid, version; + + private StringBuilder stringBuilder = new StringBuilder(); + + private Map ignoredValues = new HashMap<>(); + + private boolean mappingsModified = false; + private boolean withinNewMapper = false; + + private AllEntries allEntries = new AllEntries(); + + private float docBoost = 1.0f; + + public InternalParseContext(String index, @Nullable Settings indexSettings, DocumentMapperParser docMapperParser, DocumentMapper docMapper, ContentPath path) { + this.index = index; + this.indexSettings = indexSettings; + this.docMapper = docMapper; + this.docMapperParser = docMapperParser; + this.path = path; + } + + public void reset(XContentParser parser, Document document, SourceToParse source, DocumentMapper.ParseListener listener) { + this.parser = parser; + this.document = document; + if (document != null) { + this.documents = Lists.newArrayList(); + this.documents.add(document); + } else { + this.documents = null; + } + this.analyzer = null; + this.uid = null; + this.version = null; + this.id = null; + this.sourceToParse = source; + this.source = source == null ? null : sourceToParse.source(); + this.path.reset(); + this.mappingsModified = false; + this.withinNewMapper = false; + this.listener = listener == null ? DocumentMapper.ParseListener.EMPTY : listener; + this.allEntries = new AllEntries(); + this.ignoredValues.clear(); + this.docBoost = 1.0f; + } + + public boolean flyweight() { + return sourceToParse.flyweight(); + } + + public DocumentMapperParser docMapperParser() { + return this.docMapperParser; + } + + public boolean mappingsModified() { + return this.mappingsModified; + } + + public void setMappingsModified() { + this.mappingsModified = true; + } + + public void setWithinNewMapper() { + this.withinNewMapper = true; + } + + public void clearWithinNewMapper() { + this.withinNewMapper = false; + } + + public boolean isWithinNewMapper() { + return withinNewMapper; + } + + public String index() { + return this.index; + } + + @Nullable + public Settings indexSettings() { + return this.indexSettings; + } + + public String type() { + return sourceToParse.type(); + } + + public SourceToParse sourceToParse() { + return this.sourceToParse; + } + + public BytesReference source() { + return source; + } + + // only should be used by SourceFieldMapper to update with a compressed source + public void source(BytesReference source) { + this.source = source; + } + + public ContentPath path() { + return this.path; + } + + public XContentParser parser() { + return this.parser; + } + + public DocumentMapper.ParseListener listener() { + return this.listener; + } + + public Document rootDoc() { + return documents.get(0); + } + + public List docs() { + return this.documents; + } + + public Document doc() { + return this.document; + } + + public void addDoc(Document doc) { + this.documents.add(doc); + } + + public Document switchDoc(Document doc) { + Document prev = this.document; + this.document = doc; + return prev; + } + + public RootObjectMapper root() { + return docMapper.root(); + } + + public DocumentMapper docMapper() { + return this.docMapper; + } + + public AnalysisService analysisService() { + return docMapperParser.analysisService; + } + + public String id() { + return id; + } + + public void ignoredValue(String indexName, String value) { + ignoredValues.put(indexName, value); + } + + public String ignoredValue(String indexName) { + return ignoredValues.get(indexName); + } + + /** + * Really, just the id mapper should set this. + */ + public void id(String id) { + this.id = id; + } + + public Field uid() { + return this.uid; + } + + /** + * Really, just the uid mapper should set this. + */ + public void uid(Field uid) { + this.uid = uid; + } + + public Field version() { + return this.version; + } + + public void version(Field version) { + this.version = version; + } + + public AllEntries allEntries() { + return this.allEntries; + } + + public Analyzer analyzer() { + return this.analyzer; + } + + public void analyzer(Analyzer analyzer) { + this.analyzer = analyzer; + } + + public float docBoost() { + return this.docBoost; + } + + public void docBoost(float docBoost) { + this.docBoost = docBoost; + } + + /** + * A string builder that can be used to construct complex names for example. + * Its better to reuse the. + */ + public StringBuilder stringBuilder() { + stringBuilder.setLength(0); + return this.stringBuilder; + } } - public void clearWithinCopyTo() { - this.withinCopyTo = false; + public abstract boolean flyweight(); + + public abstract DocumentMapperParser docMapperParser(); + + public abstract boolean mappingsModified(); + + public abstract void setMappingsModified(); + + public abstract void setWithinNewMapper(); + + public abstract void clearWithinNewMapper(); + + public abstract boolean isWithinNewMapper(); + + /** + * Return a new context that will be within a copy-to operation. + */ + public final ParseContext createCopyToContext() { + return new FilterParseContext(this) { + @Override + public boolean isWithinCopyTo() { + return true; + } + }; } public boolean isWithinCopyTo() { - return withinCopyTo; + return false; } - public void setWithinMultiFields() { - this.withinMultiFields = true; + /** + * Return a new context that will be within multi-fields. + */ + public final ParseContext createMultiFieldContext() { + return new FilterParseContext(this) { + @Override + public boolean isWithinMultiFields() { + return true; + } + }; } - public void clearWithinMultiFields() { - this.withinMultiFields = false; + public boolean isWithinMultiFields() { + return false; } - public String index() { - return this.index; - } + public abstract String index(); @Nullable - public Settings indexSettings() { - return this.indexSettings; - } + public abstract Settings indexSettings(); - public String type() { - return sourceToParse.type(); - } + public abstract String type(); - public SourceToParse sourceToParse() { - return this.sourceToParse; - } + public abstract SourceToParse sourceToParse(); - public BytesReference source() { - return source; - } + public abstract BytesReference source(); // only should be used by SourceFieldMapper to update with a compressed source - public void source(BytesReference source) { - this.source = source; - } + public abstract void source(BytesReference source); - public ContentPath path() { - return this.path; - } + public abstract ContentPath path(); - public XContentParser parser() { - return this.parser; - } + public abstract XContentParser parser(); - public DocumentMapper.ParseListener listener() { - return this.listener; - } + public abstract DocumentMapper.ParseListener listener(); - public Document rootDoc() { - return documents.get(0); - } + public abstract Document rootDoc(); - public List docs() { - return this.documents; - } + public abstract List docs(); - public Document doc() { - return this.document; - } + public abstract Document doc(); - public void addDoc(Document doc) { - this.documents.add(doc); - } + public abstract void addDoc(Document doc); - public Document switchDoc(Document doc) { - Document prev = this.document; - this.document = doc; - return prev; - } + public abstract Document switchDoc(Document doc); - public RootObjectMapper root() { - return docMapper.root(); - } + public abstract RootObjectMapper root(); - public DocumentMapper docMapper() { - return this.docMapper; - } + public abstract DocumentMapper docMapper(); - public AnalysisService analysisService() { - return docMapperParser.analysisService; - } + public abstract AnalysisService analysisService(); - public String id() { - return id; - } + public abstract String id(); - public void ignoredValue(String indexName, String value) { - ignoredValues.put(indexName, value); - } + public abstract void ignoredValue(String indexName, String value); - public String ignoredValue(String indexName) { - return ignoredValues.get(indexName); - } + public abstract String ignoredValue(String indexName); /** * Really, just the id mapper should set this. */ - public void id(String id) { - this.id = id; - } + public abstract void id(String id); - public Field uid() { - return this.uid; - } + public abstract Field uid(); /** * Really, just the uid mapper should set this. */ - public void uid(Field uid) { - this.uid = uid; - } + public abstract void uid(Field uid); - public Field version() { - return this.version; - } + public abstract Field version(); - public void version(Field version) { - this.version = version; - } + public abstract void version(Field version); - public boolean includeInAll(Boolean includeInAll, FieldMapper mapper) { + public final boolean includeInAll(Boolean includeInAll, FieldMapper mapper) { return includeInAll(includeInAll, mapper.fieldType().indexed()); } @@ -366,13 +697,13 @@ public boolean includeInAll(Boolean includeInAll, FieldMapper mapper) { * its actual value (so, if not set, defaults to "true") and the field is indexed. */ private boolean includeInAll(Boolean specificIncludeInAll, boolean indexed) { - if (withinCopyTo) { + if (isWithinCopyTo()) { return false; } - if (withinMultiFields) { + if (isWithinMultiFields()) { return false; } - if (!docMapper.allFieldMapper().enabled()) { + if (!docMapper().allFieldMapper().enabled()) { return false; } // not explicitly set @@ -382,30 +713,34 @@ private boolean includeInAll(Boolean specificIncludeInAll, boolean indexed) { return specificIncludeInAll; } - public AllEntries allEntries() { - return this.allEntries; - } + public abstract AllEntries allEntries(); - public Analyzer analyzer() { - return this.analyzer; - } + public abstract Analyzer analyzer(); - public void analyzer(Analyzer analyzer) { - this.analyzer = analyzer; - } + public abstract void analyzer(Analyzer analyzer); - public void externalValue(Object externalValue) { - this.externalValueSet = true; - this.externalValue = externalValue; + /** + * Return a new context that will have the external value set. + */ + public final ParseContext createExternalValueContext(final Object externalValue) { + return new FilterParseContext(this) { + @Override + public boolean externalValueSet() { + return true; + } + @Override + public Object externalValue() { + return externalValue; + } + }; } public boolean externalValueSet() { - return this.externalValueSet; + return false; } public Object externalValue() { - externalValueSet = false; - return externalValue; + throw new ElasticsearchIllegalStateException("External value is not set"); } /** @@ -413,7 +748,7 @@ public Object externalValue() { * @param clazz Expected class for external value * @return null if no external value has been set or the value */ - public T parseExternalValue(Class clazz) { + public final T parseExternalValue(Class clazz) { if (!externalValueSet() || externalValue() == null) { return null; } @@ -422,24 +757,17 @@ public T parseExternalValue(Class clazz) { throw new ElasticsearchIllegalArgumentException("illegal external value class [" + externalValue().getClass().getName() + "]. Should be " + clazz.getName()); } - return (T) externalValue(); + return clazz.cast(externalValue()); } - public float docBoost() { - return this.docBoost; - } + public abstract float docBoost(); - public void docBoost(float docBoost) { - this.docBoost = docBoost; - } + public abstract void docBoost(float docBoost); /** * A string builder that can be used to construct complex names for example. * Its better to reuse the. */ - public StringBuilder stringBuilder() { - stringBuilder.setLength(0); - return this.stringBuilder; - } + public abstract StringBuilder stringBuilder(); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java index 00f94fa9bf9e4..fc556f18f49ea 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java @@ -916,7 +916,7 @@ public void parse(AbstractFieldMapper mainField, ParseContext context) throws IO return; } - context.setWithinMultiFields(); + context = context.createMultiFieldContext(); ContentPath.Type origPathType = context.path().pathType(); context.path().pathType(pathType); @@ -927,8 +927,6 @@ public void parse(AbstractFieldMapper mainField, ParseContext context) throws IO } context.path().remove(); context.path().pathType(origPathType); - - context.clearWithinMultiFields(); } // No need for locking, because locking is taken care of in ObjectMapper#merge and DocumentMapper#merge @@ -1055,7 +1053,7 @@ public ImmutableList copyToFields() { * Creates an copy of the current field with given field name and boost */ public void parse(String field, ParseContext context) throws IOException { - context.setWithinCopyTo(); + context = context.createCopyToContext(); FieldMappers mappers = context.docMapper().mappers().indexName(field); if (mappers != null && !mappers.isEmpty()) { mappers.mapper().parse(context); @@ -1109,7 +1107,6 @@ public void parse(String field, ParseContext context) throws IOException { } } - context.clearWithinCopyTo(); } diff --git a/src/main/java/org/elasticsearch/index/mapper/core/Murmur3FieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/Murmur3FieldMapper.java index acaca4505ca8b..52676a04a9dc0 100644 --- a/src/main/java/org/elasticsearch/index/mapper/core/Murmur3FieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/core/Murmur3FieldMapper.java @@ -102,8 +102,7 @@ protected void innerParseCreateField(ParseContext context, List fields) t if (value != null) { final BytesRef bytes = new BytesRef(value.toString()); final long hash = MurmurHash3.hash128(bytes.bytes, bytes.offset, bytes.length, 0, new MurmurHash3.Hash128()).h1; - context.externalValue(hash); - super.innerParseCreateField(context, fields); + super.innerParseCreateField(context.createExternalValueContext(hash), fields); } } diff --git a/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java index a0c4f0c0b225b..964207189befb 100644 --- a/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java @@ -539,9 +539,8 @@ private void parseGeohashField(ParseContext context, String geohash) throws IOEx int min = enableGeohashPrefix ? 1 : geohash.length(); for (int i = len; i >= min; i--) { - context.externalValue(geohash.substring(0, i)); // side effect of this call is adding the field - geohashMapper.parse(context); + geohashMapper.parse(context.createExternalValueContext(geohash.substring(0, i))); } } @@ -580,10 +579,8 @@ private void parse(ParseContext context, GeoPoint point, String geohash) throws parseGeohashField(context, geohash); } if (enableLatLon) { - context.externalValue(point.lat()); - latMapper.parse(context); - context.externalValue(point.lon()); - lonMapper.parse(context); + latMapper.parse(context.createExternalValueContext(point.lat())); + lonMapper.parse(context.createExternalValueContext(point.lon())); } if (hasDocValues()) { CustomGeoPointDocValuesField field = (CustomGeoPointDocValuesField) context.doc().getByKey(names().indexName()); diff --git a/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java b/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java index ec075419f4103..9604709d376ae 100755 --- a/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java +++ b/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java @@ -20,10 +20,16 @@ package org.elasticsearch.index.mapper.externalvalues; import com.spatial4j.core.shape.Point; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.mapper.*; +import org.elasticsearch.index.mapper.core.AbstractFieldMapper; import org.elasticsearch.index.mapper.core.BinaryFieldMapper; import org.elasticsearch.index.mapper.core.BooleanFieldMapper; import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; @@ -31,8 +37,13 @@ import java.io.IOException; import java.nio.charset.Charset; +import java.util.List; import java.util.Map; +import static org.elasticsearch.index.mapper.MapperBuilders.stringField; +import static org.elasticsearch.index.mapper.core.TypeParsers.parseField; +import static org.elasticsearch.index.mapper.core.TypeParsers.parseMultiField; + /** * This mapper add a new sub fields * .bin Binary type @@ -40,7 +51,17 @@ * .point GeoPoint type * .shape GeoShape type */ -public class ExternalMapper implements Mapper { +public class ExternalMapper extends AbstractFieldMapper { + /** + * Returns the actual value of the field. + * + * @param value + */ + @Override + public Object value(Object value) { + return null; + } + public static class Names { public static final String FIELD_BIN = "bin"; public static final String FIELD_BOOL = "bool"; @@ -48,16 +69,27 @@ public static class Names { public static final String FIELD_SHAPE = "shape"; } - public static class Builder extends Mapper.Builder { + public static class Builder extends AbstractFieldMapper.Builder { private BinaryFieldMapper.Builder binBuilder = new BinaryFieldMapper.Builder(Names.FIELD_BIN); private BooleanFieldMapper.Builder boolBuilder = new BooleanFieldMapper.Builder(Names.FIELD_BOOL); private GeoPointFieldMapper.Builder pointBuilder = new GeoPointFieldMapper.Builder(Names.FIELD_POINT); private GeoShapeFieldMapper.Builder shapeBuilder = new GeoShapeFieldMapper.Builder(Names.FIELD_SHAPE); + private Mapper.Builder stringBuilder; + private String generatedValue; + private String mapperName; - public Builder(String name) { - super(name); + public Builder(String name, String generatedValue, String mapperName) { + super(name, new FieldType(Defaults.FIELD_TYPE)); this.builder = this; + this.stringBuilder = stringField(name).store(false); + this.generatedValue = generatedValue; + this.mapperName = mapperName; + } + + public Builder string(Mapper.Builder content) { + this.stringBuilder = content; + return this; } @Override @@ -70,81 +102,107 @@ public ExternalMapper build(BuilderContext context) { BooleanFieldMapper boolMapper = boolBuilder.build(context); GeoPointFieldMapper pointMapper = pointBuilder.build(context); GeoShapeFieldMapper shapeMapper = shapeBuilder.build(context); + Mapper stringMapper = stringBuilder.build(context); context.path().remove(); context.path().pathType(origPathType); - return new ExternalMapper(name, binMapper, boolMapper, pointMapper, shapeMapper); + return new ExternalMapper(buildNames(context), generatedValue, mapperName, binMapper, boolMapper, pointMapper, shapeMapper, stringMapper, + multiFieldsBuilder.build(this, context), copyTo); } } public static class TypeParser implements Mapper.TypeParser { + private String generatedValue; + private String mapperName; + + TypeParser(String mapperName, String generatedValue) { + this.mapperName = mapperName; + this.generatedValue = generatedValue; + } + @SuppressWarnings({"unchecked"}) @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - ExternalMapper.Builder builder = new ExternalMapper.Builder(name); + ExternalMapper.Builder builder = new ExternalMapper.Builder(name, generatedValue, mapperName); + parseField(builder, name, node, parserContext); + for (Map.Entry entry : node.entrySet()) { + String propName = Strings.toUnderscoreCase(entry.getKey()); + Object propNode = entry.getValue(); + + parseMultiField(builder, name, node, parserContext, propName, propNode); + } + return builder; } } - private final String name; + private final String generatedValue; + private final String mapperName; private final BinaryFieldMapper binMapper; private final BooleanFieldMapper boolMapper; private final GeoPointFieldMapper pointMapper; private final GeoShapeFieldMapper shapeMapper; - - public ExternalMapper(String name, - BinaryFieldMapper binMapper, BooleanFieldMapper boolMapper, GeoPointFieldMapper pointMapper, GeoShapeFieldMapper shapeMapper) { - this.name = name; + private final Mapper stringMapper; + + public ExternalMapper(FieldMapper.Names names, + String generatedValue, String mapperName, + BinaryFieldMapper binMapper, BooleanFieldMapper boolMapper, GeoPointFieldMapper pointMapper, + GeoShapeFieldMapper shapeMapper, Mapper stringMapper, MultiFields multiFields, CopyTo copyTo) { + super(names, 1.0f, Defaults.FIELD_TYPE, false, null, null, null, null, null, null, null, ImmutableSettings.EMPTY, + multiFields, copyTo); + this.generatedValue = generatedValue; + this.mapperName = mapperName; this.binMapper = binMapper; this.boolMapper = boolMapper; this.pointMapper = pointMapper; this.shapeMapper = shapeMapper; + this.stringMapper = stringMapper; } @Override - public String name() { - return name; + public FieldType defaultFieldType() { + return Defaults.FIELD_TYPE; } @Override - public void parse(ParseContext context) throws IOException { - ContentPath.Type origPathType = context.path().pathType(); - context.path().pathType(ContentPath.Type.FULL); - context.path().add(name); + public FieldDataType defaultFieldDataType() { + return null; + } - // Let's add a Dummy Binary content - context.path().add(Names.FIELD_BIN); + @Override + public void parse(ParseContext context) throws IOException { byte[] bytes = "Hello world".getBytes(Charset.defaultCharset()); - context.externalValue(bytes); - binMapper.parse(context); - context.path().remove(); + binMapper.parse(context.createExternalValueContext(bytes)); - // Let's add a Dummy Boolean content - context.path().add(Names.FIELD_BOOL); - context.externalValue(true); - boolMapper.parse(context); - context.path().remove(); + boolMapper.parse(context.createExternalValueContext(true)); // Let's add a Dummy Point Double lat = 42.0; Double lng = 51.0; - context.path().add(Names.FIELD_POINT); GeoPoint point = new GeoPoint(lat, lng); - context.externalValue(point); - pointMapper.parse(context); - context.path().remove(); + pointMapper.parse(context.createExternalValueContext(point)); // Let's add a Dummy Shape - context.path().add(Names.FIELD_SHAPE); Point shape = ShapeBuilder.newPoint(-100, 45).build(); - context.externalValue(shape); - shapeMapper.parse(context); - context.path().remove(); + shapeMapper.parse(context.createExternalValueContext(shape)); + + context = context.createExternalValueContext(generatedValue); + + // Let's add a Original String + stringMapper.parse(context); - context.path().pathType(origPathType); + multiFields.parse(this, context); + if (copyTo != null) { + copyTo.parse(context); + } + } + + @Override + protected void parseCreateField(ParseContext context, List fields) throws IOException { + throw new UnsupportedOperationException(); } @Override @@ -158,6 +216,7 @@ public void traverse(FieldMapperListener fieldMapperListener) { boolMapper.traverse(fieldMapperListener); pointMapper.traverse(fieldMapperListener); shapeMapper.traverse(fieldMapperListener); + stringMapper.traverse(fieldMapperListener); } @Override @@ -170,20 +229,20 @@ public void close() { boolMapper.close(); pointMapper.close(); shapeMapper.close(); + stringMapper.close(); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(name); - builder.field("type", RegisterExternalTypes.EXTERNAL); - builder.startObject("fields"); - binMapper.toXContent(builder, params); - boolMapper.toXContent(builder, params); - pointMapper.toXContent(builder, params); - shapeMapper.toXContent(builder, params); - builder.endObject(); - + builder.startObject(name()); + builder.field("type", mapperName); + multiFields.toXContent(builder, params); builder.endObject(); return builder; } + + @Override + protected String contentType() { + return mapperName; + } } diff --git a/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalValuesMapperIntegrationTests.java b/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalValuesMapperIntegrationTests.java index a65cd7fd2c185..5d61f770909a5 100644 --- a/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalValuesMapperIntegrationTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalValuesMapperIntegrationTests.java @@ -24,14 +24,12 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.FilterBuilders; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.junit.Test; -import java.io.IOException; - import static org.hamcrest.Matchers.equalTo; /** @@ -48,39 +46,78 @@ protected Settings nodeSettings(int nodeOrdinal) { } @Test - public void testExternalGeoPoint() throws Exception { - prepareCreate("test-idx").addMapping("doc", createMapping()).execute().get(); + public void testExternalValues() throws Exception { + prepareCreate("test-idx").addMapping("type", + XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("field").field("type", RegisterExternalTypes.EXTERNAL).endObject() + .endObject() + .endObject().endObject()).execute().get(); ensureYellow("test-idx"); - index("test-idx", "doc", "1", "external", "dummy"); + index("test-idx", "type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("field", "1234") + .endObject()); refresh(); SearchResponse response; response = client().prepareSearch("test-idx") - .setPostFilter(FilterBuilders.termFilter("external.bool", "T")) + .setPostFilter(FilterBuilders.termFilter("field.bool", "T")) .execute().actionGet(); assertThat(response.getHits().totalHits(), equalTo((long) 1)); response = client().prepareSearch("test-idx") - .setPostFilter(FilterBuilders.geoDistanceRangeFilter("external.point").point(42.0, 51.0).to("1km")) + .setPostFilter(FilterBuilders.geoDistanceRangeFilter("field.point").point(42.0, 51.0).to("1km")) .execute().actionGet(); assertThat(response.getHits().totalHits(), equalTo((long) 1)); response = client().prepareSearch("test-idx") - .setPostFilter(FilterBuilders.geoShapeFilter("external.shape", ShapeBuilder.newPoint(-100, 45), ShapeRelation.WITHIN)) + .setPostFilter(FilterBuilders.geoShapeFilter("field.shape", ShapeBuilder.newPoint(-100, 45), ShapeRelation.WITHIN)) .execute().actionGet(); assertThat(response.getHits().totalHits(), equalTo((long) 1)); - } + response = client().prepareSearch("test-idx") + .setPostFilter(FilterBuilders.termFilter("field.field", "foo")) + .execute().actionGet(); - private XContentBuilder createMapping() throws IOException { - return XContentFactory.jsonBuilder().startObject().startObject("doc").startObject("properties") - .startObject("external").field("type", RegisterExternalTypes.EXTERNAL).endObject() - .endObject().endObject().endObject(); + assertThat(response.getHits().totalHits(), equalTo((long) 1)); } + @Test + public void testExternalValuesWithMultifield() throws Exception { + prepareCreate("test-idx").addMapping("doc", + XContentFactory.jsonBuilder().startObject().startObject("doc").startObject("properties") + .startObject("f") + .field("type", RegisterExternalTypes.EXTERNAL_UPPER) + .startObject("fields") + .startObject("f") + .field("type", "string") + .field("stored", "yes") + .startObject("fields") + .startObject("raw") + .field("type", "string") + .field("index", "not_analyzed") + .field("stored", "yes") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject().endObject().endObject()).execute().get(); + ensureYellow("test-idx"); + + index("test-idx", "doc", "1", "f", "This is my text"); + refresh(); + + SearchResponse response = client().prepareSearch("test-idx") + .setQuery(QueryBuilders.termQuery("f.f.raw", "FOO BAR")) + .execute().actionGet(); + + assertThat(response.getHits().totalHits(), equalTo((long) 1)); + } } diff --git a/src/test/java/org/elasticsearch/index/mapper/externalvalues/RegisterExternalTypes.java b/src/test/java/org/elasticsearch/index/mapper/externalvalues/RegisterExternalTypes.java index a0ab806f54045..8a9ed9cac8e55 100755 --- a/src/test/java/org/elasticsearch/index/mapper/externalvalues/RegisterExternalTypes.java +++ b/src/test/java/org/elasticsearch/index/mapper/externalvalues/RegisterExternalTypes.java @@ -28,11 +28,15 @@ public class RegisterExternalTypes extends AbstractIndexComponent { public static final String EXTERNAL = "external"; + public static final String EXTERNAL_BIS = "external_bis"; + public static final String EXTERNAL_UPPER = "external_upper"; @Inject public RegisterExternalTypes(Index index, @IndexSettings Settings indexSettings, MapperService mapperService) { super(index, indexSettings); - mapperService.documentMapperParser().putTypeParser(EXTERNAL, new ExternalMapper.TypeParser()); + mapperService.documentMapperParser().putTypeParser(EXTERNAL, new ExternalMapper.TypeParser(EXTERNAL, "foo")); + mapperService.documentMapperParser().putTypeParser(EXTERNAL_BIS, new ExternalMapper.TypeParser(EXTERNAL_BIS, "bar")); + mapperService.documentMapperParser().putTypeParser(EXTERNAL_UPPER, new ExternalMapper.TypeParser(EXTERNAL_UPPER, "FOO BAR")); } } diff --git a/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java b/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java new file mode 100644 index 0000000000000..504bf4f4a368a --- /dev/null +++ b/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java @@ -0,0 +1,177 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.externalvalues; + +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.test.ElasticsearchSingleNodeLuceneTestCase; +import org.junit.Test; + +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +/** + */ +public class SimpleExternalMappingTests extends ElasticsearchSingleNodeLuceneTestCase { + + @Test + public void testExternalValues() throws Exception { + MapperService mapperService = createIndex("test").mapperService(); + mapperService.documentMapperParser().putTypeParser(RegisterExternalTypes.EXTERNAL, + new ExternalMapper.TypeParser(RegisterExternalTypes.EXTERNAL, "foo")); + + DocumentMapper documentMapper = mapperService.documentMapperParser().parse( + XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("field").field("type", "external").endObject() + .endObject() + .endObject().endObject().string() + ); + + ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("field", "1234") + .endObject() + .bytes()); + + assertThat(doc.rootDoc().getField("field.bool"), notNullValue()); + assertThat(doc.rootDoc().getField("field.bool").stringValue(), is("T")); + + assertThat(doc.rootDoc().getField("field.point"), notNullValue()); + assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0")); + + assertThat(doc.rootDoc().getField("field.shape"), notNullValue()); + + assertThat(doc.rootDoc().getField("field.field"), notNullValue()); + assertThat(doc.rootDoc().getField("field.field").stringValue(), is("foo")); + + + } + + @Test + public void testExternalValuesWithMultifield() throws Exception { + MapperService mapperService = createIndex("test").mapperService(); + mapperService.documentMapperParser().putTypeParser(RegisterExternalTypes.EXTERNAL, + new ExternalMapper.TypeParser(RegisterExternalTypes.EXTERNAL, "foo")); + + DocumentMapper documentMapper = mapperService.documentMapperParser().parse( + XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("field") + .field("type", RegisterExternalTypes.EXTERNAL) + .startObject("fields") + .startObject("field") + .field("type", "string") + .field("stored", "yes") + .startObject("fields") + .startObject("raw") + .field("type", "string") + .field("index", "not_analyzed") + .field("stored", "yes") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject().endObject().endObject() + .string()); + + ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("field", "1234") + .endObject() + .bytes()); + + assertThat(doc.rootDoc().getField("field.bool"), notNullValue()); + assertThat(doc.rootDoc().getField("field.bool").stringValue(), is("T")); + + assertThat(doc.rootDoc().getField("field.point"), notNullValue()); + assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0")); + + assertThat(doc.rootDoc().getField("field.shape"), notNullValue()); + + assertThat(doc.rootDoc().getField("field.field"), notNullValue()); + assertThat(doc.rootDoc().getField("field.field").stringValue(), is("foo")); + + assertThat(doc.rootDoc().getField("field.field.raw"), notNullValue()); + assertThat(doc.rootDoc().getField("field.field.raw").stringValue(), is("foo")); + } + + @Test + public void testExternalValuesWithMultifieldTwoLevels() throws Exception { + MapperService mapperService = createIndex("test").mapperService(); + + mapperService.documentMapperParser().putTypeParser(RegisterExternalTypes.EXTERNAL, + new ExternalMapper.TypeParser(RegisterExternalTypes.EXTERNAL, "foo")); + mapperService.documentMapperParser().putTypeParser(RegisterExternalTypes.EXTERNAL_BIS, + new ExternalMapper.TypeParser(RegisterExternalTypes.EXTERNAL_BIS, "bar")); + + DocumentMapper documentMapper = mapperService.documentMapperParser().parse( + XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("field") + .field("type", RegisterExternalTypes.EXTERNAL) + .startObject("fields") + .startObject("field") + .field("type", "string") + .startObject("fields") + .startObject("generated") + .field("type", RegisterExternalTypes.EXTERNAL_BIS) + .endObject() + .startObject("raw") + .field("type", "string") + .endObject() + .endObject() + .endObject() + .startObject("raw") + .field("type", "string") + .endObject() + .endObject() + .endObject() + .endObject().endObject().endObject() + .string()); + + ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("field", "1234") + .endObject() + .bytes()); + + assertThat(doc.rootDoc().getField("field.bool"), notNullValue()); + assertThat(doc.rootDoc().getField("field.bool").stringValue(), is("T")); + + assertThat(doc.rootDoc().getField("field.point"), notNullValue()); + assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0")); + + assertThat(doc.rootDoc().getField("field.shape"), notNullValue()); + + assertThat(doc.rootDoc().getField("field.field"), notNullValue()); + assertThat(doc.rootDoc().getField("field.field").stringValue(), is("foo")); + + assertThat(doc.rootDoc().getField("field.field.generated.generated"), notNullValue()); + assertThat(doc.rootDoc().getField("field.field.generated.generated").stringValue(), is("bar")); + + assertThat(doc.rootDoc().getField("field.field.raw"), notNullValue()); + assertThat(doc.rootDoc().getField("field.field.raw").stringValue(), is("foo")); + + assertThat(doc.rootDoc().getField("field.raw"), notNullValue()); + assertThat(doc.rootDoc().getField("field.raw").stringValue(), is("foo")); + } +}