From e2cd8bdc61dc20cec5a2688f25fc6dc135036f42 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Wed, 31 May 2017 10:51:51 +0200 Subject: [PATCH 1/3] Introduce ParentJoinFieldMapper, a field mapper that creates parent/child relation within documents of the same index This change adds a new field mapper named ParentJoinFieldMapper. This mapper is a replacement for the ParentFieldMapper but instead of using the types in the mapping it uses an internal field to materialize parent/child relation within a single index. This change also adds a fetch sub phase that automatically retrieves the join name (parent or child name) and the parent id for child documents in the response hit fields. The compatibility with `has_parent`, `has_child` queries and `children` agg will be added in a follow up. Relates #20257 --- .../elasticsearch/join/ParentJoinPlugin.java | 16 +- .../ParentToChildrenAggregator.java | 15 +- .../fetch/ParentJoinFieldSubFetchPhase.java | 102 +++++ .../join/mapper/ParentIDFieldMapper.java | 175 ++++++++ .../join/mapper/ParentJoinFieldMapper.java | 407 ++++++++++++++++++ .../ParentJoinFieldSubFetchPhaseTests.java | 159 +++++++ .../mapper/ParentJoinFieldMapperTests.java | 353 +++++++++++++++ .../{10_basic.yml => 10_parent_child.yml} | 0 .../rest-api-spec/test/20_parent_join.yml | 65 +++ 9 files changed, 1286 insertions(+), 6 deletions(-) create mode 100644 modules/parent-join/src/main/java/org/elasticsearch/join/fetch/ParentJoinFieldSubFetchPhase.java create mode 100644 modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIDFieldMapper.java create mode 100644 modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java create mode 100644 modules/parent-join/src/test/java/org/elasticsearch/join/fetch/ParentJoinFieldSubFetchPhaseTests.java create mode 100644 modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java rename modules/parent-join/src/test/resources/rest-api-spec/test/{10_basic.yml => 10_parent_child.yml} (100%) create mode 100644 modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/ParentJoinPlugin.java b/modules/parent-join/src/main/java/org/elasticsearch/join/ParentJoinPlugin.java index dec3950836aab..83033545cfbb7 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/ParentJoinPlugin.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/ParentJoinPlugin.java @@ -20,18 +20,24 @@ package org.elasticsearch.join; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.join.aggregations.ChildrenAggregationBuilder; import org.elasticsearch.join.aggregations.InternalChildren; +import org.elasticsearch.join.fetch.ParentJoinFieldSubFetchPhase; +import org.elasticsearch.join.mapper.ParentJoinFieldMapper; import org.elasticsearch.join.query.HasChildQueryBuilder; import org.elasticsearch.join.query.HasParentQueryBuilder; +import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.search.fetch.FetchSubPhase; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; -public class ParentJoinPlugin extends Plugin implements SearchPlugin { +public class ParentJoinPlugin extends Plugin implements SearchPlugin, MapperPlugin { public ParentJoinPlugin(Settings settings) {} @Override @@ -50,5 +56,13 @@ public List getAggregations() { ); } + @Override + public Map getMappers() { + return Collections.singletonMap(ParentJoinFieldMapper.CONTENT_TYPE, new ParentJoinFieldMapper.TypeParser()); + } + @Override + public List getFetchSubPhases(FetchPhaseConstructionContext context) { + return Collections.singletonList(new ParentJoinFieldSubFetchPhase()); + } } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java index 93ba1b98da15e..600179d25931f 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java @@ -70,11 +70,16 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator { private final LongObjectPagedHashMap parentOrdToOtherBuckets; private boolean multipleBucketsPerParentOrd = false; - public ParentToChildrenAggregator(String name, AggregatorFactories factories, - SearchContext context, Aggregator parent, Query childFilter, - Query parentFilter, ValuesSource.Bytes.WithOrdinals valuesSource, - long maxOrd, List pipelineAggregators, Map metaData) - throws IOException { + public ParentToChildrenAggregator(String name, + AggregatorFactories factories, + SearchContext context, + Aggregator parent, + Query childFilter, + Query parentFilter, + ValuesSource.Bytes.WithOrdinals valuesSource, + long maxOrd, + List pipelineAggregators, + Map metaData) throws IOException { super(name, factories, context, parent, pipelineAggregators, metaData); // these two filters are cached in the parser this.childFilter = context.searcher().createNormalizedWeight(childFilter, false); diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/fetch/ParentJoinFieldSubFetchPhase.java b/modules/parent-join/src/main/java/org/elasticsearch/join/fetch/ParentJoinFieldSubFetchPhase.java new file mode 100644 index 0000000000000..8c0cc806c905d --- /dev/null +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/fetch/ParentJoinFieldSubFetchPhase.java @@ -0,0 +1,102 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.join.fetch; + +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.Version; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.join.mapper.ParentJoinFieldMapper; +import org.elasticsearch.search.SearchHitField; +import org.elasticsearch.search.fetch.FetchSubPhase; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * A sub fetch phase that retrieves the join name and the parent id for each document containing + * a {@link ParentJoinFieldMapper} field. + */ +public final class ParentJoinFieldSubFetchPhase implements FetchSubPhase { + @Override + public void hitExecute(SearchContext context, HitContext hitContext) { + if (context.storedFieldsContext() != null && context.storedFieldsContext().fetchFields() == false) { + return; + } + if (context.mapperService().getIndexSettings().getIndexVersionCreated().before(Version.V_6_0_0_alpha2)) { + return; + } + DocumentMapper docMapper = context.mapperService().documentMapper(hitContext.hit().getType()); + Tuple joinField = null; + Tuple parentField = null; + for (FieldMapper fieldMapper : docMapper.mappers()) { + if (fieldMapper instanceof ParentJoinFieldMapper) { + String joinName = getSortedDocValue(fieldMapper.name(), hitContext.reader(), hitContext.docId()); + if (joinName != null) { + ParentJoinFieldMapper joinFieldMapper = (ParentJoinFieldMapper) fieldMapper; + joinField = new Tuple<>(fieldMapper.name(), joinName); + // we retrieve the parent id only for children. + FieldMapper parentMapper = joinFieldMapper.getParentIDFieldMapper(joinName, false); + if (parentMapper != null) { + String parent = getSortedDocValue(parentMapper.name(), hitContext.reader(), hitContext.docId()); + parentField = new Tuple<>(parentMapper.name(), parent); + } + break; + } + } + } + + if (joinField == null) { + // hit has no join field. + return; + } + + Map fields = hitContext.hit().fieldsOrNull(); + if (fields == null) { + fields = new HashMap<>(); + hitContext.hit().fields(fields); + } + fields.put(joinField.v1(), new SearchHitField(joinField.v1(), Collections.singletonList(joinField.v2()))); + if (parentField != null) { + fields.put(parentField.v1(), new SearchHitField(parentField.v1(), Collections.singletonList(parentField.v2()))); + } + } + + private String getSortedDocValue(String field, LeafReader reader, int docId) { + try { + SortedDocValues docValues = reader.getSortedDocValues(field); + if (docValues == null || docValues.advanceExact(docId) == false) { + return null; + } + int ord = docValues.ordValue(); + BytesRef joinName = docValues.lookupOrd(ord); + return joinName.utf8ToString(); + } catch (IOException e) { + throw ExceptionsHelper.convertToElastic(e); + } + } +} diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIDFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIDFieldMapper.java new file mode 100644 index 0000000000000..7f6ec93aed33a --- /dev/null +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIDFieldMapper.java @@ -0,0 +1,175 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.join.mapper; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.StringFieldType; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Set; + +/** + * A field mapper used internally by the {@link ParentJoinFieldMapper} to index + * the value that link documents in the index (parent _id or _id if the document is a parent). + */ +public final class ParentIDFieldMapper extends FieldMapper { + public static final String NAME = "parent"; + public static final String CONTENT_TYPE = "parent"; + + static class Defaults { + public static final MappedFieldType FIELD_TYPE = new ParentIDFieldType(); + + static { + FIELD_TYPE.setTokenized(false); + FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.setHasDocValues(true); + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE.freeze(); + } + } + + static class Builder extends FieldMapper.Builder { + private final String parent; + private final Set children; + + Builder(String name, String parent, Set children) { + super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE); + builder = this; + this.parent = parent; + this.children = children; + } + + public Set getChildren() { + return children; + } + + @Override + public ParentIDFieldMapper build(BuilderContext context) { + fieldType.setName(name); + return new ParentIDFieldMapper(name, parent, children, fieldType, context.indexSettings()); + } + } + + public static final class ParentIDFieldType extends StringFieldType { + public ParentIDFieldType() { + setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); + setSearchAnalyzer(Lucene.KEYWORD_ANALYZER); + } + + protected ParentIDFieldType(ParentIDFieldType ref) { + super(ref); + } + + public ParentIDFieldType clone() { + return new ParentIDFieldType(this); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public IndexFieldData.Builder fielddataBuilder() { + failIfNoDocValues(); + return new DocValuesIndexFieldData.Builder(); + } + + @Override + public Object valueForDisplay(Object value) { + if (value == null) { + return null; + } + BytesRef binaryValue = (BytesRef) value; + return binaryValue.utf8ToString(); + } + } + + private final String parentName; + private Set children; + + protected ParentIDFieldMapper(String simpleName, + String parentName, + Set children, + MappedFieldType fieldType, + Settings indexSettings) { + super(simpleName, fieldType, Defaults.FIELD_TYPE, indexSettings, MultiFields.empty(), null); + this.parentName = parentName; + this.children = children; + } + + @Override + protected ParentIDFieldMapper clone() { + return (ParentIDFieldMapper) super.clone(); + } + + /** + * Returns the parent name associated with this mapper. + */ + public String getParentName() { + return parentName; + } + + /** + * Returns the children names associated with this mapper. + */ + public Collection getChildren() { + return children; + } + + @Override + protected void parseCreateField(ParseContext context, List fields) throws IOException { + if (context.externalValueSet() == false) { + throw new IllegalStateException("external value not set"); + } + String refId = (String) context.externalValue(); + BytesRef binaryValue = new BytesRef(refId); + Field field = new Field(fieldType().name(), binaryValue, fieldType()); + fields.add(field); + fields.add(new SortedDocValuesField(fieldType().name(), binaryValue)); + } + + + @Override + protected void doMerge(Mapper mergeWith, boolean updateAllTypes) { + super.doMerge(mergeWith, updateAllTypes); + ParentIDFieldMapper parentMergeWith = (ParentIDFieldMapper) mergeWith; + this.children = parentMergeWith.children; + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } +} diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java new file mode 100644 index 0000000000000..2811404bd1da4 --- /dev/null +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -0,0 +1,407 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.join.mapper; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; +import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.StringFieldType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * A {@link FieldMapper} that creates hierarchical joins (parent-join) between documents in the same index. + */ +public final class ParentJoinFieldMapper extends FieldMapper { + public static final String NAME = "join"; + public static final String CONTENT_TYPE = "join"; + + public static class Defaults { + public static final MappedFieldType FIELD_TYPE = new JoinFieldType(); + + static { + FIELD_TYPE.setTokenized(false); + FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.setHasDocValues(true); + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE.freeze(); + } + } + + static String getParentIDFieldName(String joinFieldName, String parentName) { + return joinFieldName + "#" + parentName; + } + + static void checkPreConditions(Version indexCreatedVersion, ContentPath path, String name) { + if (indexCreatedVersion.before(Version.V_6_0_0_alpha2)) { + throw new IllegalStateException("unable to create join field [" + name + + "] for index created before " + Version.V_6_0_0_alpha2); + } + + if (path.pathAsText(name).contains(".")) { + throw new IllegalArgumentException("join field [" + path.pathAsText(name) + "] " + + "cannot be added inside an object or in a multi-field"); + } + } + + static void checkParentFields(String name, List mappers) { + Set children = new HashSet<>(); + List conflicts = new ArrayList<>(); + for (ParentIDFieldMapper mapper : mappers) { + for (String child : mapper.getChildren()) { + if (children.add(child) == false) { + conflicts.add("[" + child + "] cannot have multiple parents"); + } + } + } + if (conflicts.isEmpty() == false) { + throw new IllegalArgumentException("invalid definition for join field [" + name + "]:\n" + conflicts.toString()); + } + } + + static void checkDuplicateJoinFields(ParseContext.Document doc) { + if (doc.getFields().stream().anyMatch((m) -> m.fieldType() instanceof JoinFieldType)) { + throw new IllegalStateException("cannot have two join fields in the same document"); + } + } + + public static class Builder extends FieldMapper.Builder { + final List parentIDFieldBuilders = new ArrayList<>(); + + public Builder(String name) { + super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE); + builder = this; + } + + @Override + public JoinFieldType fieldType() { + return (JoinFieldType) super.fieldType(); + } + + public Builder addParent(String parent, Set children) { + String parentIDFieldName = getParentIDFieldName(name, parent); + parentIDFieldBuilders.add(new ParentIDFieldMapper.Builder(parentIDFieldName, parent, children)); + return this; + } + + @Override + public ParentJoinFieldMapper build(BuilderContext context) { + checkPreConditions(context.indexCreatedVersion(), context.path(), name); + fieldType.setName(name); + final List parentIDFields = new ArrayList<>(); + parentIDFieldBuilders.stream().map((e) -> e.build(context)).forEach(parentIDFields::add); + checkParentFields(name(), parentIDFields); + return new ParentJoinFieldMapper(name, fieldType, context.indexSettings(), + Collections.unmodifiableList(parentIDFields)); + } + } + + public static class TypeParser implements Mapper.TypeParser { + @Override + public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + final IndexSettings indexSettings = parserContext.mapperService().getIndexSettings(); + if (indexSettings.getIndexMetaData().isRoutingPartitionedIndex()) { + throw new IllegalStateException("cannot set join field [" + name + "] for the partitioned index " + + "[" + indexSettings.getIndex().getName() + "]"); + } + + Builder builder = new Builder(name); + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + if ("type".equals(entry.getKey())) { + continue; + } + + final String parent = entry.getKey(); + Set children; + if (entry.getValue() instanceof List) { + children = new HashSet<>(); + for (Object childObj : (List) entry.getValue()) { + if (childObj instanceof String) { + children.add(childObj.toString()); + } else { + throw new MapperParsingException("[" + parent + "] expected an array of strings but was:" + + childObj.getClass().getSimpleName()); + } + } + children = Collections.unmodifiableSet(children); + } else if (entry.getValue() instanceof String) { + children = Collections.singleton(entry.getValue().toString()); + } else { + throw new MapperParsingException("[" + parent + "] expected string but was:" + + entry.getValue().getClass().getSimpleName()); + } + builder.addParent(parent, children); + iterator.remove(); + } + return builder; + } + } + + public static final class JoinFieldType extends StringFieldType { + public JoinFieldType() { + setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); + setSearchAnalyzer(Lucene.KEYWORD_ANALYZER); + } + + protected JoinFieldType(JoinFieldType ref) { + super(ref); + } + + public JoinFieldType clone() { + return new JoinFieldType(this); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public IndexFieldData.Builder fielddataBuilder() { + failIfNoDocValues(); + return new DocValuesIndexFieldData.Builder(); + } + + @Override + public Object valueForDisplay(Object value) { + if (value == null) { + return null; + } + BytesRef binaryValue = (BytesRef) value; + return binaryValue.utf8ToString(); + } + } + + private List parentIDFields; + + protected ParentJoinFieldMapper(String simpleName, + MappedFieldType fieldType, + Settings indexSettings, + List parentIDFields) { + super(simpleName, fieldType, Defaults.FIELD_TYPE, indexSettings, MultiFields.empty(), null); + this.parentIDFields = parentIDFields; + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + @Override + protected ParentJoinFieldMapper clone() { + return (ParentJoinFieldMapper) super.clone(); + } + + @Override + public JoinFieldType fieldType() { + return (JoinFieldType) super.fieldType(); + } + + @Override + public Iterator iterator() { + return parentIDFields.stream().map((field) -> (Mapper) field).iterator(); + } + + /** + * Returns true if name is a parent name in the field. + */ + public boolean hasParent(String name) { + return parentIDFields.stream().anyMatch((mapper) -> name.equals(mapper.getParentName())); + } + + /** + * Returns true if name is a child name in the field. + */ + public boolean hasChild(String name) { + return parentIDFields.stream().anyMatch((mapper) -> mapper.getChildren().contains(name)); + } + + /** + * Returns the parent ID field mapper associated with a parent name + * if isParent is true and a child name otherwise. + */ + public ParentIDFieldMapper getParentIDFieldMapper(String name, boolean isParent) { + for (ParentIDFieldMapper mapper : parentIDFields) { + if (isParent && name.equals(mapper.getParentName())) { + return mapper; + } else if (isParent == false && mapper.getChildren().contains(name)) { + return mapper; + } + } + return null; + } + + @Override + protected void doMerge(Mapper mergeWith, boolean updateAllTypes) { + super.doMerge(mergeWith, updateAllTypes); + ParentJoinFieldMapper joinMergeWith = (ParentJoinFieldMapper) mergeWith; + List conflicts = new ArrayList<>(); + for (ParentIDFieldMapper mapper : parentIDFields) { + if (joinMergeWith.getParentIDFieldMapper(mapper.getParentName(), true) == null) { + conflicts.add("cannot remove parent [" + mapper.getParentName() + "] in join field [" + name() + "]"); + } + } + + final ArrayList newParentIDFields = new ArrayList<>(); + for (ParentIDFieldMapper mergeWithMapper : joinMergeWith.parentIDFields) { + ParentIDFieldMapper self = getParentIDFieldMapper(mergeWithMapper.getParentName(), true); + if (self == null) { + if (getParentIDFieldMapper(mergeWithMapper.getParentName(), false) != null) { + // it is forbidden to add a parent to an existing child + conflicts.add("cannot create parent [" + mergeWithMapper.getParentName() + "] from an existing child"); + } + for (String child : mergeWithMapper.getChildren()) { + if (getParentIDFieldMapper(child, true) != null) { + // it is forbidden to add a parent to an existing child + conflicts.add("cannot create child [" + child + "] from an existing parent"); + } + } + newParentIDFields.add(mergeWithMapper); + } else { + for (String child : self.getChildren()) { + if (mergeWithMapper.getChildren().contains(child) == false) { + conflicts.add("cannot remove child [" + child + "] in join field [" + name() + "]"); + } + } + ParentIDFieldMapper merged = (ParentIDFieldMapper) self.merge(mergeWithMapper, false); + newParentIDFields.add(merged); + } + } + if (conflicts.isEmpty() == false) { + throw new IllegalStateException("invalid update for join field [" + name() + "]:\n" + conflicts.toString()); + } + this.parentIDFields = Collections.unmodifiableList(newParentIDFields); + } + + @Override + public FieldMapper updateFieldType(Map fullNameToFieldType) { + ParentJoinFieldMapper fieldMapper = (ParentJoinFieldMapper) super.updateFieldType(fullNameToFieldType); + final List newMappers = new ArrayList<> (); + for (ParentIDFieldMapper mapper : fieldMapper.parentIDFields) { + newMappers.add((ParentIDFieldMapper) mapper.updateFieldType(fullNameToFieldType)); + } + fieldMapper.parentIDFields = Collections.unmodifiableList(newMappers); + return fieldMapper; + } + + @Override + protected void parseCreateField(ParseContext context, List fields) throws IOException { + throw new UnsupportedOperationException("parsing is implemented in parse(), this method should NEVER be called"); + } + + @Override + public Mapper parse(ParseContext context) throws IOException { + // Only one join field per document + checkDuplicateJoinFields(context.doc()); + + context.path().add(simpleName()); + XContentParser.Token token = context.parser().currentToken(); + String name = null; + String parent = null; + if (token == XContentParser.Token.START_OBJECT) { + String currentFieldName = null; + while ((token = context.parser().nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = context.parser().currentName(); + } else if (token == XContentParser.Token.VALUE_STRING) { + if ("name".equals(currentFieldName)) { + name = context.parser().text(); + } else if ("parent".equals(currentFieldName)) { + parent = context.parser().text(); + } else { + throw new IllegalArgumentException("unknown field name [" + currentFieldName + "] in join field [" + name() + "]"); + } + } + } + } else if (token == XContentParser.Token.VALUE_STRING) { + name = context.parser().text(); + parent = null; + } else { + throw new IllegalStateException("[" + name + "] expected START_OBJECT or VALUE_STRING but was: " + token); + } + + ParentIDFieldMapper parentIDField = getParentIDFieldMapper(name, true); + ParentIDFieldMapper childParentIDField = getParentIDFieldMapper(name, false); + if (parentIDField == null && childParentIDField == null) { + throw new IllegalArgumentException("unknown join name [" + name + "] for field [" + name() + "]"); + } + if (childParentIDField != null) { + // Index the document as a child + if (parent == null) { + throw new IllegalArgumentException("[parent] is missing for join field [" + name() + "]"); + } + if (context.sourceToParse().routing() == null) { + throw new IllegalArgumentException("[routing] is missing for join field [" + name() + "]"); + } + assert childParentIDField.getChildren().contains(name); + ParseContext externalContext = context.createExternalValueContext(parent); + childParentIDField.parse(externalContext); + } + if (parentIDField != null) { + // Index the document as a parent + assert parentIDField.getParentName().equals(name); + ParseContext externalContext = context.createExternalValueContext(context.sourceToParse().id()); + parentIDField.parse(externalContext); + } + + BytesRef binaryValue = new BytesRef(name); + Field field = new Field(fieldType().name(), binaryValue, fieldType()); + context.doc().add(field); + context.doc().add(new SortedDocValuesField(fieldType().name(), binaryValue)); + context.path().remove(); + return null; + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + builder.field("type", contentType()); + for (ParentIDFieldMapper field : parentIDFields) { + if (field.getChildren().size() == 1) { + builder.field(field.getParentName(), field.getChildren().iterator().next()); + } else { + builder.field(field.getParentName(), field.getChildren()); + } + } + } +} diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/fetch/ParentJoinFieldSubFetchPhaseTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/fetch/ParentJoinFieldSubFetchPhaseTests.java new file mode 100644 index 0000000000000..7eb2c8f3576cc --- /dev/null +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/fetch/ParentJoinFieldSubFetchPhaseTests.java @@ -0,0 +1,159 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.join.fetch; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.join.ParentJoinPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class ParentJoinFieldSubFetchPhaseTests extends ESSingleNodeTestCase { + @Override + protected Collection> getPlugins() { + return Collections.singletonList(ParentJoinPlugin.class); + } + + public void testSingleParentJoinField() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject() + .startObject("properties") + .startObject("join_field") + .field("type", "join") + .field("parent", "child") + .field("child", "grand_child") + .field("product", "item") + .endObject() + .endObject() + .endObject().string(); + IndexService service = createIndex("test", Settings.EMPTY); + service.mapperService().merge("doc", new CompressedXContent(mapping), + MapperService.MergeReason.MAPPING_UPDATE, true); + + // empty document + client().prepareIndex("test", "doc", "0") + .setSource().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + + // parent document + client().prepareIndex("test", "doc", "1") + .setSource("join_field", Collections.singletonMap("name", "parent")) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + + // child document + Map joinField = new HashMap<>(); + joinField.put("name", "child"); + joinField.put("parent", "1"); + client().prepareIndex("test", "doc", "2") + .setSource("join_field", joinField).setRouting("1") + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + + // grand_child document + joinField.clear(); + joinField.put("name", "grand_child"); + joinField.put("parent", "2"); + client().prepareIndex("test", "doc", "3") + .setSource("join_field", joinField).setRouting("2") + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + + // product document + client().prepareIndex("test", "doc", "4") + .setSource("join_field", Collections.singletonMap("name", "product")) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + + // item document + joinField.clear(); + joinField.put("name", "item"); + joinField.put("parent", "4"); + client().prepareIndex("test", "doc", "5") + .setSource("join_field", joinField).setRouting("4").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + + SearchResponse response = client().prepareSearch("test") + .setQuery(QueryBuilders.termQuery("join_field", "parent")) + .get(); + assertThat(response.getHits().totalHits, equalTo(1L)); + assertThat(response.getHits().getHits().length, equalTo(1)); + assertThat(response.getHits().getHits()[0].field("join_field").getValue(), equalTo("parent")); + assertNull(response.getHits().getHits()[0].field("join_field#parent")); + + response = client().prepareSearch("test") + .setQuery(QueryBuilders.termQuery("join_field", "child")) + .get(); + assertThat(response.getHits().totalHits, equalTo(1L)); + assertThat(response.getHits().getHits().length, equalTo(1)); + assertThat(response.getHits().getHits()[0].field("join_field").getValue(), equalTo("child")); + assertThat(response.getHits().getHits()[0].field("join_field#parent").getValue(), equalTo("1")); + assertNull(response.getHits().getHits()[0].field("join_field#child")); + + response = client().prepareSearch("test") + .setQuery(QueryBuilders.termQuery("join_field", "grand_child")) + .get(); + assertThat(response.getHits().totalHits, equalTo(1L)); + assertThat(response.getHits().getHits().length, equalTo(1)); + assertThat(response.getHits().getHits()[0].field("join_field").getValue(), equalTo("grand_child")); + assertThat(response.getHits().getHits()[0].field("join_field#child").getValue(), equalTo("2")); + + + response = client().prepareSearch("test") + .setQuery(QueryBuilders.termQuery("join_field", "product")) + .get(); + assertThat(response.getHits().totalHits, equalTo(1L)); + assertThat(response.getHits().getHits().length, equalTo(1)); + assertThat(response.getHits().getHits()[0].field("join_field").getValue(), equalTo("product")); + assertNull(response.getHits().getHits()[0].field("join_field#product")); + + response = client().prepareSearch("test") + .setQuery(QueryBuilders.termQuery("join_field", "item")) + .get(); + assertThat(response.getHits().totalHits, equalTo(1L)); + assertThat(response.getHits().getHits().length, equalTo(1)); + assertThat(response.getHits().getHits()[0].field("join_field").getValue(), equalTo("item")); + assertThat(response.getHits().getHits()[0].field("join_field#product").getValue(), equalTo("4")); + + response = client().prepareSearch("test") + .addSort(SortBuilders.fieldSort("join_field")) + .get(); + assertThat(response.getHits().totalHits, equalTo(6L)); + assertThat(response.getHits().getHits().length, equalTo(6)); + assertThat(response.getHits().getHits()[0].field("join_field").getValue(), equalTo("child")); + assertThat(response.getHits().getHits()[0].field("join_field#parent").getValue(), equalTo("1")); + assertNull(response.getHits().getHits()[0].field("join_field#child")); + assertThat(response.getHits().getHits()[1].field("join_field").getValue(), equalTo("grand_child")); + assertThat(response.getHits().getHits()[1].field("join_field#child").getValue(), equalTo("2")); + assertThat(response.getHits().getHits()[2].field("join_field").getValue(), equalTo("item")); + assertThat(response.getHits().getHits()[2].field("join_field#product").getValue(), equalTo("4")); + assertThat(response.getHits().getHits()[3].field("join_field").getValue(), equalTo("parent")); + assertNull(response.getHits().getHits()[3].field("join_field#parent")); + assertThat(response.getHits().getHits()[4].field("join_field").getValue(), equalTo("product")); + assertNull(response.getHits().getHits()[4].field("join_field#product")); + assertNull(response.getHits().getHits()[5].field("join_field")); + } +} diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java new file mode 100644 index 0000000000000..b0c5eb8e6809a --- /dev/null +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java @@ -0,0 +1,353 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.join.mapper; + +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MapperException; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.join.ParentJoinPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.util.Collection; +import java.util.Collections; + +import static org.hamcrest.Matchers.containsString; + +public class ParentJoinFieldMapperTests extends ESSingleNodeTestCase { + @Override + protected Collection> getPlugins() { + return Collections.singletonList(ParentJoinPlugin.class); + } + + public void testSingleLevel() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject() + .startObject("properties") + .startObject("join_field") + .field("type", "join") + .field("parent", "child") + .endObject() + .endObject() + .endObject().string(); + DocumentMapper docMapper = createIndex("test") + .mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + + // Doc without join + ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "0", + XContentFactory.jsonBuilder().startObject().endObject().bytes(), XContentType.JSON)); + assertNull(doc.rootDoc().getBinaryValue("join_field")); + + // Doc parent + doc = docMapper.parse(SourceToParse.source("test", "type", "1", + XContentFactory.jsonBuilder().startObject() + .field("join_field", "parent") + .endObject().bytes(), XContentType.JSON)); + assertEquals("1", doc.rootDoc().getBinaryValue("join_field#parent").utf8ToString()); + assertEquals("parent", doc.rootDoc().getBinaryValue("join_field").utf8ToString()); + + // Doc child + doc = docMapper.parse(SourceToParse.source("test", "type", "2", + XContentFactory.jsonBuilder().startObject() + .startObject("join_field") + .field("name", "child") + .field("parent", "1") + .endObject() + .endObject().bytes(), XContentType.JSON).routing("1")); + assertEquals("1", doc.rootDoc().getBinaryValue("join_field#parent").utf8ToString()); + assertEquals("child", doc.rootDoc().getBinaryValue("join_field").utf8ToString()); + + // Unkwnown join name + MapperException exc = expectThrows(MapperParsingException.class, + () -> docMapper.parse(SourceToParse.source("test", "type", "1", + XContentFactory.jsonBuilder().startObject() + .field("join_field", "unknown") + .endObject().bytes(), XContentType.JSON))); + assertThat(exc.getRootCause().getMessage(), containsString("unknown join name [unknown] for field [join_field]")); + } + + public void testMultipleLevels() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject() + .startObject("properties") + .startObject("join_field") + .field("type", "join") + .field("parent", "child") + .field("child", "grand_child") + .endObject() + .endObject() + .endObject().string(); + DocumentMapper docMapper = createIndex("test").mapperService() + .documentMapperParser().parse("type", new CompressedXContent(mapping)); + + // Doc without join + ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "0", + XContentFactory.jsonBuilder().startObject().endObject().bytes(), XContentType.JSON)); + assertNull(doc.rootDoc().getBinaryValue("join_field")); + + // Doc parent + doc = docMapper.parse(SourceToParse.source("test", "type", "1", + XContentFactory.jsonBuilder() + .startObject() + .field("join_field", "parent") + .endObject().bytes(), XContentType.JSON)); + assertEquals("1", doc.rootDoc().getBinaryValue("join_field#parent").utf8ToString()); + assertEquals("parent", doc.rootDoc().getBinaryValue("join_field").utf8ToString()); + + // Doc child + doc = docMapper.parse(SourceToParse.source("test", "type", "2", + XContentFactory.jsonBuilder().startObject() + .startObject("join_field") + .field("name", "child") + .field("parent", "1") + .endObject() + .endObject().bytes(), XContentType.JSON).routing("1")); + assertEquals("1", doc.rootDoc().getBinaryValue("join_field#parent").utf8ToString()); + assertEquals("2", doc.rootDoc().getBinaryValue("join_field#child").utf8ToString()); + assertEquals("child", doc.rootDoc().getBinaryValue("join_field").utf8ToString()); + + // Doc child missing parent + MapperException exc = expectThrows(MapperParsingException.class, + () -> docMapper.parse(SourceToParse.source("test", "type", "2", + XContentFactory.jsonBuilder().startObject() + .field("join_field", "child") + .endObject().bytes(), XContentType.JSON).routing("1"))); + assertThat(exc.getRootCause().getMessage(), containsString("[parent] is missing for join field [join_field]")); + + // Doc child missing routing + exc = expectThrows(MapperParsingException.class, + () -> docMapper.parse(SourceToParse.source("test", "type", "2", + XContentFactory.jsonBuilder().startObject() + .startObject("join_field") + .field("name", "child") + .field("parent", "1") + .endObject() + .endObject().bytes(), XContentType.JSON))); + assertThat(exc.getRootCause().getMessage(), containsString("[routing] is missing for join field [join_field]")); + + // Doc grand_child + doc = docMapper.parse(SourceToParse.source("test", "type", "3", + XContentFactory.jsonBuilder().startObject() + .startObject("join_field") + .field("name", "grand_child") + .field("parent", "2") + .endObject() + .endObject().bytes(), XContentType.JSON).routing("1")); + assertEquals("2", doc.rootDoc().getBinaryValue("join_field#child").utf8ToString()); + assertEquals("grand_child", doc.rootDoc().getBinaryValue("join_field").utf8ToString()); + + // Unkwnown join name + exc = expectThrows(MapperParsingException.class, + () -> docMapper.parse(SourceToParse.source("test", "type", "1", + XContentFactory.jsonBuilder().startObject() + .field("join_field", "unknown") + .endObject().bytes(), XContentType.JSON))); + assertThat(exc.getRootCause().getMessage(), containsString("unknown join name [unknown] for field [join_field]")); + } + + public void testUpdateRelations() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("properties") + .startObject("join_field") + .field("type", "join") + .field("parent", "child") + .array("child", "grand_child1", "grand_child2") + .endObject() + .endObject().endObject().string(); + IndexService indexService = createIndex("test"); + indexService.mapperService().merge("type", new CompressedXContent(mapping), + MapperService.MergeReason.MAPPING_UPDATE, false); + + { + final String updateMapping = XContentFactory.jsonBuilder().startObject().startObject("properties") + .startObject("join_field") + .field("type", "join") + .array("child", "grand_child1", "grand_child2") + .endObject() + .endObject().endObject().string(); + IllegalStateException exc = expectThrows(IllegalStateException.class, + () -> indexService.mapperService().merge("type", new CompressedXContent(updateMapping), + MapperService.MergeReason.MAPPING_UPDATE, false)); + assertThat(exc.getMessage(), containsString("cannot remove parent [parent] in join field [join_field]")); + } + + { + final String updateMapping = XContentFactory.jsonBuilder().startObject().startObject("properties") + .startObject("join_field") + .field("type", "join") + .field("parent", "child") + .field("child", "grand_child1") + .endObject() + .endObject().endObject().string(); + IllegalStateException exc = expectThrows(IllegalStateException.class, + () -> indexService.mapperService().merge("type", new CompressedXContent(updateMapping), + MapperService.MergeReason.MAPPING_UPDATE, false)); + assertThat(exc.getMessage(), containsString("cannot remove child [grand_child2] in join field [join_field]")); + } + + { + final String updateMapping = XContentFactory.jsonBuilder().startObject().startObject("properties") + .startObject("join_field") + .field("type", "join") + .field("uber_parent", "parent") + .field("parent", "child") + .array("child", "grand_child1", "grand_child2") + .endObject() + .endObject().endObject().string(); + IllegalStateException exc = expectThrows(IllegalStateException.class, + () -> indexService.mapperService().merge("type", new CompressedXContent(updateMapping), + MapperService.MergeReason.MAPPING_UPDATE, false)); + assertThat(exc.getMessage(), containsString("cannot create child [parent] from an existing parent")); + } + + { + final String updateMapping = XContentFactory.jsonBuilder().startObject().startObject("properties") + .startObject("join_field") + .field("type", "join") + .field("parent", "child") + .array("child", "grand_child1", "grand_child2") + .field("grand_child2", "grand_grand_child") + .endObject() + .endObject().endObject().string(); + IllegalStateException exc = expectThrows(IllegalStateException.class, + () -> indexService.mapperService().merge("type", new CompressedXContent(updateMapping), + MapperService.MergeReason.MAPPING_UPDATE, false)); + assertThat(exc.getMessage(), containsString("cannot create parent [grand_child2] from an existing child]")); + } + + { + final String updateMapping = XContentFactory.jsonBuilder().startObject().startObject("properties") + .startObject("join_field") + .field("type", "join") + .array("parent", "child", "child2") + .array("child", "grand_child1", "grand_child2") + .endObject() + .endObject().endObject().string(); + indexService.mapperService().merge("type", new CompressedXContent(updateMapping), + MapperService.MergeReason.MAPPING_UPDATE, true); + ParentJoinFieldMapper mapper = (ParentJoinFieldMapper) indexService.mapperService() + .docMappers(false).iterator().next().mappers().getMapper("join_field"); + assertTrue(mapper.hasChild("child2")); + assertFalse(mapper.hasParent("child2")); + assertTrue(mapper.hasChild("grand_child2")); + assertFalse(mapper.hasParent("grand_child2")); + } + + { + final String updateMapping = XContentFactory.jsonBuilder().startObject().startObject("properties") + .startObject("join_field") + .field("type", "join") + .array("parent", "child", "child2") + .array("child", "grand_child1", "grand_child2") + .array("other", "child_other1", "child_other2") + .endObject() + .endObject().endObject().string(); + indexService.mapperService().merge("type", new CompressedXContent(updateMapping), + MapperService.MergeReason.MAPPING_UPDATE, true); + ParentJoinFieldMapper mapper = (ParentJoinFieldMapper) indexService.mapperService() + .docMappers(false).iterator().next().mappers().getMapper("join_field"); + assertTrue(mapper.hasParent("other")); + assertFalse(mapper.hasChild("other")); + assertTrue(mapper.hasChild("child_other1")); + assertFalse(mapper.hasParent("child_other1")); + assertTrue(mapper.hasChild("child_other2")); + assertFalse(mapper.hasParent("child_other2")); + } + } + + public void testInvalidJoinFieldInsideObject() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("properties") + .startObject("object") + .startObject("properties") + .startObject("join_field") + .field("type", "join") + .field("parent", "child") + .endObject() + .endObject() + .endObject() + .endObject().endObject().string(); + IndexService indexService = createIndex("test"); + MapperParsingException exc = expectThrows(MapperParsingException.class, + () -> indexService.mapperService().merge("type", new CompressedXContent(mapping), + MapperService.MergeReason.MAPPING_UPDATE, false)); + assertThat(exc.getRootCause().getMessage(), + containsString("join field [object.join_field] cannot be added inside an object or in a multi-field")); + } + + public void testInvalidJoinFieldInsideMultiFields() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("properties") + .startObject("number") + .field("type", "integer") + .startObject("fields") + .startObject("join_field") + .field("type", "join") + .field("parent", "child") + .endObject() + .endObject() + .endObject() + .endObject().endObject().string(); + IndexService indexService = createIndex("test"); + MapperParsingException exc = expectThrows(MapperParsingException.class, + () -> indexService.mapperService().merge("type", new CompressedXContent(mapping), + MapperService.MergeReason.MAPPING_UPDATE, false)); + assertThat(exc.getRootCause().getMessage(), + containsString("join field [number.join_field] cannot be added inside an object or in a multi-field")); + } + + public void testMultipleJoinFields() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject() + .startObject("properties") + .startObject("join_field") + .field("type", "join") + .field("parent", "child") + .field("child", "grand_child") + .endObject() + .startObject("another_join_field") + .field("type", "join") + .field("product", "item") + .endObject() + .endObject() + .endObject().string(); + DocumentMapper docMapper = createIndex("test").mapperService() + .documentMapperParser().parse("type", new CompressedXContent(mapping)); + + // Doc without join + ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "0", + XContentFactory.jsonBuilder().startObject().endObject().bytes(), XContentType.JSON)); + assertNull(doc.rootDoc().getBinaryValue("join_field")); + + // Doc parent + MapperParsingException exc = expectThrows(MapperParsingException.class, + () -> docMapper.parse(SourceToParse.source("test", "type", "1", + XContentFactory.jsonBuilder() + .startObject() + .field("join_field", "parent") + .startObject("another_join_field") + .field("name", "item") + .field("parent", "0") + .endObject() + .endObject().bytes(), XContentType.JSON))); + assertThat(exc.getRootCause().getMessage(), containsString("cannot have two join fields in the same document")); + } +} diff --git a/modules/parent-join/src/test/resources/rest-api-spec/test/10_basic.yml b/modules/parent-join/src/test/resources/rest-api-spec/test/10_parent_child.yml similarity index 100% rename from modules/parent-join/src/test/resources/rest-api-spec/test/10_basic.yml rename to modules/parent-join/src/test/resources/rest-api-spec/test/10_parent_child.yml diff --git a/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml b/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml new file mode 100644 index 0000000000000..97af63ab00722 --- /dev/null +++ b/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml @@ -0,0 +1,65 @@ +setup: + - do: + indices.create: + index: test + body: + mappings: + doc: + properties: + join_field: { "type": "join", "parent": "child", "child": "grand_child" } + + - do: + index: + index: test + type: doc + id: 1 + body: { "join_field": { "name": "parent" } } + + - do: + index: + index: test + type: doc + id: 2 + routing: 1 + body: { "join_field": { "name": "child", "parent": "1" } } + + - do: + index: + index: test + type: doc + id: 3 + routing: 1 + body: { "join_field": { "name": "grand_child", "parent": "2" } } + + - do: + indices.refresh: {} + +--- +"Test basic": + - skip: + version: " - 5.99.99" + reason: parent-join was added in 6.0 + + - do: + search: + body: { sort: ["join_field"] } + + - match: { hits.total: 3 } + - match: { hits.hits.0._index: "test" } + - match: { hits.hits.0._type: "doc" } + - match: { hits.hits.0._id: "2" } + - match: { hits.hits.0.fields.join_field: ["child"] } + - match: { hits.hits.0.fields.join_field#parent: ["1"] } + - is_false: hits.hits.0.fields.join_field#child } + - match: { hits.hits.1._index: "test" } + - match: { hits.hits.1._type: "doc" } + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.1.fields.join_field: ["grand_child"] } + - match: { hits.hits.1.fields.join_field#child: ["2"] } + - match: { hits.hits.2._index: "test" } + - match: { hits.hits.2._type: "doc" } + - match: { hits.hits.2._id: "1" } + - match: { hits.hits.2.fields.join_field: ["parent"] } + - is_false: hits.hits.2.fields.join_field#parent + + From 3cb3a34e126aa9422bb316b7e7d8cd24feac167c Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Wed, 31 May 2017 10:57:19 +0200 Subject: [PATCH 2/3] Remove unrelated change --- .../aggregations/ParentToChildrenAggregator.java | 15 +++++---------- .../join/mapper/ParentIDFieldMapper.java | 2 +- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java index 600179d25931f..93ba1b98da15e 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java @@ -70,16 +70,11 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator { private final LongObjectPagedHashMap parentOrdToOtherBuckets; private boolean multipleBucketsPerParentOrd = false; - public ParentToChildrenAggregator(String name, - AggregatorFactories factories, - SearchContext context, - Aggregator parent, - Query childFilter, - Query parentFilter, - ValuesSource.Bytes.WithOrdinals valuesSource, - long maxOrd, - List pipelineAggregators, - Map metaData) throws IOException { + public ParentToChildrenAggregator(String name, AggregatorFactories factories, + SearchContext context, Aggregator parent, Query childFilter, + Query parentFilter, ValuesSource.Bytes.WithOrdinals valuesSource, + long maxOrd, List pipelineAggregators, Map metaData) + throws IOException { super(name, factories, context, parent, pipelineAggregators, metaData); // these two filters are cached in the parser this.childFilter = context.searcher().createNormalizedWeight(childFilter, false); diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIDFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIDFieldMapper.java index 7f6ec93aed33a..e4e9daed5fbec 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIDFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIDFieldMapper.java @@ -59,7 +59,7 @@ static class Defaults { } } - static class Builder extends FieldMapper.Builder { + static class Builder extends FieldMapper.Builder { private final String parent; private final Set children; From d03d9b50312ef1c51e22403d544695e121d25ec3 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Wed, 31 May 2017 16:01:35 +0200 Subject: [PATCH 3/3] after review --- .../fetch/ParentJoinFieldSubFetchPhase.java | 2 +- ...ldMapper.java => ParentIdFieldMapper.java} | 31 ++++--- .../join/mapper/ParentJoinFieldMapper.java | 89 ++++++++++--------- 3 files changed, 61 insertions(+), 61 deletions(-) rename modules/parent-join/src/main/java/org/elasticsearch/join/mapper/{ParentIDFieldMapper.java => ParentIdFieldMapper.java} (86%) diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/fetch/ParentJoinFieldSubFetchPhase.java b/modules/parent-join/src/main/java/org/elasticsearch/join/fetch/ParentJoinFieldSubFetchPhase.java index 8c0cc806c905d..9e9215c834f5a 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/fetch/ParentJoinFieldSubFetchPhase.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/fetch/ParentJoinFieldSubFetchPhase.java @@ -60,7 +60,7 @@ public void hitExecute(SearchContext context, HitContext hitContext) { ParentJoinFieldMapper joinFieldMapper = (ParentJoinFieldMapper) fieldMapper; joinField = new Tuple<>(fieldMapper.name(), joinName); // we retrieve the parent id only for children. - FieldMapper parentMapper = joinFieldMapper.getParentIDFieldMapper(joinName, false); + FieldMapper parentMapper = joinFieldMapper.getParentIdFieldMapper(joinName, false); if (parentMapper != null) { String parent = getSortedDocValue(parentMapper.name(), hitContext.reader(), hitContext.docId()); parentField = new Tuple<>(parentMapper.name(), parent); diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIDFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java similarity index 86% rename from modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIDFieldMapper.java rename to modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index e4e9daed5fbec..d2f747fdbaa4e 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIDFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -43,12 +43,11 @@ * A field mapper used internally by the {@link ParentJoinFieldMapper} to index * the value that link documents in the index (parent _id or _id if the document is a parent). */ -public final class ParentIDFieldMapper extends FieldMapper { - public static final String NAME = "parent"; - public static final String CONTENT_TYPE = "parent"; +public final class ParentIdFieldMapper extends FieldMapper { + static final String CONTENT_TYPE = "parent"; static class Defaults { - public static final MappedFieldType FIELD_TYPE = new ParentIDFieldType(); + public static final MappedFieldType FIELD_TYPE = new ParentIdFieldType(); static { FIELD_TYPE.setTokenized(false); @@ -59,7 +58,7 @@ static class Defaults { } } - static class Builder extends FieldMapper.Builder { + static class Builder extends FieldMapper.Builder { private final String parent; private final Set children; @@ -75,24 +74,24 @@ public Set getChildren() { } @Override - public ParentIDFieldMapper build(BuilderContext context) { + public ParentIdFieldMapper build(BuilderContext context) { fieldType.setName(name); - return new ParentIDFieldMapper(name, parent, children, fieldType, context.indexSettings()); + return new ParentIdFieldMapper(name, parent, children, fieldType, context.indexSettings()); } } - public static final class ParentIDFieldType extends StringFieldType { - public ParentIDFieldType() { + public static final class ParentIdFieldType extends StringFieldType { + public ParentIdFieldType() { setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); setSearchAnalyzer(Lucene.KEYWORD_ANALYZER); } - protected ParentIDFieldType(ParentIDFieldType ref) { + protected ParentIdFieldType(ParentIdFieldType ref) { super(ref); } - public ParentIDFieldType clone() { - return new ParentIDFieldType(this); + public ParentIdFieldType clone() { + return new ParentIdFieldType(this); } @Override @@ -119,7 +118,7 @@ public Object valueForDisplay(Object value) { private final String parentName; private Set children; - protected ParentIDFieldMapper(String simpleName, + protected ParentIdFieldMapper(String simpleName, String parentName, Set children, MappedFieldType fieldType, @@ -130,8 +129,8 @@ protected ParentIDFieldMapper(String simpleName, } @Override - protected ParentIDFieldMapper clone() { - return (ParentIDFieldMapper) super.clone(); + protected ParentIdFieldMapper clone() { + return (ParentIdFieldMapper) super.clone(); } /** @@ -164,7 +163,7 @@ protected void parseCreateField(ParseContext context, List field @Override protected void doMerge(Mapper mergeWith, boolean updateAllTypes) { super.doMerge(mergeWith, updateAllTypes); - ParentIDFieldMapper parentMergeWith = (ParentIDFieldMapper) mergeWith; + ParentIdFieldMapper parentMergeWith = (ParentIdFieldMapper) mergeWith; this.children = parentMergeWith.children; } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index 2811404bd1da4..3f258516a1204 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -51,6 +51,7 @@ /** * A {@link FieldMapper} that creates hierarchical joins (parent-join) between documents in the same index. + * TODO Should be restricted to a single join field per index */ public final class ParentJoinFieldMapper extends FieldMapper { public static final String NAME = "join"; @@ -68,7 +69,7 @@ public static class Defaults { } } - static String getParentIDFieldName(String joinFieldName, String parentName) { + static String getParentIdFieldName(String joinFieldName, String parentName) { return joinFieldName + "#" + parentName; } @@ -84,10 +85,10 @@ static void checkPreConditions(Version indexCreatedVersion, ContentPath path, St } } - static void checkParentFields(String name, List mappers) { + static void checkParentFields(String name, List mappers) { Set children = new HashSet<>(); List conflicts = new ArrayList<>(); - for (ParentIDFieldMapper mapper : mappers) { + for (ParentIdFieldMapper mapper : mappers) { for (String child : mapper.getChildren()) { if (children.add(child) == false) { conflicts.add("[" + child + "] cannot have multiple parents"); @@ -106,7 +107,7 @@ static void checkDuplicateJoinFields(ParseContext.Document doc) { } public static class Builder extends FieldMapper.Builder { - final List parentIDFieldBuilders = new ArrayList<>(); + final List parentIdFieldBuilders = new ArrayList<>(); public Builder(String name) { super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE); @@ -119,8 +120,8 @@ public JoinFieldType fieldType() { } public Builder addParent(String parent, Set children) { - String parentIDFieldName = getParentIDFieldName(name, parent); - parentIDFieldBuilders.add(new ParentIDFieldMapper.Builder(parentIDFieldName, parent, children)); + String parentIdFieldName = getParentIdFieldName(name, parent); + parentIdFieldBuilders.add(new ParentIdFieldMapper.Builder(parentIdFieldName, parent, children)); return this; } @@ -128,11 +129,11 @@ public Builder addParent(String parent, Set children) { public ParentJoinFieldMapper build(BuilderContext context) { checkPreConditions(context.indexCreatedVersion(), context.path(), name); fieldType.setName(name); - final List parentIDFields = new ArrayList<>(); - parentIDFieldBuilders.stream().map((e) -> e.build(context)).forEach(parentIDFields::add); - checkParentFields(name(), parentIDFields); + final List parentIdFields = new ArrayList<>(); + parentIdFieldBuilders.stream().map((e) -> e.build(context)).forEach(parentIdFields::add); + checkParentFields(name(), parentIdFields); return new ParentJoinFieldMapper(name, fieldType, context.indexSettings(), - Collections.unmodifiableList(parentIDFields)); + Collections.unmodifiableList(parentIdFields)); } } @@ -213,14 +214,14 @@ public Object valueForDisplay(Object value) { } } - private List parentIDFields; + private List parentIdFields; protected ParentJoinFieldMapper(String simpleName, MappedFieldType fieldType, Settings indexSettings, - List parentIDFields) { + List parentIdFields) { super(simpleName, fieldType, Defaults.FIELD_TYPE, indexSettings, MultiFields.empty(), null); - this.parentIDFields = parentIDFields; + this.parentIdFields = parentIdFields; } @Override @@ -240,29 +241,29 @@ public JoinFieldType fieldType() { @Override public Iterator iterator() { - return parentIDFields.stream().map((field) -> (Mapper) field).iterator(); + return parentIdFields.stream().map((field) -> (Mapper) field).iterator(); } /** * Returns true if name is a parent name in the field. */ public boolean hasParent(String name) { - return parentIDFields.stream().anyMatch((mapper) -> name.equals(mapper.getParentName())); + return parentIdFields.stream().anyMatch((mapper) -> name.equals(mapper.getParentName())); } /** * Returns true if name is a child name in the field. */ public boolean hasChild(String name) { - return parentIDFields.stream().anyMatch((mapper) -> mapper.getChildren().contains(name)); + return parentIdFields.stream().anyMatch((mapper) -> mapper.getChildren().contains(name)); } /** - * Returns the parent ID field mapper associated with a parent name + * Returns the parent Id field mapper associated with a parent name * if isParent is true and a child name otherwise. */ - public ParentIDFieldMapper getParentIDFieldMapper(String name, boolean isParent) { - for (ParentIDFieldMapper mapper : parentIDFields) { + public ParentIdFieldMapper getParentIdFieldMapper(String name, boolean isParent) { + for (ParentIdFieldMapper mapper : parentIdFields) { if (isParent && name.equals(mapper.getParentName())) { return mapper; } else if (isParent == false && mapper.getChildren().contains(name)) { @@ -277,51 +278,51 @@ protected void doMerge(Mapper mergeWith, boolean updateAllTypes) { super.doMerge(mergeWith, updateAllTypes); ParentJoinFieldMapper joinMergeWith = (ParentJoinFieldMapper) mergeWith; List conflicts = new ArrayList<>(); - for (ParentIDFieldMapper mapper : parentIDFields) { - if (joinMergeWith.getParentIDFieldMapper(mapper.getParentName(), true) == null) { + for (ParentIdFieldMapper mapper : parentIdFields) { + if (joinMergeWith.getParentIdFieldMapper(mapper.getParentName(), true) == null) { conflicts.add("cannot remove parent [" + mapper.getParentName() + "] in join field [" + name() + "]"); } } - final ArrayList newParentIDFields = new ArrayList<>(); - for (ParentIDFieldMapper mergeWithMapper : joinMergeWith.parentIDFields) { - ParentIDFieldMapper self = getParentIDFieldMapper(mergeWithMapper.getParentName(), true); + final List newParentIdFields = new ArrayList<>(); + for (ParentIdFieldMapper mergeWithMapper : joinMergeWith.parentIdFields) { + ParentIdFieldMapper self = getParentIdFieldMapper(mergeWithMapper.getParentName(), true); if (self == null) { - if (getParentIDFieldMapper(mergeWithMapper.getParentName(), false) != null) { + if (getParentIdFieldMapper(mergeWithMapper.getParentName(), false) != null) { // it is forbidden to add a parent to an existing child conflicts.add("cannot create parent [" + mergeWithMapper.getParentName() + "] from an existing child"); } for (String child : mergeWithMapper.getChildren()) { - if (getParentIDFieldMapper(child, true) != null) { + if (getParentIdFieldMapper(child, true) != null) { // it is forbidden to add a parent to an existing child conflicts.add("cannot create child [" + child + "] from an existing parent"); } } - newParentIDFields.add(mergeWithMapper); + newParentIdFields.add(mergeWithMapper); } else { for (String child : self.getChildren()) { if (mergeWithMapper.getChildren().contains(child) == false) { conflicts.add("cannot remove child [" + child + "] in join field [" + name() + "]"); } } - ParentIDFieldMapper merged = (ParentIDFieldMapper) self.merge(mergeWithMapper, false); - newParentIDFields.add(merged); + ParentIdFieldMapper merged = (ParentIdFieldMapper) self.merge(mergeWithMapper, false); + newParentIdFields.add(merged); } } if (conflicts.isEmpty() == false) { throw new IllegalStateException("invalid update for join field [" + name() + "]:\n" + conflicts.toString()); } - this.parentIDFields = Collections.unmodifiableList(newParentIDFields); + this.parentIdFields = Collections.unmodifiableList(newParentIdFields); } @Override public FieldMapper updateFieldType(Map fullNameToFieldType) { ParentJoinFieldMapper fieldMapper = (ParentJoinFieldMapper) super.updateFieldType(fullNameToFieldType); - final List newMappers = new ArrayList<> (); - for (ParentIDFieldMapper mapper : fieldMapper.parentIDFields) { - newMappers.add((ParentIDFieldMapper) mapper.updateFieldType(fullNameToFieldType)); + final List newMappers = new ArrayList<> (); + for (ParentIdFieldMapper mapper : fieldMapper.parentIdFields) { + newMappers.add((ParentIdFieldMapper) mapper.updateFieldType(fullNameToFieldType)); } - fieldMapper.parentIDFields = Collections.unmodifiableList(newMappers); + fieldMapper.parentIdFields = Collections.unmodifiableList(newMappers); return fieldMapper; } @@ -361,12 +362,12 @@ public Mapper parse(ParseContext context) throws IOException { throw new IllegalStateException("[" + name + "] expected START_OBJECT or VALUE_STRING but was: " + token); } - ParentIDFieldMapper parentIDField = getParentIDFieldMapper(name, true); - ParentIDFieldMapper childParentIDField = getParentIDFieldMapper(name, false); - if (parentIDField == null && childParentIDField == null) { + ParentIdFieldMapper parentIdField = getParentIdFieldMapper(name, true); + ParentIdFieldMapper childParentIdField = getParentIdFieldMapper(name, false); + if (parentIdField == null && childParentIdField == null) { throw new IllegalArgumentException("unknown join name [" + name + "] for field [" + name() + "]"); } - if (childParentIDField != null) { + if (childParentIdField != null) { // Index the document as a child if (parent == null) { throw new IllegalArgumentException("[parent] is missing for join field [" + name() + "]"); @@ -374,15 +375,15 @@ public Mapper parse(ParseContext context) throws IOException { if (context.sourceToParse().routing() == null) { throw new IllegalArgumentException("[routing] is missing for join field [" + name() + "]"); } - assert childParentIDField.getChildren().contains(name); + assert childParentIdField.getChildren().contains(name); ParseContext externalContext = context.createExternalValueContext(parent); - childParentIDField.parse(externalContext); + childParentIdField.parse(externalContext); } - if (parentIDField != null) { + if (parentIdField != null) { // Index the document as a parent - assert parentIDField.getParentName().equals(name); + assert parentIdField.getParentName().equals(name); ParseContext externalContext = context.createExternalValueContext(context.sourceToParse().id()); - parentIDField.parse(externalContext); + parentIdField.parse(externalContext); } BytesRef binaryValue = new BytesRef(name); @@ -396,7 +397,7 @@ public Mapper parse(ParseContext context) throws IOException { @Override protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { builder.field("type", contentType()); - for (ParentIDFieldMapper field : parentIDFields) { + for (ParentIdFieldMapper field : parentIdFields) { if (field.getChildren().size() == 1) { builder.field(field.getParentName(), field.getChildren().iterator().next()); } else {