From d178bdcecff3d202394c9c8b64eac9191b61ea13 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 12 Sep 2012 14:22:37 +0200 Subject: [PATCH] Added `has_parent` filter (#2243) The `has_parent` filter accepts a query and a parent type. The query is executed in the parent document space, which is specified by the parent type. This filter return child documents which associated parents have matched. For the rest `has_parent` filter has the same options and works in the same manner as the `has_child` filter. This is an experimental filter. Filter example ################### ``` { "has_parent" : { "parent_type" : "blog" "query" : { "term" : { "tag" : "something" } } } } ``` The `parent_type` field name can also be abbreviated to `type`. Memory considerations ############### With the current implementation, all _id values are loaded to memory (heap) in order to support fast lookups, so make sure there is enough mem for it. This issue originates from issue #792 --- .../elasticsearch/common/CacheRecycler.java | 28 ++ .../index/cache/id/IdReaderTypeCache.java | 17 +- .../index/cache/id/simple/SimpleIdCache.java | 5 + .../id/simple/SimpleIdReaderTypeCache.java | 13 +- .../index/query/FilterBuilders.java | 4 + .../index/query/HasParentFilterBuilder.java | 84 ++++++ .../index/query/HasParentFilterParser.java | 126 +++++++++ .../index/search/child/HasParentFilter.java | 241 ++++++++++++++++++ .../indices/query/IndicesQueriesRegistry.java | 1 + .../search/child/ChildSearchBenchmark.java | 70 ++++- .../child/SimpleChildQuerySearchTests.java | 77 +++++- 11 files changed, 651 insertions(+), 15 deletions(-) create mode 100644 src/main/java/org/elasticsearch/index/query/HasParentFilterBuilder.java create mode 100644 src/main/java/org/elasticsearch/index/query/HasParentFilterParser.java create mode 100644 src/main/java/org/elasticsearch/index/search/child/HasParentFilter.java diff --git a/src/main/java/org/elasticsearch/common/CacheRecycler.java b/src/main/java/org/elasticsearch/common/CacheRecycler.java index 752bfc32b6db0..6de69c2ab9e08 100644 --- a/src/main/java/org/elasticsearch/common/CacheRecycler.java +++ b/src/main/java/org/elasticsearch/common/CacheRecycler.java @@ -20,6 +20,7 @@ package org.elasticsearch.common; import gnu.trove.map.hash.*; +import gnu.trove.set.hash.THashSet; import org.elasticsearch.common.trove.ExtTDoubleObjectHashMap; import org.elasticsearch.common.trove.ExtTHashMap; import org.elasticsearch.common.trove.ExtTLongObjectHashMap; @@ -33,6 +34,7 @@ public class CacheRecycler { public static void clear() { hashMap.clear(); + hashSet.clear(); doubleObjectHashMap.clear(); longObjectHashMap.clear(); longLongHashMap.clear(); @@ -91,6 +93,32 @@ public static void pushHashMap(ExtTHashMap map) { ref.add(map); } + // ----- THashSet ----- + + private static SoftWrapper> hashSet = new SoftWrapper>(); + + public static THashSet popHashSet() { + Queue ref = hashSet.get(); + if (ref == null) { + return new THashSet(); + } + THashSet set = ref.poll(); + if (set == null) { + return new THashSet(); + } + return set; + } + + public static void pushHashSet(THashSet map) { + Queue ref = hashSet.get(); + if (ref == null) { + ref = ConcurrentCollections.newQueue(); + hashSet.set(ref); + } + map.clear(); + ref.add(map); + } + // ------ ExtTDoubleObjectHashMap ----- private static SoftWrapper> doubleObjectHashMap = new SoftWrapper>(); diff --git a/src/main/java/org/elasticsearch/index/cache/id/IdReaderTypeCache.java b/src/main/java/org/elasticsearch/index/cache/id/IdReaderTypeCache.java index bd680a3a8e2e3..b94467b22c286 100644 --- a/src/main/java/org/elasticsearch/index/cache/id/IdReaderTypeCache.java +++ b/src/main/java/org/elasticsearch/index/cache/id/IdReaderTypeCache.java @@ -26,7 +26,22 @@ */ public interface IdReaderTypeCache { + /** + * @param docId The Lucene docId of the child document to return the parent _uid for. + * @return The parent _uid for the specified docId (which is a child document) + */ HashedBytesArray parentIdByDoc(int docId); - int docById(HashedBytesArray id); + /** + * @param uid The uid of the document to return the lucene docId for + * @return The lucene docId for the specified uid + */ + int docById(HashedBytesArray uid); + + /** + * @param docId The lucene docId of the document to return _uid for + * @return The _uid of the specified docId + */ + HashedBytesArray idByDoc(int docId); + } diff --git a/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdCache.java b/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdCache.java index 6e6925789ba22..6e6c802e94d69 100644 --- a/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdCache.java +++ b/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdCache.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.cache.id.simple; import gnu.trove.impl.Constants; +import gnu.trove.map.hash.TIntObjectHashMap; import org.apache.lucene.index.*; import org.apache.lucene.util.StringHelper; import org.elasticsearch.ElasticSearchException; @@ -138,6 +139,7 @@ public void refresh(IndexReader[] readers) throws Exception { // when traversing, make sure to ignore deleted docs, so the key->docId will be correct if (!reader.isDeleted(termDocs.doc())) { typeBuilder.idToDoc.put(idAsBytes, termDocs.doc()); + typeBuilder.docToId[termDocs.doc()] = idAsBytes; } } } while (termEnum.next()); @@ -205,6 +207,7 @@ public void refresh(IndexReader[] readers) throws Exception { for (Map.Entry typeBuilderEntry : entry.getValue().entrySet()) { types.put(typeBuilderEntry.getKey(), new SimpleIdReaderTypeCache(typeBuilderEntry.getKey(), typeBuilderEntry.getValue().idToDoc, + typeBuilderEntry.getValue().docToId, typeBuilderEntry.getValue().parentIdsValues.toArray(new HashedBytesArray[typeBuilderEntry.getValue().parentIdsValues.size()]), typeBuilderEntry.getValue().parentIdsOrdinals)); } @@ -246,6 +249,7 @@ private boolean refreshNeeded(IndexReader[] readers) { static class TypeBuilder { final ExtTObjectIntHasMap idToDoc = new ExtTObjectIntHasMap(Constants.DEFAULT_CAPACITY, Constants.DEFAULT_LOAD_FACTOR, -1); + final HashedBytesArray[] docToId; final ArrayList parentIdsValues = new ArrayList(); final int[] parentIdsOrdinals; int t = 1; // current term number (0 indicated null value) @@ -254,6 +258,7 @@ static class TypeBuilder { parentIdsOrdinals = new int[reader.maxDoc()]; // the first one indicates null value parentIdsValues.add(null); + docToId = new HashedBytesArray[reader.maxDoc()]; } /** diff --git a/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdReaderTypeCache.java b/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdReaderTypeCache.java index 6c41572cc70a1..1cbe8f069cead 100644 --- a/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdReaderTypeCache.java +++ b/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdReaderTypeCache.java @@ -32,14 +32,17 @@ public class SimpleIdReaderTypeCache implements IdReaderTypeCache { private final ExtTObjectIntHasMap idToDoc; + private final HashedBytesArray[] docIdToId; + private final HashedBytesArray[] parentIdsValues; private final int[] parentIdsOrdinals; - public SimpleIdReaderTypeCache(String type, ExtTObjectIntHasMap idToDoc, + public SimpleIdReaderTypeCache(String type, ExtTObjectIntHasMap idToDoc, HashedBytesArray[] docIdToId, HashedBytesArray[] parentIdsValues, int[] parentIdsOrdinals) { this.type = type; this.idToDoc = idToDoc; + this.docIdToId = docIdToId; this.idToDoc.trimToSize(); this.parentIdsValues = parentIdsValues; this.parentIdsOrdinals = parentIdsOrdinals; @@ -53,8 +56,12 @@ public HashedBytesArray parentIdByDoc(int docId) { return parentIdsValues[parentIdsOrdinals[docId]]; } - public int docById(HashedBytesArray id) { - return idToDoc.get(id); + public int docById(HashedBytesArray uid) { + return idToDoc.get(uid); + } + + public HashedBytesArray idByDoc(int docId) { + return docIdToId[docId]; } /** diff --git a/src/main/java/org/elasticsearch/index/query/FilterBuilders.java b/src/main/java/org/elasticsearch/index/query/FilterBuilders.java index 4e6e8f0f5ac1c..769b22a2f3be9 100644 --- a/src/main/java/org/elasticsearch/index/query/FilterBuilders.java +++ b/src/main/java/org/elasticsearch/index/query/FilterBuilders.java @@ -378,6 +378,10 @@ public static HasChildFilterBuilder hasChildFilter(String type, QueryBuilder que return new HasChildFilterBuilder(type, query); } + public static HasParentFilterBuilder hasParentFilter(String parentType, QueryBuilder query) { + return new HasParentFilterBuilder(parentType, query); + } + public static BoolFilterBuilder boolFilter() { return new BoolFilterBuilder(); } diff --git a/src/main/java/org/elasticsearch/index/query/HasParentFilterBuilder.java b/src/main/java/org/elasticsearch/index/query/HasParentFilterBuilder.java new file mode 100644 index 0000000000000..eebb452312a57 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/query/HasParentFilterBuilder.java @@ -0,0 +1,84 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.query; + +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * Builder for the 'has_parent' filter. + */ +public class HasParentFilterBuilder extends BaseFilterBuilder { + + private final QueryBuilder queryBuilder; + private final String parentType; + private String scope; + private String filterName; + private String executionType; + + /** + * @param parentType The parent type + * @param parentQuery The query that will be matched with parent documents + */ + public HasParentFilterBuilder(String parentType, QueryBuilder parentQuery) { + this.parentType = parentType; + this.queryBuilder = parentQuery; + } + + public HasParentFilterBuilder scope(String scope) { + this.scope = scope; + return this; + } + + public HasParentFilterBuilder filterName(String filterName) { + this.filterName = filterName; + return this; + } + + /** + * Expert: Sets the low level parent to child filtering implementation. Can be: 'indirect' or 'uid' + * + * This option is experimental and will be removed. + */ + public HasParentFilterBuilder executionType(String executionType) { + this.executionType = executionType; + return this; + } + + @Override + protected void doXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(HasParentFilterParser.NAME); + builder.field("query"); + queryBuilder.toXContent(builder, params); + builder.field("parent_type", parentType); + if (scope != null) { + builder.field("_scope", scope); + } + if (filterName != null) { + builder.field("_name", filterName); + } + if (executionType != null) { + builder.field("execution_type", executionType); + } + builder.endObject(); + } +} + diff --git a/src/main/java/org/elasticsearch/index/query/HasParentFilterParser.java b/src/main/java/org/elasticsearch/index/query/HasParentFilterParser.java new file mode 100644 index 0000000000000..3b4d1614b1cd0 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/query/HasParentFilterParser.java @@ -0,0 +1,126 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.query; + +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.FilteredQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.internal.ParentFieldMapper; +import org.elasticsearch.index.search.child.HasParentFilter; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; + +/** + * + */ +public class HasParentFilterParser implements FilterParser { + + public static final String NAME = "has_parent"; + + @Inject + public HasParentFilterParser() { + } + + @Override + public String[] names() { + return new String[]{NAME, Strings.toCamelCase(NAME)}; + } + + @Override + public Filter parse(QueryParseContext parseContext) throws IOException, QueryParsingException { + XContentParser parser = parseContext.parser(); + + Query query = null; + boolean queryFound = false; + String parentType = null; + String executionType = "uid"; + String scope = null; + + String filterName = null; + String currentFieldName = null; + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + if ("query".equals(currentFieldName)) { + // TODO handle `query` element before `type` element... + String[] origTypes = QueryParseContext.setTypesWithPrevious(parentType == null ? null : new String[]{parentType}); + try { + query = parseContext.parseInnerQuery(); + queryFound = true; + } finally { + QueryParseContext.setTypes(origTypes); + } + } else { + throw new QueryParsingException(parseContext.index(), "[has_parent] filter does not support [" + currentFieldName + "]"); + } + } else if (token.isValue()) { + if ("type".equals(currentFieldName) || "parent_type".equals(currentFieldName) || "parentType".equals(currentFieldName)) { + parentType = parser.text(); + } else if ("_scope".equals(currentFieldName)) { + scope = parser.text(); + } else if ("_name".equals(currentFieldName)) { + filterName = parser.text(); + // TODO: change to execution_type + } else if ("execution_type".equals(currentFieldName) || "executionType".equals(currentFieldName)) { // This option is experimental and will most likely be removed. + executionType = parser.text(); + } else { + throw new QueryParsingException(parseContext.index(), "[has_parent] filter does not support [" + currentFieldName + "]"); + } + } + } + if (!queryFound) { + throw new QueryParsingException(parseContext.index(), "[parent] filter requires 'query' field"); + } + if (query == null) { + return null; + } + + if (parentType == null) { + throw new QueryParsingException(parseContext.index(), "[parent] filter requires 'parent_type' field"); + } + + DocumentMapper parentDocMapper = parseContext.mapperService().documentMapper(parentType); + if (parentDocMapper == null) { + throw new QueryParsingException(parseContext.index(), "[parent] filter configured 'parent_type' [" + parentType + "] is not a valid type"); + } + + // wrap the query with type query + query = new FilteredQuery(query, parseContext.cacheFilter(parentDocMapper.typeFilter(), null)); + + SearchContext searchContext = SearchContext.current(); + + HasParentFilter parentFilter = HasParentFilter.create(executionType, query, scope, parentType, searchContext); + searchContext.addScopePhase(parentFilter); + + if (filterName != null) { + parseContext.addNamedFilter(filterName, parentFilter); + } + return parentFilter; + } + +} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/index/search/child/HasParentFilter.java b/src/main/java/org/elasticsearch/index/search/child/HasParentFilter.java new file mode 100644 index 0000000000000..7ea61fb67b032 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/search/child/HasParentFilter.java @@ -0,0 +1,241 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.search.child; + +import gnu.trove.set.hash.THashSet; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.*; +import org.apache.lucene.util.FixedBitSet; +import org.elasticsearch.ElasticSearchIllegalStateException; +import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.bytes.HashedBytesArray; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.lucene.docset.GetDocSet; +import org.elasticsearch.common.lucene.search.NoopCollector; +import org.elasticsearch.common.trove.ExtTHashMap; +import org.elasticsearch.index.cache.id.IdReaderTypeCache; +import org.elasticsearch.search.internal.ScopePhase; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.Map; + +import static com.google.common.collect.Maps.newHashMap; + +/** + * A filter that only return child documents that are linked to the parent documents that matched with the inner query. + */ +public abstract class HasParentFilter extends Filter implements ScopePhase.CollectorPhase { + + final Query query; + final String scope; + final String parentType; + final SearchContext context; + + HasParentFilter(Query query, String scope, String parentType, SearchContext context) { + this.query = query; + this.scope = scope; + this.parentType = parentType; + this.context = context; + } + + public String scope() { + return scope; + } + + public Query query() { + return query; + } + + public static HasParentFilter create(String executionType, Query query, String scope, String parentType, SearchContext context) { + // This mechanism is experimental and will most likely be removed. + if ("indirect".equals(executionType)) { + return new InDirect(query, scope, parentType, context); + } else if ("uid".equals(executionType)) { + return new UidParentFilter(query, scope, parentType, context); + } + throw new ElasticSearchIllegalStateException("Illegal has_parent execution type: " + executionType); + } + + static class UidParentFilter extends HasParentFilter { + + THashSet parents; + + UidParentFilter(Query query, String scope, String parentType, SearchContext context) { + super(query, scope, parentType, context); + } + + public boolean requiresProcessing() { + return parents == null; + } + + public Collector collector() { + parents = CacheRecycler.popHashSet(); + return new ParentUidsCollector(parents, context, parentType); + } + + public void processCollector(Collector collector) { + parents = ((ParentUidsCollector) collector).collectedUids; + } + + public DocIdSet getDocIdSet(IndexReader reader) throws IOException { + IdReaderTypeCache idReaderTypeCache = context.idCache().reader(reader).type(parentType); + return new ChildrenDocSet(reader, parents, idReaderTypeCache); + } + + public void clear() { + CacheRecycler.pushHashSet(parents); + parents = null; + } + + static class ChildrenDocSet extends GetDocSet { + + final IndexReader reader; + final THashSet parents; + final IdReaderTypeCache idReaderTypeCache; + + ChildrenDocSet(IndexReader reader, THashSet parents, IdReaderTypeCache idReaderTypeCache) { + super(reader.maxDoc()); + this.reader = reader; + this.parents = parents; + this.idReaderTypeCache = idReaderTypeCache; + } + + public boolean get(int doc) { + return !reader.isDeleted(doc) && parents.contains(idReaderTypeCache.parentIdByDoc(doc)); + } + + } + + static class ParentUidsCollector extends NoopCollector { + + final THashSet collectedUids; + final SearchContext context; + final String parentType; + + IdReaderTypeCache typeCache; + + ParentUidsCollector(THashSet collectedUids, SearchContext context, String parentType) { + this.collectedUids = collectedUids; + this.context = context; + this.parentType = parentType; + } + + public void collect(int doc) throws IOException { + collectedUids.add(typeCache.idByDoc(doc)); + } + + public void setNextReader(IndexReader reader, int docBase) throws IOException { + typeCache = context.idCache().reader(reader).type(parentType); + } + } + + } + + static class InDirect extends HasParentFilter { + + Map parentDocs; + + InDirect(Query query, String scope, String parentType, SearchContext context) { + super(query, scope, parentType, context); + } + + public boolean requiresProcessing() { + return parentDocs == null; + } + + public Collector collector() { + return new ParentDocsCollector(); + } + + public void processCollector(Collector collector) { + parentDocs = ((ParentDocsCollector) collector).segmentResults; + } + + public DocIdSet getDocIdSet(IndexReader reader) throws IOException { + return new ChildrenDocSet(reader, parentDocs, context, parentType); + } + + public void clear() { + parentDocs = null; + } + + static class ChildrenDocSet extends GetDocSet { + + final IdReaderTypeCache currentTypeCache; + final IndexReader currentReader; + final Tuple[] readersToTypeCache; + final Map parentDocs; + + ChildrenDocSet(IndexReader currentReader, Map parentDocs, + SearchContext context, String parentType) { + super(currentReader.maxDoc()); + this.currentTypeCache = context.idCache().reader(currentReader).type(parentType); + this.currentReader = currentReader; + this.parentDocs = parentDocs; + this.readersToTypeCache = new Tuple[context.searcher().subReaders().length]; + for (int i = 0; i < readersToTypeCache.length; i++) { + IndexReader reader = context.searcher().subReaders()[i]; + readersToTypeCache[i] = new Tuple(reader, context.idCache().reader(reader).type(parentType)); + } + } + + public boolean get(int doc) { + if (currentReader.isDeleted(doc) || doc == -1) { + return false; + } + + HashedBytesArray parentId = currentTypeCache.parentIdByDoc(doc); + if (parentId == null) { + return false; + } + + for (Tuple readerTypeCacheTuple : readersToTypeCache) { + int parentDocId = readerTypeCacheTuple.v2().docById(parentId); + if (parentDocId == -1) { + continue; + } + + FixedBitSet currentParentDocs = parentDocs.get(readerTypeCacheTuple.v1().getCoreCacheKey()); + if (currentParentDocs.get(parentDocId)) { + return true; + } + } + return false; + } + } + + static class ParentDocsCollector extends NoopCollector { + + final Map segmentResults = newHashMap(); + FixedBitSet current; + + public void collect(int doc) throws IOException { + current.set(doc); + } + + public void setNextReader(IndexReader reader, int docBase) throws IOException { + segmentResults.put(reader.getCoreCacheKey(), current = new FixedBitSet(reader.maxDoc())); + } + } + } + +} + diff --git a/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java b/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java index 93c7013e3db34..91483d33d0caa 100644 --- a/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java +++ b/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java @@ -82,6 +82,7 @@ public IndicesQueriesRegistry(Settings settings, @Nullable ClusterService cluste Map filterParsers = Maps.newHashMap(); addFilterParser(filterParsers, new HasChildFilterParser()); + addFilterParser(filterParsers, new HasParentFilterParser()); addFilterParser(filterParsers, new NestedFilterParser()); addFilterParser(filterParsers, new TypeFilterParser()); addFilterParser(filterParsers, new IdsFilterParser()); diff --git a/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchBenchmark.java b/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchBenchmark.java index 864a680bfa7be..094e7ece33e42 100644 --- a/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchBenchmark.java +++ b/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchBenchmark.java @@ -30,6 +30,8 @@ import org.elasticsearch.common.unit.SizeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.query.FilterBuilders; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.node.Node; import java.io.IOException; @@ -40,6 +42,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.query.FilterBuilders.hasParentFilter; import static org.elasticsearch.index.query.QueryBuilders.*; import static org.elasticsearch.node.NodeBuilder.nodeBuilder; @@ -52,16 +55,12 @@ public static void main(String[] args) throws Exception { Settings settings = settingsBuilder() .put("index.engine.robin.refreshInterval", "-1") .put("gateway.type", "local") - .put(SETTING_NUMBER_OF_SHARDS, 2) - .put(SETTING_NUMBER_OF_REPLICAS, 1) + .put(SETTING_NUMBER_OF_SHARDS, 1) + .put(SETTING_NUMBER_OF_REPLICAS, 0) .build(); Node node1 = nodeBuilder().settings(settingsBuilder().put(settings).put("name", "node1")).node(); - Node node2 = nodeBuilder().settings(settingsBuilder().put(settings).put("name", "node2")).node(); - - Node clientNode = nodeBuilder().settings(settingsBuilder().put(settings).put("name", "client")).client(true).node(); - - Client client = clientNode.client(); + Client client = node1.client(); long COUNT = SizeValue.parseSizeValue("1m").singles(); int CHILD_COUNT = 5; @@ -162,6 +161,59 @@ public static void main(String[] args) throws Exception { } System.out.println("--> has_child Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms"); + String[] executionTypes = new String[]{"uid", "indirect"}; // either uid (faster, in general a bit more memory) or indirect (slower, but in general a bit less memory) + for (String executionType : executionTypes) { + System.out.println("--> Running has_parent filter with " + executionType + " execution type"); + // run parent child constant query + for (int j = 0; j < QUERY_WARMUP; j++) { + SearchResponse searchResponse = client.prepareSearch() + .setQuery(constantScoreQuery( + hasParentFilter("parent", termQuery("name", "test1")).executionType(executionType) + )) + .execute().actionGet(); + if (searchResponse.failedShards() > 0) { + System.err.println("Search Failures " + Arrays.toString(searchResponse.shardFailures())); + } + if (searchResponse.hits().totalHits() != CHILD_COUNT) { + System.err.println("--> mismatch on hits [" + j + "], got [" + searchResponse.hits().totalHits() + "], expected [" + CHILD_COUNT + "]"); + } + } + + totalQueryTime = 0; + for (int j = 1; j <= QUERY_COUNT; j++) { + SearchResponse searchResponse = client.prepareSearch() + .setQuery(constantScoreQuery( + hasParentFilter("parent", termQuery("name", "test1")).executionType(executionType) + )) + .execute().actionGet(); + if (searchResponse.failedShards() > 0) { + System.err.println("Search Failures " + Arrays.toString(searchResponse.shardFailures())); + } + if (searchResponse.hits().totalHits() != CHILD_COUNT) { + System.err.println("--> mismatch on hits [" + j + "], got [" + searchResponse.hits().totalHits() + "], expected [" + CHILD_COUNT + "]"); + } + totalQueryTime += searchResponse.tookInMillis(); + } + System.out.println("--> has_parent[" + executionType + "] Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms"); + + System.out.println("--> Running has_parent[" + executionType + "] filter with match_all query as parent query"); + totalQueryTime = 0; + for (int j = 1; j <= QUERY_COUNT; j++) { + SearchResponse searchResponse = client.prepareSearch() + .setQuery(constantScoreQuery( + hasParentFilter("parent", matchAllQuery()).executionType(executionType) + )) + .execute().actionGet(); + if (searchResponse.failedShards() > 0) { + System.err.println("Search Failures " + Arrays.toString(searchResponse.shardFailures())); + } + if (searchResponse.hits().totalHits() != 5000000) { + System.err.println("--> mismatch on hits [" + j + "], got [" + searchResponse.hits().totalHits() + "], expected [" + 5000000 + "]"); + } + totalQueryTime += searchResponse.tookInMillis(); + } + System.out.println("--> has_parent[" + executionType + "] with match_all query as parent query Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms"); + } System.out.println("--> Running top_children query"); // run parent child score query for (int j = 0; j < QUERY_WARMUP; j++) { @@ -183,10 +235,8 @@ public static void main(String[] args) throws Exception { } System.out.println("--> top_children Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms"); - clientNode.close(); - + client.close(); node1.close(); - node2.close(); } private static XContentBuilder parentSource(String id, String nameValue) throws IOException { diff --git a/src/test/java/org/elasticsearch/test/integration/search/child/SimpleChildQuerySearchTests.java b/src/test/java/org/elasticsearch/test/integration/search/child/SimpleChildQuerySearchTests.java index eebcf0908d0ac..9f5384bc68241 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/child/SimpleChildQuerySearchTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/child/SimpleChildQuerySearchTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.test.integration.search.child; +import com.beust.jcommander.internal.Maps; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.search.ShardSearchFailure; @@ -31,9 +32,12 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.util.Arrays; +import java.util.*; +import static com.google.common.collect.Lists.newArrayList; +import static com.google.common.collect.Maps.newHashMap; import static org.elasticsearch.index.query.FilterBuilders.hasChildFilter; +import static org.elasticsearch.index.query.FilterBuilders.hasParentFilter; import static org.elasticsearch.index.query.QueryBuilders.*; import static org.elasticsearch.search.facet.FacetBuilders.termsFacet; import static org.hamcrest.MatcherAssert.assertThat; @@ -237,6 +241,77 @@ public void simpleChildQuery() throws Exception { assertThat(searchResponse.hits().totalHits(), equalTo(2l)); assertThat(searchResponse.hits().getAt(0).id(), anyOf(equalTo("p2"), equalTo("p1"))); assertThat(searchResponse.hits().getAt(1).id(), anyOf(equalTo("p2"), equalTo("p1"))); + + // HAS PARENT FILTER + searchResponse = client.prepareSearch("test").setQuery(constantScoreQuery(hasParentFilter("parent", termQuery("p_field", "p_value2")))).execute().actionGet(); + assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0)); + assertThat(searchResponse.failedShards(), equalTo(0)); + assertThat(searchResponse.hits().totalHits(), equalTo(2l)); + assertThat(searchResponse.hits().getAt(0).id(), equalTo("c3")); + assertThat(searchResponse.hits().getAt(1).id(), equalTo("c4")); + + searchResponse = client.prepareSearch("test").setQuery(constantScoreQuery(hasParentFilter("parent", termQuery("p_field", "p_value1")))).execute().actionGet(); + assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0)); + assertThat(searchResponse.failedShards(), equalTo(0)); + assertThat(searchResponse.hits().totalHits(), equalTo(2l)); + assertThat(searchResponse.hits().getAt(0).id(), equalTo("c1")); + assertThat(searchResponse.hits().getAt(1).id(), equalTo("c2")); + } + + @Test + public void testHasParentFilter() throws Exception { + client.admin().indices().prepareDelete().execute().actionGet(); + client.admin().indices().prepareCreate("test").execute().actionGet(); + client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); + client.admin().indices().preparePutMapping("test").setType("child").setSource(XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_parent").field("type", "parent").endObject() + .endObject().endObject()).execute().actionGet(); + + Map> parentToChildren = newHashMap(); + // Childless parent + client.prepareIndex("test", "parent", "p0").setSource("p_field", "p0").execute().actionGet(); + parentToChildren.put("p0", new ArrayList()); + + String previousParentId = null; + int numChildDocs = 32; + int numChildDocsPerParent = 0; + for (int i = 1; i <= numChildDocs; i++) { + if (previousParentId == null || i % numChildDocsPerParent == 0) { + previousParentId = "p" + i; + client.prepareIndex("test", "parent", previousParentId).setSource("p_field", previousParentId).execute().actionGet(); + client.admin().indices().prepareFlush("test").execute().actionGet(); + numChildDocsPerParent++; + } + + String childId = "c" + i; + client.prepareIndex("test", "child", childId) + .setSource("c_field", childId) + .setParent(previousParentId) + .execute().actionGet(); + + if (!parentToChildren.containsKey(previousParentId)) { + parentToChildren.put(previousParentId, new ArrayList()); + } + parentToChildren.get(previousParentId).add(childId); + } + client.admin().indices().prepareRefresh().execute().actionGet(); + + assertThat(parentToChildren.isEmpty(), equalTo(false)); + for (Map.Entry> parentToChildrenEntry : parentToChildren.entrySet()) { + SearchResponse searchResponse = client.prepareSearch("test") + .setQuery(constantScoreQuery(hasParentFilter("parent", termQuery("p_field", parentToChildrenEntry.getKey())))) + .setSize(numChildDocsPerParent) + .execute().actionGet(); + + assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0)); + assertThat(searchResponse.failedShards(), equalTo(0)); + List childIds = parentToChildrenEntry.getValue(); + assertThat(searchResponse.hits().totalHits(), equalTo((long) childIds.size())); + int counter = 0; + for (String childId : childIds) { + assertThat(searchResponse.hits().getAt(counter++).id(), equalTo(childId)); + } + } } @Test