Skip to content

Commit

Permalink
Added has_parent filter (elastic#2243)
Browse files Browse the repository at this point in the history
The `has_parent` filter accepts a query and a parent type. The query is executed in the parent document space, which is specified by the parent type. This filter return child documents which associated parents have matched. For the rest `has_parent` filter has the same options and works in the same manner as the `has_child` filter.

This is an experimental filter.

Filter example
###################
```
{
    "has_parent" : {
        "parent_type" : "blog"
        "query" : {
            "term" : {
                "tag" : "something"
            }
        }
    }
}
```
The `parent_type` field name can also be abbreviated to `type`.

Memory considerations
###############
With the current implementation, all _id values are loaded to memory (heap) in order to support fast lookups, so make sure there is enough mem for it.

This issue originates from issue elastic#792
  • Loading branch information
martijnvg authored and kimchy committed Sep 13, 2012
1 parent fa74b1d commit 922efb7
Show file tree
Hide file tree
Showing 11 changed files with 651 additions and 15 deletions.
28 changes: 28 additions & 0 deletions src/main/java/org/elasticsearch/common/CacheRecycler.java
Expand Up @@ -20,6 +20,7 @@
package org.elasticsearch.common;

import gnu.trove.map.hash.*;
import gnu.trove.set.hash.THashSet;
import org.elasticsearch.common.trove.ExtTDoubleObjectHashMap;
import org.elasticsearch.common.trove.ExtTHashMap;
import org.elasticsearch.common.trove.ExtTLongObjectHashMap;
Expand All @@ -33,6 +34,7 @@ public class CacheRecycler {

public static void clear() {
hashMap.clear();
hashSet.clear();
doubleObjectHashMap.clear();
longObjectHashMap.clear();
longLongHashMap.clear();
Expand Down Expand Up @@ -91,6 +93,32 @@ public static void pushHashMap(ExtTHashMap map) {
ref.add(map);
}

// ----- THashSet -----

private static SoftWrapper<Queue<THashSet>> hashSet = new SoftWrapper<Queue<THashSet>>();

public static <T> THashSet<T> popHashSet() {
Queue<THashSet> ref = hashSet.get();
if (ref == null) {
return new THashSet<T>();
}
THashSet set = ref.poll();
if (set == null) {
return new THashSet<T>();
}
return set;
}

public static void pushHashSet(THashSet map) {
Queue<THashSet> ref = hashSet.get();
if (ref == null) {
ref = ConcurrentCollections.newQueue();
hashSet.set(ref);
}
map.clear();
ref.add(map);
}

// ------ ExtTDoubleObjectHashMap -----

private static SoftWrapper<Queue<ExtTDoubleObjectHashMap>> doubleObjectHashMap = new SoftWrapper<Queue<ExtTDoubleObjectHashMap>>();
Expand Down
Expand Up @@ -26,7 +26,22 @@
*/
public interface IdReaderTypeCache {

/**
* @param docId The Lucene docId of the child document to return the parent _uid for.
* @return The parent _uid for the specified docId (which is a child document)
*/
HashedBytesArray parentIdByDoc(int docId);

int docById(HashedBytesArray id);
/**
* @param uid The uid of the document to return the lucene docId for
* @return The lucene docId for the specified uid
*/
int docById(HashedBytesArray uid);

/**
* @param docId The lucene docId of the document to return _uid for
* @return The _uid of the specified docId
*/
HashedBytesArray idByDoc(int docId);

}
Expand Up @@ -20,6 +20,7 @@
package org.elasticsearch.index.cache.id.simple;

import gnu.trove.impl.Constants;
import gnu.trove.map.hash.TIntObjectHashMap;
import org.apache.lucene.index.*;
import org.apache.lucene.util.StringHelper;
import org.elasticsearch.ElasticSearchException;
Expand Down Expand Up @@ -138,6 +139,7 @@ public void refresh(IndexReader[] readers) throws Exception {
// when traversing, make sure to ignore deleted docs, so the key->docId will be correct
if (!reader.isDeleted(termDocs.doc())) {
typeBuilder.idToDoc.put(idAsBytes, termDocs.doc());
typeBuilder.docToId[termDocs.doc()] = idAsBytes;
}
}
} while (termEnum.next());
Expand Down Expand Up @@ -205,6 +207,7 @@ public void refresh(IndexReader[] readers) throws Exception {
for (Map.Entry<String, TypeBuilder> typeBuilderEntry : entry.getValue().entrySet()) {
types.put(typeBuilderEntry.getKey(), new SimpleIdReaderTypeCache(typeBuilderEntry.getKey(),
typeBuilderEntry.getValue().idToDoc,
typeBuilderEntry.getValue().docToId,
typeBuilderEntry.getValue().parentIdsValues.toArray(new HashedBytesArray[typeBuilderEntry.getValue().parentIdsValues.size()]),
typeBuilderEntry.getValue().parentIdsOrdinals));
}
Expand Down Expand Up @@ -246,6 +249,7 @@ private boolean refreshNeeded(IndexReader[] readers) {

static class TypeBuilder {
final ExtTObjectIntHasMap<HashedBytesArray> idToDoc = new ExtTObjectIntHasMap<HashedBytesArray>(Constants.DEFAULT_CAPACITY, Constants.DEFAULT_LOAD_FACTOR, -1);
final HashedBytesArray[] docToId;
final ArrayList<HashedBytesArray> parentIdsValues = new ArrayList<HashedBytesArray>();
final int[] parentIdsOrdinals;
int t = 1; // current term number (0 indicated null value)
Expand All @@ -254,6 +258,7 @@ static class TypeBuilder {
parentIdsOrdinals = new int[reader.maxDoc()];
// the first one indicates null value
parentIdsValues.add(null);
docToId = new HashedBytesArray[reader.maxDoc()];
}

/**
Expand Down
Expand Up @@ -32,14 +32,17 @@ public class SimpleIdReaderTypeCache implements IdReaderTypeCache {

private final ExtTObjectIntHasMap<HashedBytesArray> idToDoc;

private final HashedBytesArray[] docIdToId;

private final HashedBytesArray[] parentIdsValues;

private final int[] parentIdsOrdinals;

public SimpleIdReaderTypeCache(String type, ExtTObjectIntHasMap<HashedBytesArray> idToDoc,
public SimpleIdReaderTypeCache(String type, ExtTObjectIntHasMap<HashedBytesArray> idToDoc, HashedBytesArray[] docIdToId,
HashedBytesArray[] parentIdsValues, int[] parentIdsOrdinals) {
this.type = type;
this.idToDoc = idToDoc;
this.docIdToId = docIdToId;
this.idToDoc.trimToSize();
this.parentIdsValues = parentIdsValues;
this.parentIdsOrdinals = parentIdsOrdinals;
Expand All @@ -53,8 +56,12 @@ public HashedBytesArray parentIdByDoc(int docId) {
return parentIdsValues[parentIdsOrdinals[docId]];
}

public int docById(HashedBytesArray id) {
return idToDoc.get(id);
public int docById(HashedBytesArray uid) {
return idToDoc.get(uid);
}

public HashedBytesArray idByDoc(int docId) {
return docIdToId[docId];
}

/**
Expand Down
Expand Up @@ -367,6 +367,10 @@ public static HasChildFilterBuilder hasChildFilter(String type, QueryBuilder que
return new HasChildFilterBuilder(type, query);
}

public static HasParentFilterBuilder hasParentFilter(String parentType, QueryBuilder query) {
return new HasParentFilterBuilder(parentType, query);
}

public static BoolFilterBuilder boolFilter() {
return new BoolFilterBuilder();
}
Expand Down
@@ -0,0 +1,84 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.index.query;

import org.elasticsearch.common.xcontent.XContentBuilder;

import java.io.IOException;

/**
* Builder for the 'has_parent' filter.
*/
public class HasParentFilterBuilder extends BaseFilterBuilder {

private final QueryBuilder queryBuilder;
private final String parentType;
private String scope;
private String filterName;
private String executionType;

/**
* @param parentType The parent type
* @param parentQuery The query that will be matched with parent documents
*/
public HasParentFilterBuilder(String parentType, QueryBuilder parentQuery) {
this.parentType = parentType;
this.queryBuilder = parentQuery;
}

public HasParentFilterBuilder scope(String scope) {
this.scope = scope;
return this;
}

public HasParentFilterBuilder filterName(String filterName) {
this.filterName = filterName;
return this;
}

/**
* Expert: Sets the low level parent to child filtering implementation. Can be: 'indirect' or 'uid'
*
* This option is experimental and will be removed.
*/
public HasParentFilterBuilder executionType(String executionType) {
this.executionType = executionType;
return this;
}

@Override
protected void doXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(HasParentFilterParser.NAME);
builder.field("query");
queryBuilder.toXContent(builder, params);
builder.field("parent_type", parentType);
if (scope != null) {
builder.field("_scope", scope);
}
if (filterName != null) {
builder.field("_name", filterName);
}
if (executionType != null) {
builder.field("execution_type", executionType);
}
builder.endObject();
}
}

126 changes: 126 additions & 0 deletions src/main/java/org/elasticsearch/index/query/HasParentFilterParser.java
@@ -0,0 +1,126 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.index.query;

import org.apache.lucene.search.Filter;
import org.apache.lucene.search.FilteredQuery;
import org.apache.lucene.search.Query;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
import org.elasticsearch.index.search.child.HasParentFilter;
import org.elasticsearch.search.internal.SearchContext;

import java.io.IOException;

/**
*
*/
public class HasParentFilterParser implements FilterParser {

public static final String NAME = "has_parent";

@Inject
public HasParentFilterParser() {
}

@Override
public String[] names() {
return new String[]{NAME, Strings.toCamelCase(NAME)};
}

@Override
public Filter parse(QueryParseContext parseContext) throws IOException, QueryParsingException {
XContentParser parser = parseContext.parser();

Query query = null;
boolean queryFound = false;
String parentType = null;
String executionType = "uid";
String scope = null;

String filterName = null;
String currentFieldName = null;
XContentParser.Token token;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if ("query".equals(currentFieldName)) {
// TODO handle `query` element before `type` element...
String[] origTypes = QueryParseContext.setTypesWithPrevious(parentType == null ? null : new String[]{parentType});
try {
query = parseContext.parseInnerQuery();
queryFound = true;
} finally {
QueryParseContext.setTypes(origTypes);
}
} else {
throw new QueryParsingException(parseContext.index(), "[has_parent] filter does not support [" + currentFieldName + "]");
}
} else if (token.isValue()) {
if ("type".equals(currentFieldName) || "parent_type".equals(currentFieldName) || "parentType".equals(currentFieldName)) {
parentType = parser.text();
} else if ("_scope".equals(currentFieldName)) {
scope = parser.text();
} else if ("_name".equals(currentFieldName)) {
filterName = parser.text();
// TODO: change to execution_type
} else if ("execution_type".equals(currentFieldName) || "executionType".equals(currentFieldName)) { // This option is experimental and will most likely be removed.
executionType = parser.text();
} else {
throw new QueryParsingException(parseContext.index(), "[has_parent] filter does not support [" + currentFieldName + "]");
}
}
}
if (!queryFound) {
throw new QueryParsingException(parseContext.index(), "[parent] filter requires 'query' field");
}
if (query == null) {
return null;
}

if (parentType == null) {
throw new QueryParsingException(parseContext.index(), "[parent] filter requires 'parent_type' field");
}

DocumentMapper parentDocMapper = parseContext.mapperService().documentMapper(parentType);
if (parentDocMapper == null) {
throw new QueryParsingException(parseContext.index(), "[parent] filter configured 'parent_type' [" + parentType + "] is not a valid type");
}

// wrap the query with type query
query = new FilteredQuery(query, parseContext.cacheFilter(parentDocMapper.typeFilter(), null));

SearchContext searchContext = SearchContext.current();

HasParentFilter parentFilter = HasParentFilter.create(executionType, query, scope, parentType, searchContext);
searchContext.addScopePhase(parentFilter);

if (filterName != null) {
parseContext.addNamedFilter(filterName, parentFilter);
}
return parentFilter;
}

}

0 comments on commit 922efb7

Please sign in to comment.