Skip to content

Commit

Permalink
Terms Filter Lookup: When on cache key defined, use terms values as k…
Browse files Browse the repository at this point in the history 
…ey to filter cache

closes #3240
  • Loading branch information
@kimchy
kimchy committed Jun 26, 2013
1 parent ed0c51d commit 7687217
Show file tree
Hide file tree
Showing 4 changed files with 21 additions and 83 deletions.
4 changes: 2 additions & 2 deletions src/main/java/org/elasticsearch/index/cache/filter/support/CacheKeyFilter.java
View file
Expand Up @@ -49,7 +49,7 @@ public Key(String str) {
public byte[] bytes() {
return this.bytes;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
Expand Down Expand Up @@ -107,5 +107,5 @@ public String toString() {
}
}

Key cacheKey();
Object cacheKey();
}
2 changes: 1 addition & 1 deletion src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
View file
Expand Up @@ -251,7 +251,7 @@ public Filter termFilter(Object value, @Nullable QueryParseContext context) {
@Override
public Filter termsFilter(List values, @Nullable QueryParseContext context) {
if (context == null) {
return super.termFilter(values, context);
return super.termsFilter(values, context);
}
List<BytesRef> bValues = new ArrayList<BytesRef>(values.size());
for (Object value : values) {
Expand Down
9 changes: 4 additions & 5 deletions src/main/java/org/elasticsearch/index/query/TermsFilterParser.java
View file
Expand Up @@ -168,16 +168,15 @@ public Filter parse(QueryParseContext parseContext) throws IOException, QueryPar

// external lookup, use it
TermsLookup termsLookup = new TermsLookup(fieldMapper, lookupIndex, lookupType, lookupId, lookupRouting, lookupPath, parseContext);
if (cacheKey == null) {
cacheKey = new CacheKeyFilter.Key(termsLookup.toString());
}
Filter filter = termsFilterCache.lookupTermsFilter(cacheKey, termsLookup);

Filter filter = termsFilterCache.termsFilter(termsLookup, cacheKey);
if (filter == null) {
return null;
}

// cache the whole filter by default, or if explicitly told to
if (cache == null || cache) {
filter = parseContext.cacheFilter(filter, null); // cacheKey is passed as null, so we don't double cache the key
filter = parseContext.cacheFilter(filter, cacheKey);
}
return filter;
}
Expand Down
89 changes: 14 additions & 75 deletions src/main/java/org/elasticsearch/indices/cache/filter/terms/IndicesTermsFilterCache.java
View file
Expand Up @@ -22,17 +22,13 @@
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.Weigher;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
Expand All @@ -42,7 +38,6 @@
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.cache.filter.support.CacheKeyFilter;

import java.io.IOException;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
Expand All @@ -56,7 +51,7 @@ public class IndicesTermsFilterCache extends AbstractComponent {

private final Client client;

private final Cache<CacheKeyFilter.Key, TermsFilterValue> cache;
private final Cache<BytesRef, TermsFilterValue> cache;

@Inject
public IndicesTermsFilterCache(Settings settings, Client client) {
Expand All @@ -67,7 +62,7 @@ public IndicesTermsFilterCache(Settings settings, Client client) {
TimeValue expireAfterWrite = componentSettings.getAsTime("expire_after_write", null);
TimeValue expireAfterAccess = componentSettings.getAsTime("expire_after_access", null);

CacheBuilder<CacheKeyFilter.Key, TermsFilterValue> builder = CacheBuilder.newBuilder()
CacheBuilder<BytesRef, TermsFilterValue> builder = CacheBuilder.newBuilder()
.maximumWeight(size.bytes())
.weigher(new TermsFilterValueWeigher());

Expand All @@ -81,18 +76,16 @@ public IndicesTermsFilterCache(Settings settings, Client client) {
this.cache = builder.build();
}

/**
* An external lookup terms filter. Note, already implements the {@link CacheKeyFilter} so no need
* to double cache key it.
*/
public Filter lookupTermsFilter(final CacheKeyFilter.Key cacheKey, final TermsLookup lookup) {
return new LookupTermsFilter(lookup, cacheKey, this);
}

@Nullable
private Filter termsFilter(final CacheKeyFilter.Key cacheKey, final TermsLookup lookup) throws RuntimeException {
public Filter termsFilter(final TermsLookup lookup, @Nullable CacheKeyFilter.Key cacheKey) throws RuntimeException {
BytesRef key;
if (cacheKey != null) {
key = new BytesRef(cacheKey.bytes());
} else {
key = new BytesRef(lookup.toString());
}
try {
return cache.get(cacheKey, new Callable<TermsFilterValue>() {
return cache.get(key, new Callable<TermsFilterValue>() {
@Override
public TermsFilterValue call() throws Exception {
GetResponse getResponse = client.get(new GetRequest(lookup.getIndex(), lookup.getType(), lookup.getId()).preference("_local").routing(lookup.getRouting())).actionGet();
Expand Down Expand Up @@ -134,70 +127,16 @@ public void clear(String reason) {
}

public void clear(String reason, String[] keys) {
final BytesRef spare = new BytesRef();
for (String key : keys) {
cache.invalidate(new CacheKeyFilter.Key(Strings.toUTF8Bytes(key, spare)));
}
}

static class LookupTermsFilter extends Filter implements CacheKeyFilter {

private final TermsLookup lookup;
private final CacheKeyFilter.Key cacheKey;
private final IndicesTermsFilterCache cache;
boolean termsFilterCalled;
private Filter termsFilter;

LookupTermsFilter(TermsLookup lookup, CacheKeyFilter.Key cacheKey, IndicesTermsFilterCache cache) {
this.lookup = lookup;
this.cacheKey = cacheKey;
this.cache = cache;
}

@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
// only call the terms filter once per execution (across segments per single search request)
if (!termsFilterCalled) {
termsFilterCalled = true;
termsFilter = cache.termsFilter(cacheKey, lookup);
}
if (termsFilter == null) return null;
return termsFilter.getDocIdSet(context, acceptDocs);
}

@Override
public Key cacheKey() {
return this.cacheKey;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;

LookupTermsFilter that = (LookupTermsFilter) o;

if (!cacheKey.equals(that.cacheKey)) return false;

return true;
}

@Override
public int hashCode() {
return cacheKey.hashCode();
}

@Override
public String toString() {
return "terms(" + lookup.toString() + ")";
cache.invalidate(new BytesRef(key));
}
}

static class TermsFilterValueWeigher implements Weigher<CacheKeyFilter.Key, TermsFilterValue> {
static class TermsFilterValueWeigher implements Weigher<BytesRef, TermsFilterValue> {

@Override
public int weigh(CacheKeyFilter.Key key, TermsFilterValue value) {
return (int) (key.bytes().length + value.sizeInBytes);
public int weigh(BytesRef key, TermsFilterValue value) {
return (int) (key.length + value.sizeInBytes);
}
}

Expand Down

0 comments on commit 7687217

Please sign in to comment.