Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Resolve now in date ranges in percolator and alias filters at search time instead of parse time #8534

Merged
merged 1 commit into from Nov 19, 2014
Merged

Resolve now in date ranges in percolator and alias filters at search time instead of parse time #8534

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
36 changes: 36 additions & 0 deletions src/main/java/org/elasticsearch/common/lucene/search/NoCacheQuery.java
View file
Open in desktop
@@ -0,0 +1,36 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.common.lucene.search;

import org.apache.lucene.search.Query;

/**
* Queries are never cached directly, but a query can be wrapped in a filter that may end being cached.
* Filters that wrap this query either directly or indirectly will never be cached.
*/
public abstract class NoCacheQuery extends Query {

@Override
public final String toString(String s) {
return "no_cache(" + innerToString(s) + ")";
}

public abstract String innerToString(String s);
}
49 changes: 49 additions & 0 deletions src/main/java/org/elasticsearch/common/lucene/search/ResolvableFilter.java
View file
Open in desktop
@@ -0,0 +1,49 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.common.lucene.search;

import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.Bits;

import java.io.IOException;

/**
* A filter implementation that resolves details at the last possible moment between filter parsing and execution.
* For example a date filter based on 'now'.
*/
public abstract class ResolvableFilter extends Filter {

/**
* @return The actual filter instance to be executed containing the latest details.
*/
public abstract Filter resolve();

@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
Filter resolvedFilter = resolve();
if (resolvedFilter != null) {
return resolvedFilter.getDocIdSet(context, acceptDocs);
} else {
return null;
}
}
}
130 changes: 107 additions & 23 deletions src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
View file
Open in desktop
Expand Up @@ -21,13 +21,15 @@

import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.NumericRangeFilter;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.ToStringUtils;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Nullable;
Expand All @@ -37,6 +39,8 @@
import org.elasticsearch.common.joda.FormatDateTimeFormatter;
import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.lucene.search.NoCacheFilter;
import org.elasticsearch.common.lucene.search.NoCacheQuery;
import org.elasticsearch.common.lucene.search.ResolvableFilter;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.common.util.LocaleUtils;
Expand All @@ -52,6 +56,7 @@
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
import org.elasticsearch.index.similarity.SimilarityProvider;
import org.elasticsearch.search.internal.SearchContext;
import org.joda.time.DateTimeZone;

import java.io.IOException;
Expand Down Expand Up @@ -293,39 +298,35 @@ public Query fuzzyQuery(String value, Fuzziness fuzziness, int prefixLength, int

@Override
public Query termQuery(Object value, @Nullable QueryParseContext context) {
long lValue = parseToMilliseconds(value, context);
long lValue = parseToMilliseconds(value);
return NumericRangeQuery.newLongRange(names.indexName(), precisionStep,
lValue, lValue, true, true);
}

public long parseToMilliseconds(Object value, @Nullable QueryParseContext context) {
return parseToMilliseconds(value, context, false);
public long parseToMilliseconds(Object value) {
return parseToMilliseconds(value, false, null, dateMathParser);
}

public long parseToMilliseconds(Object value, @Nullable QueryParseContext context, boolean includeUpper) {
return parseToMilliseconds(value, context, includeUpper, null, dateMathParser);
}

public long parseToMilliseconds(Object value, @Nullable QueryParseContext context, boolean includeUpper, @Nullable DateTimeZone zone, @Nullable DateMathParser forcedDateParser) {
public long parseToMilliseconds(Object value, boolean includeUpper, @Nullable DateTimeZone zone, @Nullable DateMathParser forcedDateParser) {
if (value instanceof Number) {
return ((Number) value).longValue();
}
return parseToMilliseconds(convertToString(value), context, includeUpper, zone, forcedDateParser);
return parseToMilliseconds(convertToString(value), includeUpper, zone, forcedDateParser);
}

public long parseToMilliseconds(String value, @Nullable QueryParseContext context, boolean includeUpper, @Nullable DateTimeZone zone, @Nullable DateMathParser forcedDateParser) {
long now = context == null ? System.currentTimeMillis() : context.nowInMillis();
public long parseToMilliseconds(String value, boolean includeUpper, @Nullable DateTimeZone zone, @Nullable DateMathParser forcedDateParser) {
SearchContext sc = SearchContext.current();
long now = sc == null ? System.currentTimeMillis() : sc.nowInMillis();
DateMathParser dateParser = dateMathParser;
if (forcedDateParser != null) {
dateParser = forcedDateParser;
}
long time = includeUpper && roundCeil ? dateParser.parseRoundCeil(value, now, zone) : dateParser.parse(value, now, zone);
return time;
return includeUpper && roundCeil ? dateParser.parseRoundCeil(value, now, zone) : dateParser.parse(value, now, zone);
}

@Override
public Filter termFilter(Object value, @Nullable QueryParseContext context) {
final long lValue = parseToMilliseconds(value, context);
final long lValue = parseToMilliseconds(value);
return NumericRangeFilter.newLongRange(names.indexName(), precisionStep,
lValue, lValue, true, true);
}
Expand All @@ -336,9 +337,18 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower
}

public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser, @Nullable QueryParseContext context) {
// If the current search context is null we're parsing percolator query or a index alias filter.
if (SearchContext.current() == null) {
return new LateParsingQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser);
} else {
return innerRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser);
}
}

private Query innerRangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser) {
return NumericRangeQuery.newLongRange(names.indexName(), precisionStep,
lowerTerm == null ? null : parseToMilliseconds(lowerTerm, context, false, timeZone, forcedDateParser == null ? dateMathParser : forcedDateParser),
upperTerm == null ? null : parseToMilliseconds(upperTerm, context, includeUpper, timeZone, forcedDateParser == null ? dateMathParser : forcedDateParser),
lowerTerm == null ? null : parseToMilliseconds(lowerTerm, false, timeZone, forcedDateParser == null ? dateMathParser : forcedDateParser),
upperTerm == null ? null : parseToMilliseconds(upperTerm, includeUpper, timeZone, forcedDateParser == null ? dateMathParser : forcedDateParser),
includeLower, includeUpper);
}

Expand All @@ -363,6 +373,16 @@ public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Obje
* - the String to parse does not have already a timezone defined (ie. `2014-01-01T00:00:00+03:00`)
*/
public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser, @Nullable QueryParseContext context, @Nullable Boolean explicitCaching) {
IndexNumericFieldData fieldData = parseContext != null ? (IndexNumericFieldData) parseContext.getForField(this) : null;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why do we actually need to only wrap this in a LateParsingFilter if there is no context? can't we do this all the time and in our tests do the resolve instead? I don't like the special casing based on the context here to be honest

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like to do that too, but this does mean that the resolve method is invoked for each segment that filter is going to execute on. See QueryParseContext#cacheFilter(...) around line 214.

Would this side effect be acceptable?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should just stick with what we have for now :/ filter needs a rewrite method

// If the current search context is null we're parsing percolator query or a index alias filter.
if (SearchContext.current() == null) {
return new LateParsingFilter(fieldData, lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser, explicitCaching);
} else {
return innerRangeFilter(fieldData, lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser, explicitCaching);
}
}

private Filter innerRangeFilter(IndexNumericFieldData fieldData, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser, @Nullable Boolean explicitCaching) {
boolean cache;
boolean cacheable = true;
Long lowerVal = null;
Expand All @@ -373,7 +393,7 @@ public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Obje
} else {
String value = convertToString(lowerTerm);
cacheable = !hasDateExpressionWithNoRounding(value);
lowerVal = parseToMilliseconds(value, context, false, timeZone, forcedDateParser);
lowerVal = parseToMilliseconds(value, false, timeZone, forcedDateParser);
}
}
if (upperTerm != null) {
Expand All @@ -382,7 +402,7 @@ public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Obje
} else {
String value = convertToString(upperTerm);
cacheable = cacheable && !hasDateExpressionWithNoRounding(value);
upperVal = parseToMilliseconds(value, context, includeUpper, timeZone, forcedDateParser);
upperVal = parseToMilliseconds(value, includeUpper, timeZone, forcedDateParser);
}
}

Expand All @@ -397,12 +417,10 @@ public Filter rangeFilter(QueryParseContext parseContext, Object lowerTerm, Obje
}

Filter filter;
if (parseContext != null) {
filter = NumericRangeFieldDataFilter.newLongRange(
(IndexNumericFieldData) parseContext.getForField(this), lowerVal,upperVal, includeLower, includeUpper
);
if (fieldData != null) {
filter = NumericRangeFieldDataFilter.newLongRange(fieldData, lowerVal,upperVal, includeLower, includeUpper);
} else {
filter = NumericRangeFilter.newLongRange(
filter = NumericRangeFilter.newLongRange(
names.indexName(), precisionStep, lowerVal, upperVal, includeLower, includeUpper
);
}
Expand Down Expand Up @@ -590,4 +608,70 @@ private long parseStringValue(String value) {
}
}
}

private final class LateParsingFilter extends ResolvableFilter {

final IndexNumericFieldData fieldData;
final Object lowerTerm;
final Object upperTerm;
final boolean includeLower;
final boolean includeUpper;
final DateTimeZone timeZone;
final DateMathParser forcedDateParser;
final Boolean explicitCaching;

public LateParsingFilter(IndexNumericFieldData fieldData, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, DateTimeZone timeZone, DateMathParser forcedDateParser, Boolean explicitCaching) {
this.fieldData = fieldData;
this.lowerTerm = lowerTerm;
this.upperTerm = upperTerm;
this.includeLower = includeLower;
this.includeUpper = includeUpper;
this.timeZone = timeZone;
this.forcedDateParser = forcedDateParser;
this.explicitCaching = explicitCaching;
}

@Override
public Filter resolve() {
return innerRangeFilter(fieldData, lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser, explicitCaching);
}
}

public final class LateParsingQuery extends NoCacheQuery {

final Object lowerTerm;
final Object upperTerm;
final boolean includeLower;
final boolean includeUpper;
final DateTimeZone timeZone;
final DateMathParser forcedDateParser;

public LateParsingQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, DateTimeZone timeZone, DateMathParser forcedDateParser) {
this.lowerTerm = lowerTerm;
this.upperTerm = upperTerm;
this.includeLower = includeLower;
this.includeUpper = includeUpper;
this.timeZone = timeZone;
this.forcedDateParser = forcedDateParser;
}

@Override
public Query rewrite(IndexReader reader) throws IOException {
Query query = innerRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, forcedDateParser);
return query.rewrite(reader);
}

@Override
public String innerToString(String s) {
final StringBuilder sb = new StringBuilder();
return sb.append(names.indexName()).append(':')
.append(includeLower ? '[' : '{')
.append((lowerTerm == null) ? "*" : lowerTerm.toString())
.append(" TO ")
.append((upperTerm == null) ? "*" : upperTerm.toString())
.append(includeUpper ? ']' : '}')
.append(ToStringUtils.boost(getBoost()))
.toString();
}
}
}
40 changes: 34 additions & 6 deletions src/main/java/org/elasticsearch/index/query/QueryParseContext.java
View file
Open in desktop
Expand Up @@ -21,24 +21,29 @@

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.queryparser.classic.MapperQueryParser;
import org.apache.lucene.queryparser.classic.QueryParserSettings;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.similarities.Similarity;
import org.elasticsearch.cache.recycler.CacheRecycler;
import org.apache.lucene.util.Bits;
import org.elasticsearch.Version;
import org.elasticsearch.cache.recycler.CacheRecycler;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.lucene.search.NoCacheFilter;
import org.elasticsearch.common.lucene.search.NoCacheQuery;
import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.common.lucene.search.ResolvableFilter;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.cache.filter.support.CacheKeyFilter;
import org.elasticsearch.index.cache.fixedbitset.FixedBitSetFilter;
import org.elasticsearch.index.cache.query.parser.QueryParserCache;
import org.elasticsearch.index.engine.IndexEngine;
import org.elasticsearch.index.cache.fixedbitset.FixedBitSetFilter;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldMappers;
Expand Down Expand Up @@ -192,17 +197,37 @@ public FixedBitSetFilter fixedBitSetFilter(Filter filter) {
return indexQueryParser.fixedBitSetFilterCache.getFixedBitSetFilter(filter);
}

public Filter cacheFilter(Filter filter, @Nullable CacheKeyFilter.Key cacheKey) {
public Filter cacheFilter(Filter filter, @Nullable final CacheKeyFilter.Key cacheKey) {
if (filter == null) {
return null;
}
if (this.disableFilterCaching || this.propagateNoCache || filter instanceof NoCacheFilter) {
return filter;
}
if (cacheKey != null) {
filter = new CacheKeyFilter.Wrapper(filter, cacheKey);
if (filter instanceof ResolvableFilter) {
final ResolvableFilter resolvableFilter = (ResolvableFilter) filter;
// We need to wrap it another filter, because this method is invoked at query parse time, which
// may not be during search execution time. (for example index alias filter and percolator)
return new Filter() {
@Override
public DocIdSet getDocIdSet(AtomicReaderContext atomicReaderContext, Bits bits) throws IOException {
Filter filter = resolvableFilter.resolve();
if (filter == null) {
return null;
}
if (cacheKey != null) {
filter = new CacheKeyFilter.Wrapper(filter, cacheKey);
}
filter = indexQueryParser.indexCache.filter().cache(filter);
return filter.getDocIdSet(atomicReaderContext, bits);
}
};
} else {
if (cacheKey != null) {
filter = new CacheKeyFilter.Wrapper(filter, cacheKey);
}
return indexQueryParser.indexCache.filter().cache(filter);
}
return indexQueryParser.indexCache.filter().cache(filter);
}

public <IFD extends IndexFieldData<?>> IFD getForField(FieldMapper<?> mapper) {
Expand Down Expand Up @@ -254,6 +279,9 @@ public Query parseInnerQuery() throws IOException, QueryParsingException {
// if we are at END_OBJECT, move to the next one...
parser.nextToken();
}
if (result instanceof NoCacheQuery) {
propagateNoCache = true;
}
if (CustomQueryWrappingFilter.shouldUseCustomQueryWrappingFilter(result)) {
requireCustomQueryWrappingFilter = true;
// If later on, either directly or indirectly this query gets wrapped in a query filter it must never
Expand Down