From a47fd5ff6e3fdf06009dbc08b701dfcdbf0df890 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 25 Nov 2013 15:11:38 +0100 Subject: [PATCH] Added execution option to `range` filter, with the `index` and `fielddata` as values. Deprecated `numeric_range` filter in favor for the `range` filter with `fielddata` as execution. Closes #4034 --- .../filters/numeric-range-filter.asciidoc | 2 ++ .../query-dsl/filters/range-filter.asciidoc | 19 +++++++++++- .../index/query/FilterBuilders.java | 3 ++ .../query/NumericRangeFilterBuilder.java | 4 ++- .../index/query/NumericRangeFilterParser.java | 1 + .../index/query/RangeFilterBuilder.java | 23 ++++++++++++++ .../index/query/RangeFilterParser.java | 30 +++++++++++++++++-- .../count/query/SimpleQueryTests.java | 24 +++++++++++---- .../query/SimpleIndexQueryParserTests.java | 15 ++++++++++ .../search/query/SimpleQueryTests.java | 13 ++++---- 10 files changed, 117 insertions(+), 17 deletions(-) diff --git a/docs/reference/query-dsl/filters/numeric-range-filter.asciidoc b/docs/reference/query-dsl/filters/numeric-range-filter.asciidoc index 64a4d225040b7..aea15a510844e 100644 --- a/docs/reference/query-dsl/filters/numeric-range-filter.asciidoc +++ b/docs/reference/query-dsl/filters/numeric-range-filter.asciidoc @@ -1,6 +1,8 @@ [[query-dsl-numeric-range-filter]] === Numeric Range Filter +deprecated[0.90.8,Use the <> with the `fielddata` execution mode instead] + Filters documents with fields that have values within a certain numeric range. Similar to <>, except diff --git a/docs/reference/query-dsl/filters/range-filter.asciidoc b/docs/reference/query-dsl/filters/range-filter.asciidoc index 0196c49afcc66..938fbf8095162 100644 --- a/docs/reference/query-dsl/filters/range-filter.asciidoc +++ b/docs/reference/query-dsl/filters/range-filter.asciidoc @@ -32,8 +32,25 @@ The `range` filter accepts the following parameters: deprecated[0.90.4,The `from`, `to`, `include_lower` and `include_upper` parameters have been deprecated in favour of `gt`,`gte`,`lt`,`lte`] +[float] +==== Execution + +added[0.90.8] + +The `execution` option controls how the range filter internally executes. The `execution` option accepts the following values: + +[horizontal] +`index`:: Uses field's inverted in order to determine of documents fall with in the range filter's from and to range +`fielddata`:: Uses field data in order to determine of documents fall with in the range filter's from and to range. + +In general for small ranges the `index` execution is faster and for longer ranges the `fielddata` execution is faster. + +The `fielddata` execution as the same suggests uses field data and therefor requires more memory, so make you have +sufficient memory on your nodes in order to use this execution mode. It usually makes sense to use it on fields you're +already faceting or sorting by. + [float] ==== Caching -The result of the filter is automatically cached by default. The +The result of the filter is only automatically cached by default if the `execution` is set to `index`. The `_cache` can be set to `false` to turn it off. diff --git a/src/main/java/org/elasticsearch/index/query/FilterBuilders.java b/src/main/java/org/elasticsearch/index/query/FilterBuilders.java index 6a8e197ed2bcc..3d972a306143f 100644 --- a/src/main/java/org/elasticsearch/index/query/FilterBuilders.java +++ b/src/main/java/org/elasticsearch/index/query/FilterBuilders.java @@ -300,7 +300,10 @@ public static RangeFilterBuilder rangeFilter(String name) { * field data cache (loading all the values for the specified field into memory) * * @param name The field name + * @deprecated The numeric_range filter will be removed at some point in time in favor for the range filter with + * the execution mode fielddata. */ + @Deprecated public static NumericRangeFilterBuilder numericRangeFilter(String name) { return new NumericRangeFilterBuilder(name); } diff --git a/src/main/java/org/elasticsearch/index/query/NumericRangeFilterBuilder.java b/src/main/java/org/elasticsearch/index/query/NumericRangeFilterBuilder.java index 538cc6ce36ec8..f1ff148aa0950 100644 --- a/src/main/java/org/elasticsearch/index/query/NumericRangeFilterBuilder.java +++ b/src/main/java/org/elasticsearch/index/query/NumericRangeFilterBuilder.java @@ -28,8 +28,10 @@ *

*

Uses the field data cache (loading all the values for the specified field into memory). * - * + * @deprecated This filter will be removed at some point in time in favor for the range filter with the execution + * mode fielddata. */ +@Deprecated public class NumericRangeFilterBuilder extends BaseFilterBuilder { private final String name; diff --git a/src/main/java/org/elasticsearch/index/query/NumericRangeFilterParser.java b/src/main/java/org/elasticsearch/index/query/NumericRangeFilterParser.java index 1729e14a1e9a8..19b65508ff1d1 100644 --- a/src/main/java/org/elasticsearch/index/query/NumericRangeFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/NumericRangeFilterParser.java @@ -34,6 +34,7 @@ /** * */ +@Deprecated public class NumericRangeFilterParser implements FilterParser { public static final String NAME = "numeric_range"; diff --git a/src/main/java/org/elasticsearch/index/query/RangeFilterBuilder.java b/src/main/java/org/elasticsearch/index/query/RangeFilterBuilder.java index 2f639293cfdcd..3e561f6050721 100644 --- a/src/main/java/org/elasticsearch/index/query/RangeFilterBuilder.java +++ b/src/main/java/org/elasticsearch/index/query/RangeFilterBuilder.java @@ -45,6 +45,8 @@ public class RangeFilterBuilder extends BaseFilterBuilder { private String filterName; + private String execution; + /** * A filter that restricts search results to values that are within the given range. * @@ -351,6 +353,24 @@ public RangeFilterBuilder cacheKey(String cacheKey) { return this; } + /** + * Sets the execution mode that controls how the range filter is executed. Valid values are: "index" and "fielddata". + *

    + *
  1. The index execution uses the field's inverted in order to determine of documents fall with in + * the range filter's from and to range. + *
  2. The fielddata execution uses field data in order to determine of documents fall with in the + * range filter's from and to range. Since field data is an in memory data structure, you need to have + * sufficient memory on your nodes in order to use this execution mode. + *
+ * + * In general for small ranges the index execution is faster and for longer ranges the + * fielddata execution is faster. + */ + public RangeFilterBuilder setExecution(String execution) { + this.execution = execution; + return this; + } + @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(RangeFilterParser.NAME); @@ -371,6 +391,9 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep if (cacheKey != null) { builder.field("_cache_key", cacheKey); } + if (execution != null) { + builder.field("execution", execution); + } builder.endObject(); } diff --git a/src/main/java/org/elasticsearch/index/query/RangeFilterParser.java b/src/main/java/org/elasticsearch/index/query/RangeFilterParser.java index b19b436e3a5b6..19598b86439f0 100644 --- a/src/main/java/org/elasticsearch/index/query/RangeFilterParser.java +++ b/src/main/java/org/elasticsearch/index/query/RangeFilterParser.java @@ -25,7 +25,9 @@ import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.cache.filter.support.CacheKeyFilter; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.core.NumberFieldMapper; import java.io.IOException; @@ -51,13 +53,14 @@ public String[] names() { public Filter parse(QueryParseContext parseContext) throws IOException, QueryParsingException { XContentParser parser = parseContext.parser(); - boolean cache = true; + Boolean cache = null; CacheKeyFilter.Key cacheKey = null; String fieldName = null; Object from = null; Object to = null; boolean includeLower = true; boolean includeUpper = true; + String execution = "index"; String filterName = null; String currentFieldName = null; @@ -103,6 +106,8 @@ public Filter parse(QueryParseContext parseContext) throws IOException, QueryPar cache = parser.booleanValue(); } else if ("_cache_key".equals(currentFieldName) || "_cacheKey".equals(currentFieldName)) { cacheKey = new CacheKeyFilter.Key(parser.text()); + } else if ("execution".equals(currentFieldName)) { + execution = parser.text(); } else { throw new QueryParsingException(parseContext.index(), "[range] filter does not support [" + currentFieldName + "]"); } @@ -110,18 +115,37 @@ public Filter parse(QueryParseContext parseContext) throws IOException, QueryPar } if (fieldName == null) { - throw new QueryParsingException(parseContext.index(), "No field specified for range filter"); + throw new QueryParsingException(parseContext.index(), "[range] filter no field specified for range filter"); } Filter filter = null; MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName); if (smartNameFieldMappers != null) { if (smartNameFieldMappers.hasMapper()) { - filter = smartNameFieldMappers.mapper().rangeFilter(from, to, includeLower, includeUpper, parseContext); + if (execution.equals("index")) { + if (cache == null) { + cache = true; + } + filter = smartNameFieldMappers.mapper().rangeFilter(from, to, includeLower, includeUpper, parseContext); + } else if ("fielddata".equals(execution)) { + if (cache == null) { + cache = false; + } + FieldMapper mapper = smartNameFieldMappers.mapper(); + if (!(mapper instanceof NumberFieldMapper)) { + throw new QueryParsingException(parseContext.index(), "[range] filter field [" + fieldName + "] is not a numeric type"); + } + filter = ((NumberFieldMapper) mapper).rangeFilter(parseContext.fieldData(), from, to, includeLower, includeUpper, parseContext); + } else { + throw new QueryParsingException(parseContext.index(), "[range] filter doesn't support [" + execution + "] execution"); + } } } if (filter == null) { + if (cache == null) { + cache = true; + } filter = new TermRangeFilter(fieldName, BytesRefs.toBytesRef(from), BytesRefs.toBytesRef(to), includeLower, includeUpper); } diff --git a/src/test/java/org/elasticsearch/count/query/SimpleQueryTests.java b/src/test/java/org/elasticsearch/count/query/SimpleQueryTests.java index cb60768f4f03c..22f9d831ba943 100644 --- a/src/test/java/org/elasticsearch/count/query/SimpleQueryTests.java +++ b/src/test/java/org/elasticsearch/count/query/SimpleQueryTests.java @@ -1021,16 +1021,16 @@ public void testNumericRangeFilter_2826() throws Exception { client().admin().indices().prepareRefresh().execute().actionGet(); CountResponse response = client().prepareCount("test").setQuery( filteredQuery(matchAllQuery(), FilterBuilders.boolFilter() - .should(FilterBuilders.rangeFilter("num_long").from(1).to(2)) - .should(FilterBuilders.rangeFilter("num_long").from(3).to(4))) + .should(rangeFilter("num_long", 1, 2)) + .should(rangeFilter("num_long", 3, 4))) ).execute().actionGet(); assertHitCount(response, 4l); // This made 2826 fail! (only with bit based filters) response = client().prepareCount("test").setQuery( filteredQuery(matchAllQuery(), FilterBuilders.boolFilter() - .should(FilterBuilders.numericRangeFilter("num_long").from(1).to(2)) - .should(FilterBuilders.numericRangeFilter("num_long").from(3).to(4))) + .should(rangeFilter("num_long", 1, 2)) + .should(rangeFilter("num_long", 3, 4))) ).execute().actionGet(); assertHitCount(response, 4l); @@ -1039,8 +1039,8 @@ public void testNumericRangeFilter_2826() throws Exception { response = client().prepareCount("test").setQuery( filteredQuery(matchAllQuery(), FilterBuilders.boolFilter() .must(FilterBuilders.termFilter("field1", "test1")) - .should(FilterBuilders.rangeFilter("num_long").from(1).to(2)) - .should(FilterBuilders.rangeFilter("num_long").from(3).to(4))) + .should(rangeFilter("num_long", 1, 2)) + .should(rangeFilter("num_long", 3, 4))) ).execute().actionGet(); assertHitCount(response, 2l); @@ -1098,4 +1098,16 @@ public void testSimpleSpan() throws ElasticSearchException, IOException { assertNoFailures(response); assertHitCount(response, 3l); } + + public static FilterBuilder rangeFilter(String field, Object from, Object to) { + if (randomBoolean()) { + if (randomBoolean()) { + return FilterBuilders.rangeFilter(field).from(from).to(to); + } else { + return FilterBuilders.rangeFilter(field).from(from).to(to).setExecution("fielddata"); + } + } else { + return FilterBuilders.numericRangeFilter(field).from(from).to(to); + } + } } diff --git a/src/test/java/org/elasticsearch/index/query/SimpleIndexQueryParserTests.java b/src/test/java/org/elasticsearch/index/query/SimpleIndexQueryParserTests.java index 7d5c96d9ed69d..6f3e15945dd02 100644 --- a/src/test/java/org/elasticsearch/index/query/SimpleIndexQueryParserTests.java +++ b/src/test/java/org/elasticsearch/index/query/SimpleIndexQueryParserTests.java @@ -922,6 +922,21 @@ public void testNumericRangeFilteredQueryBuilder() throws IOException { assertThat(rangeFilter.isIncludeUpper(), equalTo(false)); } + @Test + public void testRangeFilteredQueryBuilder_executionFieldData() throws IOException { + IndexQueryParserService queryParser = queryParser(); + Query parsedQuery = queryParser.parse(filteredQuery(termQuery("name.first", "shay"), rangeFilter("age").from(23).to(54).includeLower(true).includeUpper(false).setExecution("fielddata"))).query(); + assertThat(parsedQuery, instanceOf(XFilteredQuery.class)); + Filter filter = ((XFilteredQuery) parsedQuery).getFilter(); + assertThat(filter, instanceOf(NumericRangeFieldDataFilter.class)); + NumericRangeFieldDataFilter rangeFilter = (NumericRangeFieldDataFilter) filter; + assertThat(rangeFilter.getField(), equalTo("age")); + assertThat(rangeFilter.getLowerVal().intValue(), equalTo(23)); + assertThat(rangeFilter.getUpperVal().intValue(), equalTo(54)); + assertThat(rangeFilter.isIncludeLower(), equalTo(true)); + assertThat(rangeFilter.isIncludeUpper(), equalTo(false)); + } + @Test public void testNumericRangeFilteredQuery() throws IOException { IndexQueryParserService queryParser = queryParser(); diff --git a/src/test/java/org/elasticsearch/search/query/SimpleQueryTests.java b/src/test/java/org/elasticsearch/search/query/SimpleQueryTests.java index cb82bdcb420d5..3537889febbe8 100644 --- a/src/test/java/org/elasticsearch/search/query/SimpleQueryTests.java +++ b/src/test/java/org/elasticsearch/search/query/SimpleQueryTests.java @@ -51,6 +51,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.count.query.SimpleQueryTests.rangeFilter; import static org.elasticsearch.index.query.FilterBuilders.*; import static org.elasticsearch.index.query.QueryBuilders.*; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; @@ -1471,16 +1472,16 @@ public void testNumericRangeFilter_2826() throws Exception { client().admin().indices().prepareRefresh().execute().actionGet(); SearchResponse response = client().prepareSearch("test").setFilter( FilterBuilders.boolFilter() - .should(FilterBuilders.rangeFilter("num_long").from(1).to(2)) - .should(FilterBuilders.rangeFilter("num_long").from(3).to(4)) + .should(rangeFilter("num_long", 1, 2)) + .should(rangeFilter("num_long", 3, 4)) ).execute().actionGet(); assertThat(response.getHits().totalHits(), equalTo(4l)); // This made 2826 fail! (only with bit based filters) response = client().prepareSearch("test").setFilter( FilterBuilders.boolFilter() - .should(FilterBuilders.numericRangeFilter("num_long").from(1).to(2)) - .should(FilterBuilders.numericRangeFilter("num_long").from(3).to(4)) + .should(rangeFilter("num_long", 1, 2)) + .should(rangeFilter("num_long", 3, 4)) ).execute().actionGet(); assertThat(response.getHits().totalHits(), equalTo(4l)); @@ -1489,8 +1490,8 @@ public void testNumericRangeFilter_2826() throws Exception { response = client().prepareSearch("test").setFilter( FilterBuilders.boolFilter() .must(FilterBuilders.termFilter("field1", "test1")) - .should(FilterBuilders.rangeFilter("num_long").from(1).to(2)) - .should(FilterBuilders.rangeFilter("num_long").from(3).to(4)) + .should(rangeFilter("num_long", 1, 2)) + .should(rangeFilter("num_long", 3, 4)) ).execute().actionGet(); assertThat(response.getHits().totalHits(), equalTo(2l));