From ed4eecc00e266fbc47cd5ae63d8b3d8a483a620b Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Thu, 21 Nov 2019 22:45:52 +0100 Subject: [PATCH] Pre-sort shards based on the max/min value of the primary sort field (#49092) This change automatically pre-sort search shards on search requests that use a primary sort based on the value of a field. When possible, the can_match phase will extract the min/max (depending on the provided sort order) values of each shard and use it to pre-sort the shards prior to running the subsequent phases. This feature can be useful to ensure that shards that contain recent data are executed first so that intermediate merge have more chance to contain contiguous data (think of date_histogram for instance) but it could also be used in a follow up to early terminate sorted top-hits queries that don't require the total hit count. The latter could significantly speed up the retrieval of the most/least recent documents from time-based indices. Relates #49091 --- .../search/AbstractSearchAsyncAction.java | 4 +- .../search/CanMatchPreFilterSearchPhase.java | 81 ++++++-- .../action/search/SearchActionListener.java | 2 +- .../action/search/TransportSearchAction.java | 7 +- .../cluster/routing/GroupShardsIterator.java | 11 +- .../index/mapper/DateFieldMapper.java | 2 +- .../index/mapper/NumberFieldMapper.java | 6 +- .../elasticsearch/search/SearchService.java | 32 ++- .../search/sort/FieldSortBuilder.java | 126 ++++++++++++ .../elasticsearch/search/sort/MinAndMax.java | 83 ++++++++ .../CanMatchPreFilterSearchPhaseTests.java | 90 ++++++++- .../search/SearchServiceTests.java | 12 +- .../search/sort/AbstractSortTestCase.java | 10 +- .../search/sort/FieldSortBuilderTests.java | 188 ++++++++++++++++++ .../index/engine/FrozenIndexTests.java | 12 +- 15 files changed, 616 insertions(+), 50 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java index 18051aa99db4f..2d2b9213c63ff 100644 --- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java @@ -113,8 +113,8 @@ abstract class AbstractSearchAsyncAction exten iterators.add(iterator); } } - this.toSkipShardsIts = new GroupShardsIterator<>(toSkipIterators); - this.shardsIts = new GroupShardsIterator<>(iterators); + this.toSkipShardsIts = new GroupShardsIterator<>(toSkipIterators, false); + this.shardsIts = new GroupShardsIterator<>(iterators, false); // we need to add 1 for non active partition, since we count it in the total. This means for each shard in the iterator we sum up // it's number of active shards but use 1 as the default if no replica of a shard is active at this point. // on a per shards level we use shardIt.remaining() to increment the totalOps pointer but add 1 for the current shard result diff --git a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java index 66187d5220bbe..aba32d2c850a0 100644 --- a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java @@ -23,15 +23,25 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.search.SearchService; +import org.elasticsearch.search.SearchService.CanMatchResponse; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.AliasFilter; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.MinAndMax; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.transport.Transport; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.Executor; import java.util.function.BiFunction; import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.Stream; /** @@ -40,8 +50,12 @@ * from the search. The extra round trip to the search shards is very cheap and is not subject to rejections * which allows to fan out to more shards at the same time without running into rejections even if we are hitting a * large portion of the clusters indices. + * This phase can also be used to pre-sort shards based on min/max values in each shard of the provided primary sort. + * When the query primary sort is perform on a field, this phase extracts the min/max value in each shard and + * sort them according to the provided order. This can be useful for instance to ensure that shards that contain recent + * data are executed first when sorting by descending timestamp. */ -final class CanMatchPreFilterSearchPhase extends AbstractSearchAsyncAction { +final class CanMatchPreFilterSearchPhase extends AbstractSearchAsyncAction { private final Function, SearchPhase> phaseFactory; private final GroupShardsIterator shardsIts; @@ -58,26 +72,26 @@ final class CanMatchPreFilterSearchPhase extends AbstractSearchAsyncAction listener) { + SearchActionListener listener) { getSearchTransport().sendCanMatch(getConnection(shardIt.getClusterAlias(), shard.currentNodeId()), buildShardSearchRequest(shardIt), getTask(), listener); } @Override - protected SearchPhase getNextPhase(SearchPhaseResults results, + protected SearchPhase getNextPhase(SearchPhaseResults results, SearchPhaseContext context) { - return phaseFactory.apply(getIterator((BitSetSearchPhaseResults) results, shardsIts)); + return phaseFactory.apply(getIterator((CanMatchSearchPhaseResults) results, shardsIts)); } - private GroupShardsIterator getIterator(BitSetSearchPhaseResults results, + private GroupShardsIterator getIterator(CanMatchSearchPhaseResults results, GroupShardsIterator shardsIts) { int cardinality = results.getNumPossibleMatches(); FixedBitSet possibleMatches = results.getPossibleMatches(); @@ -86,6 +100,7 @@ private GroupShardsIterator getIterator(BitSetSearchPhaseRe // to produce a valid search result with all the aggs etc. possibleMatches.set(0); } + SearchSourceBuilder source = getRequest().source(); int i = 0; for (SearchShardIterator iter : shardsIts) { if (possibleMatches.get(i++)) { @@ -94,24 +109,48 @@ private GroupShardsIterator getIterator(BitSetSearchPhaseRe iter.resetAndSkip(); } } - return shardsIts; + if (shouldSortShards(results.minAndMaxes) == false) { + return shardsIts; + } + FieldSortBuilder fieldSort = FieldSortBuilder.getPrimaryFieldSortOrNull(source); + return new GroupShardsIterator<>(sortShards(shardsIts, results.minAndMaxes, fieldSort.order()), false); } - private static final class BitSetSearchPhaseResults extends SearchPhaseResults { + private static List sortShards(GroupShardsIterator shardsIts, + MinAndMax[] minAndMaxes, + SortOrder order) { + return IntStream.range(0, shardsIts.size()) + .boxed() + .sorted(shardComparator(shardsIts, minAndMaxes, order)) + .map(ord -> shardsIts.get(ord)) + .collect(Collectors.toList()); + } + private static boolean shouldSortShards(MinAndMax[] minAndMaxes) { + return Arrays.stream(minAndMaxes).anyMatch(Objects::nonNull); + } + + private static Comparator shardComparator(GroupShardsIterator shardsIts, + MinAndMax[] minAndMaxes, + SortOrder order) { + final Comparator comparator = Comparator.comparing(index -> minAndMaxes[index], MinAndMax.getComparator(order)); + return comparator.thenComparing(index -> shardsIts.get(index).shardId()); + } + + private static final class CanMatchSearchPhaseResults extends SearchPhaseResults { private final FixedBitSet possibleMatches; + private final MinAndMax[] minAndMaxes; private int numPossibleMatches; - BitSetSearchPhaseResults(int size) { + CanMatchSearchPhaseResults(int size) { super(size); possibleMatches = new FixedBitSet(size); + minAndMaxes = new MinAndMax[size]; } @Override - void consumeResult(SearchService.CanMatchResponse result) { - if (result.canMatch()) { - consumeShardFailure(result.getShardIndex()); - } + void consumeResult(CanMatchResponse result) { + consumeResult(result.getShardIndex(), result.canMatch(), result.minAndMax()); } @Override @@ -120,12 +159,18 @@ boolean hasResult(int shardIndex) { } @Override - synchronized void consumeShardFailure(int shardIndex) { + void consumeShardFailure(int shardIndex) { // we have to carry over shard failures in order to account for them in the response. - possibleMatches.set(shardIndex); - numPossibleMatches++; + consumeResult(shardIndex, true, null); } + synchronized void consumeResult(int shardIndex, boolean canMatch, MinAndMax minAndMax) { + if (canMatch) { + possibleMatches.set(shardIndex); + numPossibleMatches++; + } + minAndMaxes[shardIndex] = minAndMax; + } synchronized int getNumPossibleMatches() { return numPossibleMatches; @@ -136,7 +181,7 @@ synchronized FixedBitSet getPossibleMatches() { } @Override - Stream getSuccessfulResults() { + Stream getSuccessfulResults() { return Stream.empty(); } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchActionListener.java b/server/src/main/java/org/elasticsearch/action/search/SearchActionListener.java index d34c8c61d434a..e9b5598556ff7 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchActionListener.java @@ -28,7 +28,7 @@ */ abstract class SearchActionListener implements ActionListener { - private final int requestIndex; + final int requestIndex; private final SearchShardTarget searchShardTarget; protected SearchActionListener(SearchShardTarget searchShardTarget, diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index fda3e47cfaab7..bca785102c9c5 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -56,6 +56,7 @@ import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.profile.ProfileShardResult; import org.elasticsearch.search.profile.SearchProfileShardResults; +import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.RemoteClusterAware; @@ -615,9 +616,9 @@ static BiFunction buildConnectionLookup(St private static boolean shouldPreFilterSearchShards(SearchRequest searchRequest, GroupShardsIterator shardIterators) { SearchSourceBuilder source = searchRequest.source(); - return searchRequest.searchType() == QUERY_THEN_FETCH && // we can't do this for DFS it needs to fan out to all shards all the time - SearchService.canRewriteToMatchNone(source) && - searchRequest.getPreFilterShardSize() < shardIterators.size(); + return searchRequest.searchType() == QUERY_THEN_FETCH // we can't do this for DFS it needs to fan out to all shards all the time + && (SearchService.canRewriteToMatchNone(source) || FieldSortBuilder.hasPrimaryFieldSort(source)) + && searchRequest.getPreFilterShardSize() < shardIterators.size(); } static GroupShardsIterator mergeShardsIterators(GroupShardsIterator localShardsIterator, diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/GroupShardsIterator.java b/server/src/main/java/org/elasticsearch/cluster/routing/GroupShardsIterator.java index 21b02043a2249..a9904c96d020f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/GroupShardsIterator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/GroupShardsIterator.java @@ -38,7 +38,16 @@ public final class GroupShardsIterator implements * Constructs a enw GroupShardsIterator from the given list. */ public GroupShardsIterator(List iterators) { - CollectionUtil.timSort(iterators); + this(iterators, true); + } + + /** + * Constructs a new GroupShardsIterator from the given list. + */ + public GroupShardsIterator(List iterators, boolean useSort) { + if (useSort) { + CollectionUtil.timSort(iterators); + } this.iterators = iterators; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index cfd6fdcec7a25..7a98d9a286ea8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -252,7 +252,7 @@ public static final class DateFieldType extends MappedFieldType { protected DateMathParser dateMathParser; protected Resolution resolution; - DateFieldType() { + public DateFieldType() { super(); setTokenized(false); setHasDocValues(true); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 927bce5d9d6dd..0a473bd189e3b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -816,7 +816,7 @@ public final String typeName() { return name; } /** Get the associated numeric type */ - final NumericType numericType() { + public final NumericType numericType() { return numericType; } public abstract Query termQuery(String field, Object value); @@ -909,6 +909,10 @@ public String typeName() { return type.name; } + public NumericType numericType() { + return type.numericType(); + } + @Override public Query existsQuery(QueryShardContext context) { if (hasDocValues()) { diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 5a641b76e8f71..230b9bdc431ea 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -24,6 +24,7 @@ import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.TopDocs; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.OriginalIndices; @@ -92,6 +93,8 @@ import org.elasticsearch.search.query.ScrollQuerySearchResult; import org.elasticsearch.search.rescore.RescorerBuilder; import org.elasticsearch.search.searchafter.SearchAfterBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.MinAndMax; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.suggest.Suggest; @@ -1013,7 +1016,7 @@ public AliasFilter buildAliasFilter(ClusterState state, String index, Setfalse the query won't match any documents on the current * shard. */ - public boolean canMatch(ShardSearchRequest request) throws IOException { + public CanMatchResponse canMatch(ShardSearchRequest request) throws IOException { assert request.searchType() == SearchType.QUERY_THEN_FETCH : "unexpected search type: " + request.searchType(); IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex()); IndexShard indexShard = indexService.getShard(request.shardId().getId()); @@ -1023,18 +1026,20 @@ public boolean canMatch(ShardSearchRequest request) throws IOException { QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), searcher, request::nowInMillis, request.getClusterAlias()); Rewriteable.rewrite(request.getRewriteable(), context, false); + FieldSortBuilder sortBuilder = FieldSortBuilder.getPrimaryFieldSortOrNull(request.source()); + MinAndMax minMax = sortBuilder != null ? FieldSortBuilder.getMinMaxOrNull(context, sortBuilder) : null; if (canRewriteToMatchNone(request.source())) { QueryBuilder queryBuilder = request.source().query(); - return queryBuilder instanceof MatchNoneQueryBuilder == false; + return new CanMatchResponse(queryBuilder instanceof MatchNoneQueryBuilder == false, minMax); } - return true; // null query means match_all + // null query means match_all + return new CanMatchResponse(true, minMax); } } - public void canMatch(ShardSearchRequest request, ActionListener listener) { try { - listener.onResponse(new CanMatchResponse(canMatch(request))); + listener.onResponse(canMatch(request)); } catch (IOException e) { listener.onFailure(e); } @@ -1053,6 +1058,7 @@ public static boolean canRewriteToMatchNone(SearchSourceBuilder source) { return aggregations == null || aggregations.mustVisitAllDocs() == false; } + /* * Rewrites the search request with a light weight rewrite context in order to fetch resources asynchronously * The action listener is guaranteed to be executed on the search thread-pool @@ -1088,24 +1094,38 @@ public InternalAggregation.ReduceContext createReduceContext(boolean finalReduce public static final class CanMatchResponse extends SearchPhaseResult { private final boolean canMatch; + private final MinAndMax minAndMax; public CanMatchResponse(StreamInput in) throws IOException { super(in); this.canMatch = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_7_6_0)) { + minAndMax = in.readOptionalWriteable(MinAndMax::new); + } else { + minAndMax = null; + } } - public CanMatchResponse(boolean canMatch) { + public CanMatchResponse(boolean canMatch, MinAndMax minAndMax) { this.canMatch = canMatch; + this.minAndMax = minAndMax; } @Override public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(canMatch); + if (out.getVersion().onOrAfter(Version.V_7_6_0)) { + out.writeOptionalWriteable(minAndMax); + } } public boolean canMatch() { return canMatch; } + + public MinAndMax minAndMax() { + return minAndMax; + } } /** diff --git a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index a4e5f4c526235..98c423e6644fa 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -20,33 +20,49 @@ package org.elasticsearch.search.sort; import org.apache.logging.log4j.LogManager; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiTerms; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.Terms; import org.apache.lucene.search.SortField; import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; import org.elasticsearch.index.fielddata.plain.SortedNumericDVIndexFieldData; +import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType; +import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; +import java.util.Collections; import java.util.Locale; import java.util.Objects; +import java.util.function.Function; +import static org.elasticsearch.index.mapper.DateFieldMapper.Resolution.MILLISECONDS; +import static org.elasticsearch.index.mapper.DateFieldMapper.Resolution.NANOSECONDS; import static org.elasticsearch.search.sort.NestedSortBuilder.NESTED_FIELD; /** @@ -436,6 +452,116 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException { } } + /** + * Return true if the primary sort in the provided source + * is an instance of {@link FieldSortBuilder}. + */ + public static boolean hasPrimaryFieldSort(SearchSourceBuilder source) { + return getPrimaryFieldSortOrNull(source) != null; + } + + /** + * Return the {@link FieldSortBuilder} if the primary sort in the provided source + * is an instance of this class, null otherwise. + */ + public static FieldSortBuilder getPrimaryFieldSortOrNull(SearchSourceBuilder source) { + if (source == null || source.sorts() == null || source.sorts().isEmpty()) { + return null; + } + return source.sorts().get(0) instanceof FieldSortBuilder ? (FieldSortBuilder) source.sorts().get(0) : null; + } + + /** + * Return a {@link Function} that converts a serialized point into a {@link Number} according to the provided + * {@link SortField}. This is needed for {@link SortField} that converts values from one type to another using + * {@link FieldSortBuilder#setNumericType(String)} )} (e.g.: long to double). + */ + private static Function numericPointConverter(SortField sortField, NumberFieldType numberFieldType) { + switch (IndexSortConfig.getSortFieldType(sortField)) { + case LONG: + return v -> numberFieldType.parsePoint(v).longValue(); + + case INT: + return v -> numberFieldType.parsePoint(v).intValue(); + + case DOUBLE: + return v -> numberFieldType.parsePoint(v).doubleValue(); + + case FLOAT: + return v -> numberFieldType.parsePoint(v).floatValue(); + + default: + return v -> null; + } + } + + /** + * Return a {@link Function} that converts a serialized date point into a {@link Long} according to the provided + * {@link NumericType}. + */ + private static Function datePointConverter(DateFieldType dateFieldType, String numericTypeStr) { + if (numericTypeStr != null) { + NumericType numericType = resolveNumericType(numericTypeStr); + if (dateFieldType.resolution() == MILLISECONDS && numericType == NumericType.DATE_NANOSECONDS) { + return v -> DateUtils.toNanoSeconds(LongPoint.decodeDimension(v, 0)); + } else if (dateFieldType.resolution() == NANOSECONDS && numericType == NumericType.DATE) { + return v -> DateUtils.toMilliSeconds(LongPoint.decodeDimension(v, 0)); + } + } + return v -> LongPoint.decodeDimension(v, 0); + } + + /** + * Return the {@link MinAndMax} indexed value from the provided {@link FieldSortBuilder} or null if unknown. + * The value can be extracted on non-nested indexed mapped fields of type keyword, numeric or date, other fields + * and configurations return null. + */ + public static MinAndMax getMinMaxOrNull(QueryShardContext context, FieldSortBuilder sortBuilder) throws IOException { + SortAndFormats sort = SortBuilder.buildSort(Collections.singletonList(sortBuilder), context).get(); + SortField sortField = sort.sort.getSort()[0]; + if (sortField.getField() == null) { + return null; + } + IndexReader reader = context.getIndexReader(); + MappedFieldType fieldType = context.fieldMapper(sortField.getField()); + if (reader == null || (fieldType == null || fieldType.indexOptions() == IndexOptions.NONE)) { + return null; + } + String fieldName = fieldType.name(); + switch (IndexSortConfig.getSortFieldType(sortField)) { + case LONG: + case INT: + case DOUBLE: + case FLOAT: + final Function converter; + if (fieldType instanceof NumberFieldType) { + converter = numericPointConverter(sortField, (NumberFieldType) fieldType); + } else if (fieldType instanceof DateFieldType) { + converter = datePointConverter((DateFieldType) fieldType, sortBuilder.getNumericType()); + } else { + return null; + } + if (PointValues.size(reader, fieldName) == 0) { + return null; + } + final Comparable min = converter.apply(PointValues.getMinPackedValue(reader, fieldName)); + final Comparable max = converter.apply(PointValues.getMaxPackedValue(reader, fieldName)); + return MinAndMax.newMinMax(min, max); + + case STRING: + case STRING_VAL: + if (fieldType instanceof KeywordFieldMapper.KeywordFieldType) { + Terms terms = MultiTerms.getTerms(reader, fieldName); + if (terms == null) { + return null; + } + return terms.getMin() != null ? MinAndMax.newMinMax(terms.getMin(), terms.getMax()) : null; + } + break; + } + return null; + } + /** * Throws an exception if max children is not located at top level nested sort. */ diff --git a/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java b/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java new file mode 100644 index 0000000000000..28e07c8863b8a --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/sort/MinAndMax.java @@ -0,0 +1,83 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.sort; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.lucene.Lucene; + +import java.io.IOException; +import java.util.Comparator; +import java.util.Objects; + +/** + * A class that encapsulates a minimum and a maximum {@link Comparable}. + */ +public class MinAndMax> implements Writeable { + private final T minValue; + private final T maxValue; + + private MinAndMax(T minValue, T maxValue) { + this.minValue = Objects.requireNonNull(minValue); + this.maxValue = Objects.requireNonNull(maxValue); + } + + public MinAndMax(StreamInput in) throws IOException { + this.minValue = (T) Lucene.readSortValue(in); + this.maxValue = (T) Lucene.readSortValue(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + Lucene.writeSortValue(out, minValue); + Lucene.writeSortValue(out, maxValue); + } + + /** + * Return the minimum value. + */ + public T getMin() { + return minValue; + } + + /** + * Return the maximum value. + */ + public T getMax() { + return maxValue; + } + + public static > MinAndMax newMinMax(T min, T max) { + return new MinAndMax<>(min, max); + } + + /** + * Return a {@link Comparator} for {@link MinAndMax} values according to the provided {@link SortOrder}. + */ + public static Comparator> getComparator(SortOrder order) { + Comparator cmp = order == SortOrder.ASC ? + Comparator.comparing(v -> (Comparable) v.getMin()) : Comparator.comparing(v -> (Comparable) v.getMax()); + if (order == SortOrder.DESC) { + cmp = cmp.reversed(); + } + return Comparator.nullsLast(cmp); + } +} diff --git a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java index 361b8d224923c..05231e1c5ab5a 100644 --- a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java @@ -26,21 +26,32 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchService; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.ShardSearchRequest; +import org.elasticsearch.search.sort.MinAndMax; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.transport.Transport; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.IntStream; public class CanMatchPreFilterSearchPhaseTests extends ESTestCase { @@ -62,7 +73,7 @@ public void testFilterShards() throws InterruptedException { public void sendCanMatch(Transport.Connection connection, ShardSearchRequest request, SearchTask task, ActionListener listener) { new Thread(() -> listener.onResponse(new SearchService.CanMatchResponse(request.shardId().id() == 0 ? shard1 : - shard2))).start(); + shard2, null))).start(); } }; @@ -124,7 +135,7 @@ public void sendCanMatch(Transport.Connection connection, ShardSearchRequest req } else { new Thread(() -> { if (throwException == false) { - listener.onResponse(new SearchService.CanMatchResponse(shard1)); + listener.onResponse(new SearchService.CanMatchResponse(shard1, null)); } else { listener.onFailure(new NullPointerException()); } @@ -187,7 +198,7 @@ public void sendCanMatch( ShardSearchRequest request, SearchTask task, ActionListener listener) { - listener.onResponse(new SearchService.CanMatchResponse(randomBoolean())); + listener.onResponse(new SearchService.CanMatchResponse(randomBoolean(), null)); } }; @@ -265,4 +276,77 @@ protected void executePhaseOnShard( latch.await(); executor.shutdown(); } + + public void testSortShards() throws InterruptedException { + final TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider(0, System.nanoTime(), + System::nanoTime); + + Map lookup = new ConcurrentHashMap<>(); + DiscoveryNode primaryNode = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNode replicaNode = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT); + lookup.put("node1", new SearchAsyncActionTests.MockConnection(primaryNode)); + lookup.put("node2", new SearchAsyncActionTests.MockConnection(replicaNode)); + + for (SortOrder order : SortOrder.values()) { + List shardIds = new ArrayList<>(); + List> minAndMaxes = new ArrayList<>(); + Set shardToSkip = new HashSet<>(); + + SearchTransportService searchTransportService = new SearchTransportService(null, null) { + @Override + public void sendCanMatch(Transport.Connection connection, ShardSearchRequest request, SearchTask task, + ActionListener listener) { + Long min = rarely() ? null : randomLong(); + Long max = min == null ? null : randomLongBetween(min, Long.MAX_VALUE); + MinAndMax minMax = min == null ? null : MinAndMax.newMinMax(min, max); + boolean canMatch = frequently(); + synchronized (shardIds) { + shardIds.add(request.shardId()); + minAndMaxes.add(minMax); + if (canMatch == false) { + shardToSkip.add(request.shardId()); + } + } + new Thread(() -> listener.onResponse(new SearchService.CanMatchResponse(canMatch, minMax))).start(); + } + }; + + AtomicReference> result = new AtomicReference<>(); + CountDownLatch latch = new CountDownLatch(1); + GroupShardsIterator shardsIter = SearchAsyncActionTests.getShardsIter("logs", + new OriginalIndices(new String[]{"logs"}, SearchRequest.DEFAULT_INDICES_OPTIONS), + randomIntBetween(2, 20), randomBoolean(), primaryNode, replicaNode); + final SearchRequest searchRequest = new SearchRequest(); + searchRequest.source(new SearchSourceBuilder().sort(SortBuilders.fieldSort("timestamp").order(order))); + searchRequest.allowPartialSearchResults(true); + + CanMatchPreFilterSearchPhase canMatchPhase = new CanMatchPreFilterSearchPhase(logger, + searchTransportService, + (clusterAlias, node) -> lookup.get(node), + Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY)), + Collections.emptyMap(), Collections.emptyMap(), EsExecutors.newDirectExecutorService(), + searchRequest, null, shardsIter, timeProvider, 0, null, + (iter) -> new SearchPhase("test") { + @Override + public void run() { + result.set(iter); + latch.countDown(); + } + }, SearchResponse.Clusters.EMPTY); + + canMatchPhase.start(); + latch.await(); + ShardId[] expected = IntStream.range(0, shardIds.size()) + .boxed() + .sorted(Comparator.comparing(minAndMaxes::get, MinAndMax.getComparator(order)).thenComparing(shardIds::get)) + .map(shardIds::get) + .toArray(ShardId[]::new); + + int pos = 0; + for (SearchShardIterator i : result.get()) { + assertEquals(shardToSkip.contains(i.shardId()), i.skip()); + assertEquals(expected[pos++], i.shardId()); + } + } + } } diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java index c09b2a80e5db4..4aa819f384bc9 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java @@ -601,28 +601,28 @@ public void testCanMatch() throws IOException { SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true); int numWrapReader = numWrapInvocations.get(); assertTrue(service.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); searchRequest.source(new SearchSourceBuilder()); assertTrue(service.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); searchRequest.source(new SearchSourceBuilder().query(new MatchAllQueryBuilder())); assertTrue(service.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); searchRequest.source(new SearchSourceBuilder().query(new MatchNoneQueryBuilder()) .aggregation(new TermsAggregationBuilder("test", ValueType.STRING).minDocCount(0))); assertTrue(service.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); searchRequest.source(new SearchSourceBuilder().query(new MatchNoneQueryBuilder()) .aggregation(new GlobalAggregationBuilder("test"))); assertTrue(service.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); searchRequest.source(new SearchSourceBuilder().query(new MatchNoneQueryBuilder())); assertFalse(service.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, indexShard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); assertEquals(numWrapReader, numWrapInvocations.get()); // make sure that the wrapper is called when the context is actually created diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 28ca23df12401..396c05b311450 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.sort; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.SortField; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -182,7 +183,11 @@ public void testEqualsAndHashcode() { } } - protected QueryShardContext createMockShardContext() { + protected final QueryShardContext createMockShardContext() { + return createMockShardContext(null); + } + + protected final QueryShardContext createMockShardContext(IndexSearcher searcher) { Index index = new Index(randomAlphaOfLengthBetween(1, 10), "_na_"); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build()); @@ -192,7 +197,8 @@ protected QueryShardContext createMockShardContext() { return builder.build(idxSettings, fieldType, new IndexFieldDataCache.None(), null, null); }; return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, - null, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, null, () -> randomNonNegativeLong(), null, null) { + null, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, searcher, + () -> randomNonNegativeLong(), null, null) { @Override public MappedFieldType fieldMapper(String name) { diff --git a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java index ee9c8f8ed1105..bb713bc6c1891 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java @@ -19,20 +19,37 @@ package org.elasticsearch.search.sort; +import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.HalfFloatPoint; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.search.AssertingIndexSearcher; import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortedNumericSelector; import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.search.SortedSetSelector; import org.apache.lucene.search.SortedSetSortField; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.xcontent.XContentParseException; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.TypeFieldMapper; import org.elasticsearch.index.query.MatchNoneQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -43,12 +60,16 @@ import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; +import static org.elasticsearch.search.sort.FieldSortBuilder.getMinMaxOrNull; +import static org.elasticsearch.search.sort.FieldSortBuilder.getPrimaryFieldSortOrNull; import static org.elasticsearch.search.sort.NestedSortBuilderTests.createRandomNestedSort; import static org.hamcrest.Matchers.instanceOf; @@ -329,6 +350,32 @@ protected MappedFieldType provideMappedFieldType(String name) { fieldType.setName(name); fieldType.setHasDocValues(true); return fieldType; + } else if (name.startsWith("custom-")) { + final MappedFieldType fieldType; + if (name.startsWith("custom-keyword")) { + fieldType = new KeywordFieldMapper.KeywordFieldType(); + } else if (name.startsWith("custom-date")) { + fieldType = new DateFieldMapper.DateFieldType(); + } else { + String type = name.split("-")[1]; + if (type.equals("INT")) { + type = "integer"; + } + NumberFieldMapper.NumberType numberType = NumberFieldMapper.NumberType.valueOf(type.toUpperCase(Locale.ENGLISH)); + if (numberType != null) { + fieldType = new NumberFieldMapper.NumberFieldType(numberType); + } else { + fieldType = new KeywordFieldMapper.KeywordFieldType(); + } + } + fieldType.setName(name); + fieldType.setHasDocValues(true); + if (name.endsWith("-ni")) { + fieldType.setIndexOptions(IndexOptions.NONE); + } else { + fieldType.setIndexOptions(IndexOptions.DOCS); + } + return fieldType; } else { return super.provideMappedFieldType(name); } @@ -414,6 +461,147 @@ public QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOEx assertNotSame(rangeQuery, rewritten.getNestedSort().getFilter()); } + public void testGetPrimaryFieldSort() { + assertNull(getPrimaryFieldSortOrNull(null)); + assertNull(getPrimaryFieldSortOrNull(new SearchSourceBuilder())); + assertNull(getPrimaryFieldSortOrNull(new SearchSourceBuilder().sort(SortBuilders.scoreSort()))); + FieldSortBuilder sortBuilder = new FieldSortBuilder(MAPPED_STRING_FIELDNAME); + assertEquals(sortBuilder, getPrimaryFieldSortOrNull(new SearchSourceBuilder().sort(sortBuilder))); + assertNull(getPrimaryFieldSortOrNull(new SearchSourceBuilder() + .sort(SortBuilders.scoreSort()).sort(sortBuilder))); + assertNull(getPrimaryFieldSortOrNull(new SearchSourceBuilder() + .sort(SortBuilders.geoDistanceSort("field", 0d, 0d)).sort(sortBuilder))); + } + + public void testGetMaxNumericSortValue() throws IOException { + QueryShardContext context = createMockShardContext(); + for (NumberFieldMapper.NumberType numberType : NumberFieldMapper.NumberType.values()) { + String fieldName = "custom-" + numberType.numericType(); + assertNull(getMinMaxOrNull(context, SortBuilders.fieldSort(fieldName))); + assertNull(getMinMaxOrNull(context, SortBuilders.fieldSort(fieldName + "-ni"))); + + try (Directory dir = newDirectory()) { + int numDocs = randomIntBetween(10, 30); + final Comparable[] values = new Comparable[numDocs]; + try (RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) { + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + switch (numberType) { + case LONG: + long v1 = randomLong(); + values[i] = v1; + doc.add(new LongPoint(fieldName, v1)); + break; + + case INTEGER: + int v2 = randomInt(); + values[i] = (long) v2; + doc.add(new IntPoint(fieldName, v2)); + break; + + case DOUBLE: + double v3 = randomDouble(); + values[i] = v3; + doc.add(new DoublePoint(fieldName, v3)); + break; + + case FLOAT: + float v4 = randomFloat(); + values[i] = v4; + doc.add(new FloatPoint(fieldName, v4)); + break; + + case HALF_FLOAT: + float v5 = randomFloat(); + values[i] = (double) v5; + doc.add(new HalfFloatPoint(fieldName, v5)); + break; + + case BYTE: + byte v6 = randomByte(); + values[i] = (long) v6; + doc.add(new IntPoint(fieldName, v6)); + break; + + case SHORT: + short v7 = randomShort(); + values[i] = (long) v7; + doc.add(new IntPoint(fieldName, v7)); + break; + + default: + throw new AssertionError("unknown type " + numberType); + } + writer.addDocument(doc); + } + Arrays.sort(values); + try (DirectoryReader reader = writer.getReader()) { + QueryShardContext newContext = createMockShardContext(new AssertingIndexSearcher(random(), reader)); + if (numberType == NumberFieldMapper.NumberType.HALF_FLOAT) { + assertNull(getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName + "-ni"))); + assertNull(getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName))); + } else { + assertNull(getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName + "-ni"))); + assertEquals(values[numDocs - 1], + getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName)).getMax()); + assertEquals(values[0], getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName)).getMin()); + } + } + } + } + } + } + + public void testGetMaxNumericDateValue() throws IOException { + QueryShardContext context = createMockShardContext(); + String fieldName = "custom-date"; + assertNull(getMinMaxOrNull(context, SortBuilders.fieldSort(fieldName))); + assertNull(getMinMaxOrNull(context, SortBuilders.fieldSort(fieldName + "-ni"))); + try (Directory dir = newDirectory()) { + int numDocs = randomIntBetween(10, 30); + final long[] values = new long[numDocs]; + try (RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) { + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + values[i] = randomNonNegativeLong(); + doc.add(new LongPoint(fieldName, values[i])); + writer.addDocument(doc); + } + Arrays.sort(values); + try (DirectoryReader reader = writer.getReader()) { + QueryShardContext newContext = createMockShardContext(new AssertingIndexSearcher(random(), reader)); + assertEquals(values[numDocs - 1], getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName)).getMax()); + assertEquals(values[0], getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName)).getMin()); + } + } + } + } + + public void testGetMaxKeywordValue() throws IOException { + QueryShardContext context = createMockShardContext(); + String fieldName = "custom-keyword"; + assertNull(getMinMaxOrNull(context, SortBuilders.fieldSort(fieldName))); + assertNull(getMinMaxOrNull(context, SortBuilders.fieldSort(fieldName + "-ni"))); + try (Directory dir = newDirectory()) { + int numDocs = randomIntBetween(10, 30); + final BytesRef[] values = new BytesRef[numDocs]; + try (RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new KeywordAnalyzer())) { + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + values[i] = new BytesRef(randomAlphaOfLengthBetween(5, 10)); + doc.add(new TextField(fieldName, values[i].utf8ToString(), Field.Store.NO)); + writer.addDocument(doc); + } + Arrays.sort(values); + try (DirectoryReader reader = writer.getReader()) { + QueryShardContext newContext = createMockShardContext(new AssertingIndexSearcher(random(), reader)); + assertEquals(values[numDocs - 1], getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName)).getMax()); + assertEquals(values[0], getMinMaxOrNull(newContext, SortBuilders.fieldSort(fieldName)).getMin()); + } + } + } + } + @Override protected void assertWarnings(FieldSortBuilder testItem) { List expectedWarnings = new ArrayList<>(); diff --git a/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java b/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java index b8e711d25a5c5..396f0b413e366 100644 --- a/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java +++ b/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java @@ -256,17 +256,17 @@ public void testCanMatch() throws IOException, ExecutionException, InterruptedEx SearchService searchService = getInstanceFromNode(SearchService.class); SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true); assertTrue(searchService.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, shard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); searchRequest.source(sourceBuilder); sourceBuilder.query(QueryBuilders.rangeQuery("field").gte("2010-01-03||+2d").lte("2010-01-04||+2d/d")); assertTrue(searchService.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, shard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); sourceBuilder.query(QueryBuilders.rangeQuery("field").gt("2010-01-06T02:00").lt("2010-01-07T02:00")); assertFalse(searchService.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, shard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); } XPackClient xPackClient = new XPackClient(client()); @@ -281,17 +281,17 @@ public void testCanMatch() throws IOException, ExecutionException, InterruptedEx SearchService searchService = getInstanceFromNode(SearchService.class); SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true); assertTrue(searchService.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, shard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); sourceBuilder.query(QueryBuilders.rangeQuery("field").gte("2010-01-03||+2d").lte("2010-01-04||+2d/d")); searchRequest.source(sourceBuilder); assertTrue(searchService.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, shard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); sourceBuilder.query(QueryBuilders.rangeQuery("field").gt("2010-01-06T02:00").lt("2010-01-07T02:00")); assertFalse(searchService.canMatch(new ShardSearchRequest(OriginalIndices.NONE, searchRequest, shard.shardId(), 1, - new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null))); + new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)).canMatch()); IndicesStatsResponse response = client().admin().indices().prepareStats("index").clear().setRefresh(true).get(); assertEquals(0, response.getTotal().refresh.getTotal()); // never opened a reader