From e2ebfb48fab5d86de9137dfe624967c078cd88bf Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Tue, 25 Feb 2020 22:31:18 +0100 Subject: [PATCH 01/26] Implement Cancellable DirectoryReader Implement a Cancellable DirectoryReader that wraps the original DirectoryReader so that when a search task is cancelled the DirectoryReaders also stop their work fast. This is usuful for expensive operations like wilcard/prefix queries where the DirectoryReaders can spend lots of time and consume resources, as previously their work wouldn't stop even though the original search task was cancelled (e.g. because of timeout or dropped client connection). --- .../search/DefaultSearchContext.java | 2 +- .../elasticsearch/search/SearchService.java | 2 + .../search/internal/ContextIndexSearcher.java | 119 +++++++++++++++++- .../BestBucketsDeferringCollectorTests.java | 1 + .../bucket/filter/FilterAggregatorTests.java | 2 +- .../geogrid/GeoGridAggregatorTestCase.java | 2 +- .../DateHistogramAggregatorTests.java | 2 +- .../missing/MissingAggregatorTests.java | 3 +- .../metrics/AvgAggregatorTests.java | 27 ++-- .../HDRPercentileRanksAggregatorTests.java | 13 +- .../metrics/MaxAggregatorTests.java | 35 ++---- .../metrics/MinAggregatorTests.java | 13 +- ...TDigestPercentileRanksAggregatorTests.java | 9 +- .../metrics/WeightedAvgAggregatorTests.java | 2 +- .../search/query/QueryPhaseTests.java | 6 +- .../aggregations/AggregatorTestCase.java | 23 ++-- 16 files changed, 183 insertions(+), 78 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 17f3e27f96b25..24ab7f5e8bf43 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -155,7 +155,7 @@ final class DefaultSearchContext extends SearchContext { DefaultSearchContext(long id, ShardSearchRequest request, SearchShardTarget shardTarget, Engine.Searcher engineSearcher, ClusterService clusterService, IndexService indexService, IndexShard indexShard, BigArrays bigArrays, LongSupplier relativeTimeSupplier, TimeValue timeout, - FetchPhase fetchPhase) { + FetchPhase fetchPhase) throws IOException { this.id = id; this.request = request; this.fetchPhase = fetchPhase; diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index c6531fabf264d..d09ae01be57ef 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -708,6 +708,8 @@ private DefaultSearchContext createSearchContext(SearchRewriteContext rewriteCon searcher, clusterService, indexService, indexShard, bigArrays, threadPool::relativeTimeInMillis, timeout, fetchPhase); success = true; return searchContext; + } catch (IOException e) { + return null; } finally { if (success == false) { // we handle the case where `IndicesService#indexServiceSafe`or `IndexService#getShard`, or the DefaultSearchContext diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 8959280ad21a1..e949ce9b2cecb 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -19,9 +19,18 @@ package org.elasticsearch.search.internal; +import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.ExitableDirectoryReader; +import org.apache.lucene.index.FilterDirectoryReader; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.QueryTimeout; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.Term; import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.CollectionStatistics; @@ -64,6 +73,7 @@ import java.util.Arrays; import java.util.List; import java.util.Set; +import java.util.function.Supplier; /** * Context-aware extension of {@link IndexSearcher}. @@ -79,8 +89,18 @@ public class ContextIndexSearcher extends IndexSearcher { private QueryProfiler profiler; private Runnable checkCancelled; - public ContextIndexSearcher(IndexReader reader, Similarity similarity, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy) { - super(reader); + public ContextIndexSearcher(IndexReader reader, Similarity similarity, + QueryCache queryCache, QueryCachingPolicy queryCachingPolicy) throws IOException { + this(reader, similarity, queryCache, queryCachingPolicy, true); + } + + public ContextIndexSearcher(IndexReader reader, Similarity similarity, + QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, + boolean shouldWrap) throws IOException { + super(shouldWrap? new CancellableIndexReader((DirectoryReader) reader, new Cancellable()) : reader); + if (shouldWrap) { + ((CancellableIndexReader) getIndexReader()).setCheckCancelled(() -> checkCancelled); + } setSimilarity(similarity); setQueryCache(queryCache); setQueryCachingPolicy(queryCachingPolicy); @@ -320,4 +340,99 @@ public DirectoryReader getDirectoryReader() { assert reader instanceof DirectoryReader : "expected an instance of DirectoryReader, got " + reader.getClass(); return (DirectoryReader) reader; } + + /** + * Wraps an {@link IndexReader} with a cancellation Runnable task. + */ + private static class CancellableIndexReader extends FilterDirectoryReader { + + private final Cancellable checkCancelled; + + private CancellableIndexReader(DirectoryReader in, Cancellable checkCancelled) throws IOException { + super(in, new SubReaderWrapper() { + @Override + public LeafReader wrap(LeafReader reader) { + return new CancellableLeafReader(reader, checkCancelled); + } + }); + this.checkCancelled = checkCancelled; + } + + private void setCheckCancelled(Supplier checkCancelled) { + this.checkCancelled.setCancellable(checkCancelled); + } + + @Override + protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) { + return in; + } + + @Override + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); + } + } + + /** + * Wraps a leaf reader with a cancellable task + */ + private static class CancellableLeafReader extends ExitableDirectoryReader.ExitableFilterAtomicReader { + + private CancellableLeafReader(LeafReader leafReader, Cancellable checkCancelled) { + super(leafReader, checkCancelled); + } + + @Override + public NumericDocValues getNumericDocValues(String field) throws IOException { + return in.getNumericDocValues(field); + } + + @Override + public BinaryDocValues getBinaryDocValues(String field) throws IOException { + return in.getBinaryDocValues(field); + } + + @Override + public SortedDocValues getSortedDocValues(String field) throws IOException { + return in.getSortedDocValues(field); + } + + @Override + public SortedNumericDocValues getSortedNumericDocValues(String field) throws IOException { + return in.getSortedNumericDocValues(field); + } + + @Override + public SortedSetDocValues getSortedSetDocValues(String field) throws IOException { + return in.getSortedSetDocValues(field); + } + } + + /** + * Implementation of {@link QueryTimeout} with a Runnable task. + */ + private static class Cancellable implements QueryTimeout { + + private Supplier cancellable; + + public void setCancellable(Supplier cancellable) { + this.cancellable = cancellable; + } + + @Override + public boolean shouldExit() { + assert cancellable != null : "checkCancelled must be set immediately after the construction of CancellableIndexReader"; + if (cancellable.get() == null) { + return false; + } + cancellable.get().run(); + return false; + } + + @Override + public boolean isTimeoutEnabled() { + assert cancellable != null : "checkCancelled must be set immediately after the construction of CancellableIndexReader"; + return cancellable.get() != null; + } + } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollectorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollectorTests.java index 74ee21b2466cd..47471afc8efa6 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollectorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollectorTests.java @@ -68,6 +68,7 @@ public void testReplay() throws Exception { TopDocs topDocs = indexSearcher.search(termQuery, numDocs); SearchContext searchContext = createSearchContext(indexSearcher, createIndexSettings(), rewrittenQuery, null); + indexSearcher = searchContext.searcher(); when(searchContext.query()).thenReturn(rewrittenQuery); BestBucketsDeferringCollector collector = new BestBucketsDeferringCollector(searchContext, false) { @Override diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java index 56153d2d35f13..5998413d238dd 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java @@ -84,7 +84,7 @@ public void testRandom() throws Exception { indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); try { int value = randomInt(maxTerm - 1); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java index 2059f465c3756..c409bdc3bbb0c 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java @@ -219,7 +219,7 @@ private void testCase(Query query, int precision, GeoBoundingBox geoBoundingBox, indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); aggregationBuilder.precision(precision); if (geoBoundingBox != null) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java index 4eba98dd49b9d..68bdd96ab3b13 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java @@ -1235,7 +1235,7 @@ private void executeTestCase(boolean reduced, } try (IndexReader indexReader = DirectoryReader.open(directory)) { - IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); DateHistogramAggregationBuilder aggregationBuilder = new DateHistogramAggregationBuilder("_name"); if (configure != null) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java index dbc3ac3b490c2..a693c2ee2c685 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java @@ -175,8 +175,7 @@ private void executeTestCase(int numDocs, } try (IndexReader indexReader = DirectoryReader.open(directory)) { - IndexSearcher indexSearcher = - newSearcher(indexReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null); builder.field(fieldName); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java index 46f08d786a856..e629b2acb6471 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java @@ -261,6 +261,7 @@ private void verifyAvgOfDoubles(double[] values, double expected, double delta) ); } + @AwaitsFix(bugUrl = "Replace with integration test") public void testSingleValuedFieldPartiallyUnmapped() throws IOException { Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); @@ -537,7 +538,7 @@ public void testOrderByEmptyAggregation() throws IOException { indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); TermsAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); aggregator.preCollection(); @@ -587,7 +588,7 @@ private void testCase(AvgAggregationBuilder aggregationBuilder, Query query, indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); AvgAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); aggregator.preCollection(); @@ -611,14 +612,8 @@ public void testCacheAggregation() throws IOException { } indexWriter.close(); - Directory unmappedDirectory = newDirectory(); - RandomIndexWriter unmappedIndexWriter = new RandomIndexWriter(random(), unmappedDirectory); - unmappedIndexWriter.close(); - IndexReader indexReader = DirectoryReader.open(directory); - IndexReader unamappedIndexReader = DirectoryReader.open(unmappedDirectory); - MultiReader multiReader = new MultiReader(indexReader, unamappedIndexReader); - IndexSearcher indexSearcher = newSearcher(multiReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER); fieldType.setName("value"); @@ -639,9 +634,8 @@ public void testCacheAggregation() throws IOException { // Test that an aggregation not using a script does get cached assertTrue(aggregator.context().getQueryShardContext().isCacheable()); - multiReader.close(); + indexReader.close(); directory.close(); - unmappedDirectory.close(); } /** @@ -657,14 +651,8 @@ public void testScriptCaching() throws IOException { } indexWriter.close(); - Directory unmappedDirectory = newDirectory(); - RandomIndexWriter unmappedIndexWriter = new RandomIndexWriter(random(), unmappedDirectory); - unmappedIndexWriter.close(); - IndexReader indexReader = DirectoryReader.open(directory); - IndexReader unamappedIndexReader = DirectoryReader.open(unmappedDirectory); - MultiReader multiReader = new MultiReader(indexReader, unamappedIndexReader); - IndexSearcher indexSearcher = newSearcher(multiReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER); fieldType.setName("value"); @@ -705,8 +693,7 @@ public void testScriptCaching() throws IOException { // Test that an aggregation using a nondeterministic script does not get cached assertFalse(aggregator.context().getQueryShardContext().isCacheable()); - multiReader.close(); + indexReader.close(); directory.close(); - unmappedDirectory.close(); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java index 7ed434f3a202d..2b202de79f6c3 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java @@ -22,7 +22,6 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; @@ -62,13 +61,19 @@ public void testEmpty() throws IOException { .method(PercentilesMethod.HDR); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE); fieldType.setName("field"); - try (IndexReader reader = new MultiReader()) { + + Directory directory = newDirectory(); + RandomIndexWriter unmappedIndexWriter = new RandomIndexWriter(random(), directory); + try (IndexReader reader = unmappedIndexWriter.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); PercentileRanks ranks = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); Percentile rank = ranks.iterator().next(); assertEquals(Double.NaN, rank.getPercent(), 0d); assertEquals(0.5, rank.getValue(), 0d); assertFalse(AggregationInspectionHelper.hasValue((InternalHDRPercentileRanks)ranks)); + } finally { + unmappedIndexWriter.close(); + directory.close(); } } @@ -106,13 +111,13 @@ public void testSimple() throws IOException { } } - public void testNullValues() throws IOException { + public void testNullValues() { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new PercentileRanksAggregationBuilder("my_agg", null).field("field").method(PercentilesMethod.HDR)); assertThat(e.getMessage(), Matchers.equalTo("[values] must not be null: [my_agg]")); } - public void testEmptyValues() throws IOException { + public void testEmptyValues() { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new PercentileRanksAggregationBuilder("my_agg", new double[0]).field("field").method(PercentilesMethod.HDR)); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java index 598d19795db20..d135210e108cb 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java @@ -290,7 +290,7 @@ private void testCase(MaxAggregationBuilder aggregationBuilder, Query query, buildIndex.accept(indexWriter); indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); MaxAggregator aggregator = createAggregator(query, aggregationBuilder, indexSearcher, createIndexSettings(), fieldType); aggregator.preCollection(); @@ -457,7 +457,7 @@ public void testSingleValuedFieldGetProperty() throws IOException { indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); GlobalAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); aggregator.preCollection(); @@ -483,6 +483,7 @@ public void testSingleValuedFieldGetProperty() throws IOException { directory.close(); } + @AwaitsFix(bugUrl = "Replace with integration test") public void testSingleValuedFieldPartiallyUnmapped() throws IOException { Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); @@ -723,7 +724,7 @@ public void testEmptyAggregation() throws Exception { indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); GlobalAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); aggregator.preCollection(); @@ -766,7 +767,7 @@ public void testOrderByEmptyAggregation() throws IOException { indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); TermsAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); aggregator.preCollection(); @@ -814,7 +815,7 @@ public void testEarlyTermination() throws Exception { indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); MaxAggregationBuilder maxAggregationBuilder = new MaxAggregationBuilder("max") .field("values"); ValueCountAggregationBuilder countAggregationBuilder = new ValueCountAggregationBuilder("count", null) @@ -862,7 +863,7 @@ public void testNestedEarlyTermination() throws Exception { indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); for (Aggregator.SubAggCollectionMode collectionMode : Aggregator.SubAggCollectionMode.values()) { MaxAggregationBuilder maxAggregationBuilder = new MaxAggregationBuilder("max") @@ -920,14 +921,8 @@ public void testCacheAggregation() throws IOException { } indexWriter.close(); - Directory unmappedDirectory = newDirectory(); - RandomIndexWriter unmappedIndexWriter = new RandomIndexWriter(random(), unmappedDirectory); - unmappedIndexWriter.close(); - IndexReader indexReader = DirectoryReader.open(directory); - IndexReader unamappedIndexReader = DirectoryReader.open(unmappedDirectory); - MultiReader multiReader = new MultiReader(indexReader, unamappedIndexReader); - IndexSearcher indexSearcher = newSearcher(multiReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER); @@ -949,9 +944,8 @@ public void testCacheAggregation() throws IOException { // Test that an aggregation not using a script does get cached assertTrue(aggregator.context().getQueryShardContext().isCacheable()); - multiReader.close(); + indexReader.close(); directory.close(); - unmappedDirectory.close(); } /** @@ -967,14 +961,8 @@ public void testScriptCaching() throws Exception { } indexWriter.close(); - Directory unmappedDirectory = newDirectory(); - RandomIndexWriter unmappedIndexWriter = new RandomIndexWriter(random(), unmappedDirectory); - unmappedIndexWriter.close(); - IndexReader indexReader = DirectoryReader.open(directory); - IndexReader unamappedIndexReader = DirectoryReader.open(unmappedDirectory); - MultiReader multiReader = new MultiReader(indexReader, unamappedIndexReader); - IndexSearcher indexSearcher = newSearcher(multiReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER); fieldType.setName("value"); @@ -1014,8 +1002,7 @@ public void testScriptCaching() throws Exception { // Test that an aggregation using a nondeterministic script does not get cached assertFalse(aggregator.context().getQueryShardContext().isCacheable()); - multiReader.close(); + indexReader.close(); directory.close(); - unmappedDirectory.close(); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java index 32c539e43348a..a25c9ef5c8bc5 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java @@ -393,6 +393,7 @@ public void testGetProperty() throws IOException { }, fieldType); } + @AwaitsFix(bugUrl = "Replace with integration test") public void testSingleValuedFieldPartiallyUnmapped() throws IOException { MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER); @@ -424,6 +425,7 @@ public void testSingleValuedFieldPartiallyUnmapped() throws IOException { } } + @AwaitsFix(bugUrl = "Replace with integration test") public void testSingleValuedFieldPartiallyUnmappedWithMissing() throws IOException { MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER); @@ -647,7 +649,7 @@ public void testCaching() throws IOException { try (IndexReader indexReader = DirectoryReader.open(directory)) { - IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); InternalMin min = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); assertEquals(2.0, min.getValue(), 0); @@ -679,7 +681,7 @@ public void testScriptCaching() throws IOException { try (IndexReader indexReader = DirectoryReader.open(directory)) { - IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); InternalMin min = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), nonDeterministicAggregationBuilder, fieldType); assertTrue(min.getValue() >= 0.0 && min.getValue() <= 1.0); @@ -687,7 +689,7 @@ public void testScriptCaching() throws IOException { assertFalse(queryShardContext.isCacheable()); - indexSearcher = newSearcher(indexReader, true, true); + indexSearcher = newIndexSearcher(indexReader); min = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); assertEquals(-7.0, min.getValue(), 0); @@ -925,14 +927,11 @@ private void testC indexWriter.close(); try (IndexReader indexReader = DirectoryReader.open(directory)) { - IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); V agg = searchAndReduce(indexSearcher, query, aggregationBuilder, fieldType); verify.accept(agg); - } } } - - } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java index 420c9b8cecbcf..047a288250e51 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java @@ -22,7 +22,6 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; @@ -62,13 +61,19 @@ public void testEmpty() throws IOException { .method(PercentilesMethod.TDIGEST); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE); fieldType.setName("field"); - try (IndexReader reader = new MultiReader()) { + + Directory directory = newDirectory(); + RandomIndexWriter unmappedIndexWriter = new RandomIndexWriter(random(), directory); + try (IndexReader reader = unmappedIndexWriter.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); PercentileRanks ranks = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); Percentile rank = ranks.iterator().next(); assertEquals(Double.NaN, rank.getPercent(), 0d); assertEquals(0.5, rank.getValue(), 0d); assertFalse(AggregationInspectionHelper.hasValue(((InternalTDigestPercentileRanks)ranks))); + } finally { + unmappedIndexWriter.close(); + directory.close(); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregatorTests.java index 78c4d18218eea..9f9bb3bb94912 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregatorTests.java @@ -420,7 +420,7 @@ private void testCase(Query query, WeightedAvgAggregationBuilder aggregationBuil buildIndex.accept(indexWriter); indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + IndexSearcher indexSearcher = newIndexSearcher(indexReader); try { MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(fieldNumberType); diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java index cc75e8bef5f48..ea34ae13271a1 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java @@ -828,12 +828,12 @@ public void testMinScore() throws Exception { } - private static ContextIndexSearcher newContextSearcher(IndexReader reader) { + private static ContextIndexSearcher newContextSearcher(IndexReader reader) throws IOException { return new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); } - private static ContextIndexSearcher newEarlyTerminationContextSearcher(IndexReader reader, int size) { + private static ContextIndexSearcher newEarlyTerminationContextSearcher(IndexReader reader, int size) throws IOException { return new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()) { @@ -846,7 +846,7 @@ public void search(List leaves, Weight weight, Collector coll } // used to check that numeric long or date sort optimization was run - private static ContextIndexSearcher newOptimizedContextSearcher(IndexReader reader, int queryType) { + private static ContextIndexSearcher newOptimizedContextSearcher(IndexReader reader, int queryType) throws IOException { return new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()) { diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 2a82d831ee73e..6e1484647e34b 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -40,6 +40,7 @@ import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Weight; +import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; @@ -222,16 +223,18 @@ protected SearchContext createSearchContext(IndexSearcher indexSearcher, IndexSettings indexSettings, Query query, MultiBucketConsumer bucketConsumer, - MappedFieldType... fieldTypes) { - return createSearchContext(indexSearcher, indexSettings, query, bucketConsumer, new NoneCircuitBreakerService(), fieldTypes); + MappedFieldType... fieldTypes) throws IOException { + return createSearchContext(indexSearcher.getIndexReader(), indexSearcher.getSimilarity(), + indexSettings, query, bucketConsumer, new NoneCircuitBreakerService(), fieldTypes); } - protected SearchContext createSearchContext(IndexSearcher indexSearcher, + protected SearchContext createSearchContext(IndexReader indexReader, + Similarity similarity, IndexSettings indexSettings, Query query, MultiBucketConsumer bucketConsumer, CircuitBreakerService circuitBreakerService, - MappedFieldType... fieldTypes) { + MappedFieldType... fieldTypes) throws IOException { QueryCache queryCache = new DisabledQueryCache(indexSettings); QueryCachingPolicy queryCachingPolicy = new QueryCachingPolicy() { @Override @@ -244,8 +247,8 @@ public boolean shouldCache(Query query) { return false; } }; - ContextIndexSearcher contextIndexSearcher = new ContextIndexSearcher(indexSearcher.getIndexReader(), - indexSearcher.getSimilarity(), queryCache, queryCachingPolicy); + ContextIndexSearcher contextIndexSearcher = new ContextIndexSearcher(indexReader, + similarity, queryCache, queryCachingPolicy, false); SearchContext searchContext = mock(SearchContext.class); when(searchContext.numberOfShards()).thenReturn(1); @@ -437,6 +440,11 @@ protected A searchAndReduc AggregationBuilder builder, int maxBucket, MappedFieldType... fieldTypes) throws IOException { + MultiBucketConsumer bucketConsumer = new MultiBucketConsumer(maxBucket, + new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)); + C root = createAggregator(query, builder, searcher, bucketConsumer, fieldTypes); + searcher = root.context().searcher(); + final IndexReaderContext ctx = searcher.getTopReaderContext(); final ShardSearcher[] subSearchers; @@ -456,9 +464,6 @@ protected A searchAndReduc List aggs = new ArrayList<> (); Query rewritten = searcher.rewrite(query); Weight weight = searcher.createWeight(rewritten, ScoreMode.COMPLETE, 1f); - MultiBucketConsumer bucketConsumer = new MultiBucketConsumer(maxBucket, - new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)); - C root = createAggregator(query, builder, searcher, bucketConsumer, fieldTypes); for (ShardSearcher subSearcher : subSearchers) { MultiBucketConsumer shardBucketConsumer = new MultiBucketConsumer(maxBucket, From c8901426c48505594b333392bc7d41008644f11e Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Wed, 26 Feb 2020 16:49:41 +0100 Subject: [PATCH 02/26] fix compilation --- .../analytics/topmetrics/TopMetricsAggregatorTests.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java index e870bd870fe47..4f1c2d31aa6ab 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java @@ -339,8 +339,9 @@ public long addWithoutBreaking(long bytes) { try (IndexReader indexReader = DirectoryReader.open(directory)) { IndexSearcher indexSearcher = newSearcher(indexReader, false, false); - SearchContext searchContext = createSearchContext(indexSearcher, createIndexSettings(), new MatchAllDocsQuery(), - new MultiBucketConsumer(Integer.MAX_VALUE, breaker.getBreaker(CircuitBreaker.REQUEST)), breaker, doubleFields()); + SearchContext searchContext = createSearchContext(indexReader, indexSearcher.getSimilarity(), createIndexSettings(), + new MatchAllDocsQuery(), new MultiBucketConsumer(Integer.MAX_VALUE, breaker.getBreaker(CircuitBreaker.REQUEST)), + breaker, doubleFields()); TopMetricsAggregationBuilder builder = simpleBuilder(new FieldSortBuilder("s").order(SortOrder.ASC)); Aggregator aggregator = builder.build(searchContext.getQueryShardContext(), null) .create(searchContext, null, true); From 73b0e6d0a4cba0871eb0e941425b0f25d43efa7e Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Wed, 26 Feb 2020 17:20:54 +0100 Subject: [PATCH 03/26] Address comments --- .../search/internal/ContextIndexSearcher.java | 11 +++++++---- .../TDigestPercentileRanksAggregatorTests.java | 11 +++++------ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index e949ce9b2cecb..b1d84f27d25a1 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -94,12 +94,15 @@ public ContextIndexSearcher(IndexReader reader, Similarity similarity, this(reader, similarity, queryCache, queryCachingPolicy, true); } + // TODO: Remove the 2nd constructor and the shouldWrap so that we always wrap the IndexReader. + // Some issues must be fixed regarding tests deriving from AggregatorTestCase and more specifically + // the use of searchAndReduce and the ShardSearcher sub-searchers. public ContextIndexSearcher(IndexReader reader, Similarity similarity, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, boolean shouldWrap) throws IOException { - super(shouldWrap? new CancellableIndexReader((DirectoryReader) reader, new Cancellable()) : reader); + super(shouldWrap? new CancellableDirectoryReader((DirectoryReader) reader, new Cancellable()) : reader); if (shouldWrap) { - ((CancellableIndexReader) getIndexReader()).setCheckCancelled(() -> checkCancelled); + ((CancellableDirectoryReader) getIndexReader()).setCheckCancelled(() -> checkCancelled); } setSimilarity(similarity); setQueryCache(queryCache); @@ -344,11 +347,11 @@ public DirectoryReader getDirectoryReader() { /** * Wraps an {@link IndexReader} with a cancellation Runnable task. */ - private static class CancellableIndexReader extends FilterDirectoryReader { + private static class CancellableDirectoryReader extends FilterDirectoryReader { private final Cancellable checkCancelled; - private CancellableIndexReader(DirectoryReader in, Cancellable checkCancelled) throws IOException { + private CancellableDirectoryReader(DirectoryReader in, Cancellable checkCancelled) throws IOException { super(in, new SubReaderWrapper() { @Override public LeafReader wrap(LeafReader reader) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java index 047a288250e51..001736d49c9df 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java @@ -62,18 +62,17 @@ public void testEmpty() throws IOException { MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE); fieldType.setName("field"); - Directory directory = newDirectory(); - RandomIndexWriter unmappedIndexWriter = new RandomIndexWriter(random(), directory); - try (IndexReader reader = unmappedIndexWriter.getReader()) { + + + try (Directory directory = newDirectory(); + RandomIndexWriter unmappedIndexWriter = new RandomIndexWriter(random(), directory); + IndexReader reader = unmappedIndexWriter.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); PercentileRanks ranks = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); Percentile rank = ranks.iterator().next(); assertEquals(Double.NaN, rank.getPercent(), 0d); assertEquals(0.5, rank.getValue(), 0d); assertFalse(AggregationInspectionHelper.hasValue(((InternalTDigestPercentileRanks)ranks))); - } finally { - unmappedIndexWriter.close(); - directory.close(); } } From ffdf6d2008be37ffd588df5f5648d0794ae9b5a8 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Wed, 26 Feb 2020 17:44:09 +0100 Subject: [PATCH 04/26] Fix behaviour by properly handling cancellable.run() --- .../search/internal/ContextIndexSearcher.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index b1d84f27d25a1..89cfae759d8b1 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -428,8 +428,12 @@ public boolean shouldExit() { if (cancellable.get() == null) { return false; } - cancellable.get().run(); - return false; + try { + cancellable.get().run(); + return false; + } catch (Exception e) { + return true; + } } @Override From add7dd448fb6c219ab3788c338ab169b7716d2a4 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Thu, 27 Feb 2020 18:07:22 +0100 Subject: [PATCH 05/26] split query timeout and cancellation to be able to unset query timeout --- .../elasticsearch/search/SearchService.java | 10 +-- .../search/internal/ContextIndexSearcher.java | 84 +++++++++++-------- .../search/query/QueryPhase.java | 38 +++++---- .../search/SearchCancellationTests.java | 2 +- .../aggregations/AggregatorTestCase.java | 2 +- 5 files changed, 77 insertions(+), 59 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index d09ae01be57ef..49b560dcd5b58 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -398,7 +398,7 @@ && canRewriteToMatchNone(rewritten.source()) }, listener::onFailure)); } - private void onMatchNoDocs(SearchRewriteContext rewriteContext, ActionListener listener) { + private void onMatchNoDocs(SearchRewriteContext rewriteContext, ActionListener listener) throws IOException { // creates a lightweight search context that we use to inform context listeners // before closing SearchContext searchContext = createSearchContext(rewriteContext, defaultSearchTimeout); @@ -609,7 +609,7 @@ private SearchContext findContext(long id, TransportRequest request) throws Sear } } - final SearchContext createAndPutContext(SearchRewriteContext rewriteContext) { + final SearchContext createAndPutContext(SearchRewriteContext rewriteContext) throws IOException { SearchContext context = createContext(rewriteContext); onNewContext(context); boolean success = false; @@ -644,7 +644,7 @@ private void onNewContext(SearchContext context) { } } - final SearchContext createContext(SearchRewriteContext rewriteContext) { + final SearchContext createContext(SearchRewriteContext rewriteContext) throws IOException { final DefaultSearchContext context = createSearchContext(rewriteContext, defaultSearchTimeout); try { if (rewriteContext.request != null && openScrollContexts.get() >= maxOpenScrollContext) { @@ -695,7 +695,7 @@ public DefaultSearchContext createSearchContext(ShardSearchRequest request, Time return createSearchContext(rewriteContext.wrapSearcher(), timeout); } - private DefaultSearchContext createSearchContext(SearchRewriteContext rewriteContext, TimeValue timeout) { + private DefaultSearchContext createSearchContext(SearchRewriteContext rewriteContext, TimeValue timeout) throws IOException { boolean success = false; try { final ShardSearchRequest request = rewriteContext.request; @@ -708,8 +708,6 @@ private DefaultSearchContext createSearchContext(SearchRewriteContext rewriteCon searcher, clusterService, indexService, indexShard, bigArrays, threadPool::relativeTimeInMillis, timeout, fetchPhase); success = true; return searchContext; - } catch (IOException e) { - return null; } finally { if (success == false) { // we handle the case where `IndicesService#indexServiceSafe`or `IndexService#getShard`, or the DefaultSearchContext diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 89cfae759d8b1..88637512f8a21 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -73,7 +73,6 @@ import java.util.Arrays; import java.util.List; import java.util.Set; -import java.util.function.Supplier; /** * Context-aware extension of {@link IndexSearcher}. @@ -88,22 +87,22 @@ public class ContextIndexSearcher extends IndexSearcher { private AggregatedDfs aggregatedDfs; private QueryProfiler profiler; private Runnable checkCancelled; + private Runnable checkTimeout; + private DirReaderCancellable dirReaderCancellable; public ContextIndexSearcher(IndexReader reader, Similarity similarity, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy) throws IOException { - this(reader, similarity, queryCache, queryCachingPolicy, true); + this(reader, similarity, queryCache, queryCachingPolicy, new DirReaderCancellable()); } - // TODO: Remove the 2nd constructor and the shouldWrap so that we always wrap the IndexReader. + // TODO: Make the 2nd constructor private so that the dirCancellable is never null and the IndexReader is always wrapped. // Some issues must be fixed regarding tests deriving from AggregatorTestCase and more specifically // the use of searchAndReduce and the ShardSearcher sub-searchers. public ContextIndexSearcher(IndexReader reader, Similarity similarity, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, - boolean shouldWrap) throws IOException { - super(shouldWrap? new CancellableDirectoryReader((DirectoryReader) reader, new Cancellable()) : reader); - if (shouldWrap) { - ((CancellableDirectoryReader) getIndexReader()).setCheckCancelled(() -> checkCancelled); - } + DirReaderCancellable dirReaderCancellable) throws IOException { + super(dirReaderCancellable != null? new CancellableDirectoryReader((DirectoryReader) reader, dirReaderCancellable) : reader); + this.dirReaderCancellable = dirReaderCancellable; setSimilarity(similarity); setQueryCache(queryCache); setQueryCachingPolicy(queryCachingPolicy); @@ -115,10 +114,24 @@ public void setProfiler(QueryProfiler profiler) { /** * Set a {@link Runnable} that will be run on a regular basis while - * collecting documents. + * collecting documents and check for query cancellation. */ public void setCheckCancelled(Runnable checkCancelled) { this.checkCancelled = checkCancelled; + this.dirReaderCancellable.setCheckCancelled(checkCancelled); + } + + /** + * Set a {@link Runnable} that will be run on a regular basis while + * collecting documents and check for query timeout. + */ + public void setCheckTimeout(Runnable checkTimeout) { + this.checkTimeout = checkTimeout; + this.dirReaderCancellable.setCheckTimeout(checkTimeout); + } + + public void unsetCheckTimeout() { + this.dirReaderCancellable.unsetCheckTimeout(); } public void setAggregatedDfs(AggregatedDfs aggregatedDfs) { @@ -166,6 +179,9 @@ private void checkCancelled() { if (checkCancelled != null) { checkCancelled.run(); } + if (checkTimeout != null) { + checkTimeout.run(); + } } @SuppressWarnings({"unchecked", "rawtypes"}) @@ -349,20 +365,13 @@ public DirectoryReader getDirectoryReader() { */ private static class CancellableDirectoryReader extends FilterDirectoryReader { - private final Cancellable checkCancelled; - - private CancellableDirectoryReader(DirectoryReader in, Cancellable checkCancelled) throws IOException { + private CancellableDirectoryReader(DirectoryReader in, DirReaderCancellable cancellable) throws IOException { super(in, new SubReaderWrapper() { @Override public LeafReader wrap(LeafReader reader) { - return new CancellableLeafReader(reader, checkCancelled); + return new CancellableLeafReader(reader, cancellable); } }); - this.checkCancelled = checkCancelled; - } - - private void setCheckCancelled(Supplier checkCancelled) { - this.checkCancelled.setCancellable(checkCancelled); } @Override @@ -381,7 +390,7 @@ public CacheHelper getReaderCacheHelper() { */ private static class CancellableLeafReader extends ExitableDirectoryReader.ExitableFilterAtomicReader { - private CancellableLeafReader(LeafReader leafReader, Cancellable checkCancelled) { + private CancellableLeafReader(LeafReader leafReader, DirReaderCancellable checkCancelled) { super(leafReader, checkCancelled); } @@ -412,34 +421,41 @@ public SortedSetDocValues getSortedSetDocValues(String field) throws IOException } /** - * Implementation of {@link QueryTimeout} with a Runnable task. + * Implementation of {@link QueryTimeout} with Runnable tasks. */ - private static class Cancellable implements QueryTimeout { + private static class DirReaderCancellable implements QueryTimeout { + + private Runnable checkCancelled = null; + private Runnable checkTimeout = null; + + private void setCheckCancelled(Runnable checkCancelled) { + this.checkCancelled = checkCancelled; + } - private Supplier cancellable; + private void setCheckTimeout(Runnable checkTimeout) { + this.checkTimeout = checkTimeout; + } - public void setCancellable(Supplier cancellable) { - this.cancellable = cancellable; + private void unsetCheckTimeout() { + this.checkTimeout = null; } @Override public boolean shouldExit() { - assert cancellable != null : "checkCancelled must be set immediately after the construction of CancellableIndexReader"; - if (cancellable.get() == null) { - return false; + if (checkCancelled != null) { + checkCancelled.run(); } - try { - cancellable.get().run(); - return false; - } catch (Exception e) { - return true; + if (checkTimeout != null) { + checkTimeout.run(); } + // Always return false and rely on checkCancelled or checkTimeout to throw + // the appropriate exception + return false; } @Override public boolean isTimeoutEnabled() { - assert cancellable != null : "checkCancelled must be set immediately after the construction of CancellableIndexReader"; - return cancellable.get() != null; + return checkCancelled != null || checkTimeout != null; } } } diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 64e274e734d0c..c60fb48a01d5e 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -45,8 +45,8 @@ import org.apache.lucene.search.TopFieldCollector; import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.search.TotalHits; -import org.elasticsearch.action.search.SearchShardTask; import org.apache.lucene.search.Weight; +import org.elasticsearch.action.search.SearchShardTask; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.lucene.Lucene; @@ -54,8 +54,8 @@ import org.elasticsearch.common.util.concurrent.EWMATrackingEsThreadPoolExecutor; import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor; import org.elasticsearch.index.IndexSortConfig; -import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchPhase; import org.elasticsearch.search.SearchService; @@ -263,6 +263,7 @@ static boolean executeInternal(SearchContext searchContext) throws QueryPhaseExe } else { timeoutRunnable = null; } + searcher.setCheckTimeout(timeoutRunnable); final Runnable cancellationRunnable; if (searchContext.lowLevelCancellation()) { @@ -271,21 +272,22 @@ static boolean executeInternal(SearchContext searchContext) throws QueryPhaseExe } else { cancellationRunnable = null; } - - final Runnable checkCancelled; - if (timeoutRunnable != null && cancellationRunnable != null) { - checkCancelled = () -> { - timeoutRunnable.run(); - cancellationRunnable.run(); - }; - } else if (timeoutRunnable != null) { - checkCancelled = timeoutRunnable; - } else if (cancellationRunnable != null) { - checkCancelled = cancellationRunnable; - } else { - checkCancelled = null; - } - searcher.setCheckCancelled(checkCancelled); + searcher.setCheckCancelled(cancellationRunnable); + +// final Runnable checkCancelled; +// if (timeoutRunnable != null && cancellationRunnable != null) { +// checkCancelled = () -> { +// timeoutRunnable.run(); +// cancellationRunnable.run(); +// }; +// } else if (timeoutRunnable != null) { +// checkCancelled = timeoutRunnable; +// } else if (cancellationRunnable != null) { +// checkCancelled = cancellationRunnable; +// } else { +// checkCancelled = null; +// } +// searcher.setCheckCancelled(checkCancelled); boolean shouldRescore; // if we are optimizing sort and there are no other collectors @@ -310,6 +312,8 @@ static boolean executeInternal(SearchContext searchContext) throws QueryPhaseExe queryResult.nodeQueueSize(rExecutor.getCurrentQueueSize()); queryResult.serviceTimeEWMA((long) rExecutor.getTaskExecutionEWMA()); } + // Search phase has finished, no longer need to check for timeout + searcher.unsetCheckTimeout(); return shouldRescore; } catch (Exception e) { throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Failed to execute main query", e); diff --git a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java index cdbe140b0f83c..0beeb06d37c58 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java @@ -28,8 +28,8 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.TotalHitCountCollector; import org.apache.lucene.store.Directory; -import org.elasticsearch.core.internal.io.IOUtils; import org.apache.lucene.util.TestUtil; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.test.ESTestCase; diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 6e1484647e34b..52c91eebf610f 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -248,7 +248,7 @@ public boolean shouldCache(Query query) { } }; ContextIndexSearcher contextIndexSearcher = new ContextIndexSearcher(indexReader, - similarity, queryCache, queryCachingPolicy, false); + similarity, queryCache, queryCachingPolicy, null); SearchContext searchContext = mock(SearchContext.class); when(searchContext.numberOfShards()).thenReturn(1); From 331411be33e0e2b1bbfad1c6aad03d5af5548b8a Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Thu, 27 Feb 2020 22:45:40 +0100 Subject: [PATCH 06/26] Custom implementation of ExitableDirReader to overcome casting --- .../search/internal/ContextIndexSearcher.java | 224 +++++++++++++++--- .../search/query/QueryPhase.java | 15 -- 2 files changed, 191 insertions(+), 48 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 88637512f8a21..312243a95b07a 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -19,19 +19,16 @@ package org.elasticsearch.search.internal; -import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.ExitableDirectoryReader; import org.apache.lucene.index.FilterDirectoryReader; +import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.QueryTimeout; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.PointValues; import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.CollectionTerminatedException; @@ -53,11 +50,14 @@ import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.Weight; import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.suggest.document.CompletionTerms; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CombinedBitSet; import org.apache.lucene.util.SparseFixedBitSet; +import org.apache.lucene.util.automaton.CompiledAutomaton; import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.dfs.AggregatedDfs; @@ -388,42 +388,45 @@ public CacheHelper getReaderCacheHelper() { /** * Wraps a leaf reader with a cancellable task */ - private static class CancellableLeafReader extends ExitableDirectoryReader.ExitableFilterAtomicReader { + private static class CancellableLeafReader extends FilterLeafReader { - private CancellableLeafReader(LeafReader leafReader, DirReaderCancellable checkCancelled) { - super(leafReader, checkCancelled); - } + private final DirReaderCancellable cancellable; - @Override - public NumericDocValues getNumericDocValues(String field) throws IOException { - return in.getNumericDocValues(field); + private CancellableLeafReader(LeafReader leafReader, DirReaderCancellable cancellable) { + super(leafReader); + this.cancellable = cancellable; } @Override - public BinaryDocValues getBinaryDocValues(String field) throws IOException { - return in.getBinaryDocValues(field); + public PointValues getPointValues(String field) throws IOException { + final PointValues pointValues = in.getPointValues(field); + if (pointValues == null) { + return null; + } + return (cancellable.isEnabled()) ? new ExitablePointValues(pointValues, cancellable) : pointValues; } @Override - public SortedDocValues getSortedDocValues(String field) throws IOException { - return in.getSortedDocValues(field); + public Terms terms(String field) throws IOException { + Terms terms = in.terms(field); + if (terms == null) { + return null; + } + return (cancellable.isEnabled() && terms instanceof CompletionTerms == false) ? new ExitableTerms(terms, cancellable) : terms; } @Override - public SortedNumericDocValues getSortedNumericDocValues(String field) throws IOException { - return in.getSortedNumericDocValues(field); + public CacheHelper getCoreCacheHelper() { + return in.getCoreCacheHelper(); } @Override - public SortedSetDocValues getSortedSetDocValues(String field) throws IOException { - return in.getSortedSetDocValues(field); + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); } } - /** - * Implementation of {@link QueryTimeout} with Runnable tasks. - */ - private static class DirReaderCancellable implements QueryTimeout { + private static class DirReaderCancellable { private Runnable checkCancelled = null; private Runnable checkTimeout = null; @@ -440,22 +443,177 @@ private void unsetCheckTimeout() { this.checkTimeout = null; } - @Override - public boolean shouldExit() { + void checkCancelled() { if (checkCancelled != null) { checkCancelled.run(); } if (checkTimeout != null) { checkTimeout.run(); } - // Always return false and rely on checkCancelled or checkTimeout to throw - // the appropriate exception - return false; } - @Override - public boolean isTimeoutEnabled() { + private boolean isEnabled() { return checkCancelled != null || checkTimeout != null; } } + + /** + * Wrapper class for terms that check for query cancellation or timeout. + */ + public static class ExitableTerms extends FilterLeafReader.FilterTerms { + + private final DirReaderCancellable cancellable; + + /** Constructor **/ + public ExitableTerms(Terms terms, DirReaderCancellable cancellable) { + super(terms); + this.cancellable = cancellable; + } + + @Override + public TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm) throws IOException { + return new ExitableTermsEnum(in.intersect(compiled, startTerm), cancellable); + } + + @Override + public TermsEnum iterator() throws IOException { + return new ExitableTermsEnum(in.iterator(), cancellable); + } + } + + /** + * Wrapper class for TermsEnum that is used by ExitableTerms for implementing an + * exitable enumeration of terms. + */ + private static class ExitableTermsEnum extends FilterLeafReader.FilterTermsEnum { + + private final DirReaderCancellable cancellable; + + /** Constructor **/ + private ExitableTermsEnum(TermsEnum termsEnum, DirReaderCancellable cancellable) { + super(termsEnum); + this.cancellable = cancellable; + this.cancellable.checkCancelled(); + } + + @Override + public BytesRef next() throws IOException { + // Before every iteration, check if the iteration should exit + this.cancellable.checkCancelled(); + return in.next(); + } + } + + /** + * Wrapper class for PointValues that checks for cancellation and timeout. + */ + private static class ExitablePointValues extends PointValues { + + private final PointValues in; + private final DirReaderCancellable cancellable; + + private ExitablePointValues(PointValues in, DirReaderCancellable cancellable) { + this.in = in; + this.cancellable = cancellable; + this.cancellable.checkCancelled(); + } + + @Override + public void intersect(IntersectVisitor visitor) throws IOException { + cancellable.checkCancelled(); + in.intersect(new ExitableIntersectVisitor(visitor, cancellable)); + } + + @Override + public long estimatePointCount(IntersectVisitor visitor) { + cancellable.checkCancelled(); + return in.estimatePointCount(visitor); + } + + @Override + public byte[] getMinPackedValue() throws IOException { + cancellable.checkCancelled(); + return in.getMinPackedValue(); + } + + @Override + public byte[] getMaxPackedValue() throws IOException { + cancellable.checkCancelled(); + return in.getMaxPackedValue(); + } + + @Override + public int getNumDimensions() throws IOException { + cancellable.checkCancelled(); + return in.getNumDimensions(); + } + + @Override + public int getNumIndexDimensions() throws IOException { + cancellable.checkCancelled(); + return in.getNumIndexDimensions(); + } + + @Override + public int getBytesPerDimension() throws IOException { + cancellable.checkCancelled(); + return in.getBytesPerDimension(); + } + + @Override + public long size() { + cancellable.checkCancelled(); + return in.size(); + } + + @Override + public int getDocCount() { + cancellable.checkCancelled(); + return in.getDocCount(); + } + } + + private static class ExitableIntersectVisitor implements PointValues.IntersectVisitor { + + private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = 10; + + private final PointValues.IntersectVisitor in; + private final DirReaderCancellable cancellable; + private int calls; + + private ExitableIntersectVisitor(PointValues.IntersectVisitor in, DirReaderCancellable cancellable) { + this.in = in; + this.cancellable = cancellable; + } + + private void checkAndThrowWithSampling() { + if (calls++ % MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK == 0) { + cancellable.checkCancelled(); + } + } + + @Override + public void visit(int docID) throws IOException { + checkAndThrowWithSampling(); + in.visit(docID); + } + + @Override + public void visit(int docID, byte[] packedValue) throws IOException { + checkAndThrowWithSampling(); + in.visit(docID, packedValue); + } + + @Override + public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + cancellable.checkCancelled(); + return in.compare(minPackedValue, maxPackedValue); + } + + @Override + public void grow(int count) { + cancellable.checkCancelled(); + in.grow(count); + } + } } diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index c60fb48a01d5e..167a56a19d494 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -274,21 +274,6 @@ static boolean executeInternal(SearchContext searchContext) throws QueryPhaseExe } searcher.setCheckCancelled(cancellationRunnable); -// final Runnable checkCancelled; -// if (timeoutRunnable != null && cancellationRunnable != null) { -// checkCancelled = () -> { -// timeoutRunnable.run(); -// cancellationRunnable.run(); -// }; -// } else if (timeoutRunnable != null) { -// checkCancelled = timeoutRunnable; -// } else if (cancellationRunnable != null) { -// checkCancelled = cancellationRunnable; -// } else { -// checkCancelled = null; -// } -// searcher.setCheckCancelled(checkCancelled); - boolean shouldRescore; // if we are optimizing sort and there are no other collectors if (sortAndFormatsForRewrittenNumericSort != null && collectors.size() == 0 && searchContext.getProfilers() == null) { From d10c51a9ea4ca8d2e2813b0c854f679ad8cc3dc8 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Thu, 27 Feb 2020 23:18:03 +0100 Subject: [PATCH 07/26] revert changes --- .../aggregations/metrics/AvgAggregatorTests.java | 4 ++-- .../aggregations/metrics/MaxAggregatorTests.java | 12 ++++++------ .../aggregations/metrics/MinAggregatorTests.java | 8 ++++---- .../metrics/WeightedAvgAggregatorTests.java | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java index e629b2acb6471..0464dc053d07b 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java @@ -538,7 +538,7 @@ public void testOrderByEmptyAggregation() throws IOException { indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newIndexSearcher(indexReader); + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); TermsAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); aggregator.preCollection(); @@ -588,7 +588,7 @@ private void testCase(AvgAggregationBuilder aggregationBuilder, Query query, indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newIndexSearcher(indexReader); + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); AvgAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); aggregator.preCollection(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java index d135210e108cb..48665b32a89dd 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java @@ -290,7 +290,7 @@ private void testCase(MaxAggregationBuilder aggregationBuilder, Query query, buildIndex.accept(indexWriter); indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newIndexSearcher(indexReader); + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); MaxAggregator aggregator = createAggregator(query, aggregationBuilder, indexSearcher, createIndexSettings(), fieldType); aggregator.preCollection(); @@ -457,7 +457,7 @@ public void testSingleValuedFieldGetProperty() throws IOException { indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newIndexSearcher(indexReader); + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); GlobalAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); aggregator.preCollection(); @@ -724,7 +724,7 @@ public void testEmptyAggregation() throws Exception { indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newIndexSearcher(indexReader); + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); GlobalAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); aggregator.preCollection(); @@ -767,7 +767,7 @@ public void testOrderByEmptyAggregation() throws IOException { indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newIndexSearcher(indexReader); + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); TermsAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); aggregator.preCollection(); @@ -815,7 +815,7 @@ public void testEarlyTermination() throws Exception { indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newIndexSearcher(indexReader); + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); MaxAggregationBuilder maxAggregationBuilder = new MaxAggregationBuilder("max") .field("values"); ValueCountAggregationBuilder countAggregationBuilder = new ValueCountAggregationBuilder("count", null) @@ -863,7 +863,7 @@ public void testNestedEarlyTermination() throws Exception { indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newIndexSearcher(indexReader); + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); for (Aggregator.SubAggCollectionMode collectionMode : Aggregator.SubAggCollectionMode.values()) { MaxAggregationBuilder maxAggregationBuilder = new MaxAggregationBuilder("max") diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java index a25c9ef5c8bc5..52b23fe6a3acf 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java @@ -649,7 +649,7 @@ public void testCaching() throws IOException { try (IndexReader indexReader = DirectoryReader.open(directory)) { - IndexSearcher indexSearcher = newIndexSearcher(indexReader); + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); InternalMin min = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); assertEquals(2.0, min.getValue(), 0); @@ -681,7 +681,7 @@ public void testScriptCaching() throws IOException { try (IndexReader indexReader = DirectoryReader.open(directory)) { - IndexSearcher indexSearcher = newIndexSearcher(indexReader); + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); InternalMin min = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), nonDeterministicAggregationBuilder, fieldType); assertTrue(min.getValue() >= 0.0 && min.getValue() <= 1.0); @@ -689,7 +689,7 @@ public void testScriptCaching() throws IOException { assertFalse(queryShardContext.isCacheable()); - indexSearcher = newIndexSearcher(indexReader); + indexSearcher = newSearcher(indexReader, true, true); min = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); assertEquals(-7.0, min.getValue(), 0); @@ -927,7 +927,7 @@ private void testC indexWriter.close(); try (IndexReader indexReader = DirectoryReader.open(directory)) { - IndexSearcher indexSearcher = newIndexSearcher(indexReader); + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); V agg = searchAndReduce(indexSearcher, query, aggregationBuilder, fieldType); verify.accept(agg); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregatorTests.java index 9f9bb3bb94912..78c4d18218eea 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregatorTests.java @@ -420,7 +420,7 @@ private void testCase(Query query, WeightedAvgAggregationBuilder aggregationBuil buildIndex.accept(indexWriter); indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newIndexSearcher(indexReader); + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); try { MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(fieldNumberType); From e5fdf477933856da0877b81737c1d77640d01d54 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Thu, 27 Feb 2020 23:20:34 +0100 Subject: [PATCH 08/26] revert --- .../aggregations/AggregatorTestCase.java | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 52c91eebf610f..2a82d831ee73e 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -40,7 +40,6 @@ import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Weight; -import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; @@ -223,18 +222,16 @@ protected SearchContext createSearchContext(IndexSearcher indexSearcher, IndexSettings indexSettings, Query query, MultiBucketConsumer bucketConsumer, - MappedFieldType... fieldTypes) throws IOException { - return createSearchContext(indexSearcher.getIndexReader(), indexSearcher.getSimilarity(), - indexSettings, query, bucketConsumer, new NoneCircuitBreakerService(), fieldTypes); + MappedFieldType... fieldTypes) { + return createSearchContext(indexSearcher, indexSettings, query, bucketConsumer, new NoneCircuitBreakerService(), fieldTypes); } - protected SearchContext createSearchContext(IndexReader indexReader, - Similarity similarity, + protected SearchContext createSearchContext(IndexSearcher indexSearcher, IndexSettings indexSettings, Query query, MultiBucketConsumer bucketConsumer, CircuitBreakerService circuitBreakerService, - MappedFieldType... fieldTypes) throws IOException { + MappedFieldType... fieldTypes) { QueryCache queryCache = new DisabledQueryCache(indexSettings); QueryCachingPolicy queryCachingPolicy = new QueryCachingPolicy() { @Override @@ -247,8 +244,8 @@ public boolean shouldCache(Query query) { return false; } }; - ContextIndexSearcher contextIndexSearcher = new ContextIndexSearcher(indexReader, - similarity, queryCache, queryCachingPolicy, null); + ContextIndexSearcher contextIndexSearcher = new ContextIndexSearcher(indexSearcher.getIndexReader(), + indexSearcher.getSimilarity(), queryCache, queryCachingPolicy); SearchContext searchContext = mock(SearchContext.class); when(searchContext.numberOfShards()).thenReturn(1); @@ -440,11 +437,6 @@ protected A searchAndReduc AggregationBuilder builder, int maxBucket, MappedFieldType... fieldTypes) throws IOException { - MultiBucketConsumer bucketConsumer = new MultiBucketConsumer(maxBucket, - new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)); - C root = createAggregator(query, builder, searcher, bucketConsumer, fieldTypes); - searcher = root.context().searcher(); - final IndexReaderContext ctx = searcher.getTopReaderContext(); final ShardSearcher[] subSearchers; @@ -464,6 +456,9 @@ protected A searchAndReduc List aggs = new ArrayList<> (); Query rewritten = searcher.rewrite(query); Weight weight = searcher.createWeight(rewritten, ScoreMode.COMPLETE, 1f); + MultiBucketConsumer bucketConsumer = new MultiBucketConsumer(maxBucket, + new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)); + C root = createAggregator(query, builder, searcher, bucketConsumer, fieldTypes); for (ShardSearcher subSearcher : subSearchers) { MultiBucketConsumer shardBucketConsumer = new MultiBucketConsumer(maxBucket, From 248ee5159f39ad7e00dde8490407c6ce858e12d2 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Thu, 27 Feb 2020 23:35:58 +0100 Subject: [PATCH 09/26] revert unrelated changes --- .../bucket/BestBucketsDeferringCollectorTests.java | 1 - .../aggregations/bucket/filter/FilterAggregatorTests.java | 2 +- .../bucket/geogrid/GeoGridAggregatorTestCase.java | 2 +- .../bucket/histogram/DateHistogramAggregatorTests.java | 2 +- .../aggregations/bucket/missing/MissingAggregatorTests.java | 3 ++- .../search/aggregations/AggregatorTestCase.java | 6 +++--- .../analytics/topmetrics/TopMetricsAggregatorTests.java | 5 ++--- 7 files changed, 10 insertions(+), 11 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollectorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollectorTests.java index 47471afc8efa6..74ee21b2466cd 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollectorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollectorTests.java @@ -68,7 +68,6 @@ public void testReplay() throws Exception { TopDocs topDocs = indexSearcher.search(termQuery, numDocs); SearchContext searchContext = createSearchContext(indexSearcher, createIndexSettings(), rewrittenQuery, null); - indexSearcher = searchContext.searcher(); when(searchContext.query()).thenReturn(rewrittenQuery); BestBucketsDeferringCollector collector = new BestBucketsDeferringCollector(searchContext, false) { @Override diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java index 5998413d238dd..56153d2d35f13 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorTests.java @@ -84,7 +84,7 @@ public void testRandom() throws Exception { indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newIndexSearcher(indexReader); + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); try { int value = randomInt(maxTerm - 1); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java index c409bdc3bbb0c..2059f465c3756 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java @@ -219,7 +219,7 @@ private void testCase(Query query, int precision, GeoBoundingBox geoBoundingBox, indexWriter.close(); IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newIndexSearcher(indexReader); + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); aggregationBuilder.precision(precision); if (geoBoundingBox != null) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java index 68bdd96ab3b13..4eba98dd49b9d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java @@ -1235,7 +1235,7 @@ private void executeTestCase(boolean reduced, } try (IndexReader indexReader = DirectoryReader.open(directory)) { - IndexSearcher indexSearcher = newIndexSearcher(indexReader); + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); DateHistogramAggregationBuilder aggregationBuilder = new DateHistogramAggregationBuilder("_name"); if (configure != null) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java index a693c2ee2c685..dbc3ac3b490c2 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorTests.java @@ -175,7 +175,8 @@ private void executeTestCase(int numDocs, } try (IndexReader indexReader = DirectoryReader.open(directory)) { - IndexSearcher indexSearcher = newIndexSearcher(indexReader); + IndexSearcher indexSearcher = + newSearcher(indexReader, true, true); MissingAggregationBuilder builder = new MissingAggregationBuilder("_name", null); builder.field(fieldName); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 2a82d831ee73e..601a49019c686 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -222,7 +222,7 @@ protected SearchContext createSearchContext(IndexSearcher indexSearcher, IndexSettings indexSettings, Query query, MultiBucketConsumer bucketConsumer, - MappedFieldType... fieldTypes) { + MappedFieldType... fieldTypes) throws IOException { return createSearchContext(indexSearcher, indexSettings, query, bucketConsumer, new NoneCircuitBreakerService(), fieldTypes); } @@ -231,7 +231,7 @@ protected SearchContext createSearchContext(IndexSearcher indexSearcher, Query query, MultiBucketConsumer bucketConsumer, CircuitBreakerService circuitBreakerService, - MappedFieldType... fieldTypes) { + MappedFieldType... fieldTypes) throws IOException { QueryCache queryCache = new DisabledQueryCache(indexSettings); QueryCachingPolicy queryCachingPolicy = new QueryCachingPolicy() { @Override @@ -245,7 +245,7 @@ public boolean shouldCache(Query query) { } }; ContextIndexSearcher contextIndexSearcher = new ContextIndexSearcher(indexSearcher.getIndexReader(), - indexSearcher.getSimilarity(), queryCache, queryCachingPolicy); + indexSearcher.getSimilarity(), queryCache, queryCachingPolicy, null); SearchContext searchContext = mock(SearchContext.class); when(searchContext.numberOfShards()).thenReturn(1); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java index f040a7b7f00eb..86ece2c4e1db5 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java @@ -336,9 +336,8 @@ public long addWithoutBreaking(long bytes) { try (IndexReader indexReader = DirectoryReader.open(directory)) { IndexSearcher indexSearcher = newSearcher(indexReader, false, false); - SearchContext searchContext = createSearchContext(indexReader, indexSearcher.getSimilarity(), createIndexSettings(), - new MatchAllDocsQuery(), new MultiBucketConsumer(Integer.MAX_VALUE, breaker.getBreaker(CircuitBreaker.REQUEST)), - breaker, doubleFields()); + SearchContext searchContext = createSearchContext(indexSearcher, createIndexSettings(), new MatchAllDocsQuery(), + new MultiBucketConsumer(Integer.MAX_VALUE, breaker.getBreaker(CircuitBreaker.REQUEST)), breaker, doubleFields()); TopMetricsAggregationBuilder builder = simpleBuilder(new FieldSortBuilder("s").order(SortOrder.ASC)); Aggregator aggregator = builder.build(searchContext.getQueryShardContext(), null) .create(searchContext, null, true); From ce2d557d56d860402220bb64cfbca16c100eef25 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Fri, 28 Feb 2020 03:08:40 +0100 Subject: [PATCH 10/26] Fix issues after splitting to cancelation and timeout --- .../search/internal/ContextIndexSearcher.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 312243a95b07a..a53e456590512 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -176,12 +176,12 @@ public Weight createWeight(Query query, ScoreMode scoreMode, float boost) throws } private void checkCancelled() { - if (checkCancelled != null) { - checkCancelled.run(); - } if (checkTimeout != null) { checkTimeout.run(); } + if (checkCancelled != null) { + checkCancelled.run(); + } } @SuppressWarnings({"unchecked", "rawtypes"}) @@ -246,8 +246,7 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto Scorer scorer = weight.scorer(ctx); if (scorer != null) { try { - intersectScorerAndBitSet(scorer, liveDocsBitSet, leafCollector, - checkCancelled == null ? () -> { } : checkCancelled); + intersectScorerAndBitSet(scorer, liveDocsBitSet, leafCollector, this::checkCancelled); } catch (CollectionTerminatedException e) { // collection was terminated prematurely // continue with the following leaf @@ -257,7 +256,7 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto } private Weight wrapWeight(Weight weight) { - if (checkCancelled != null) { + if (checkCancelled != null || checkTimeout != null) { return new Weight(weight.getQuery()) { @Override public void extractTerms(Set terms) { @@ -283,7 +282,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { BulkScorer in = weight.bulkScorer(context); if (in != null) { - return new CancellableBulkScorer(in, checkCancelled); + return new CancellableBulkScorer(in, () -> checkCancelled()); } else { return null; } @@ -444,12 +443,12 @@ private void unsetCheckTimeout() { } void checkCancelled() { - if (checkCancelled != null) { - checkCancelled.run(); - } if (checkTimeout != null) { checkTimeout.run(); } + if (checkCancelled != null) { + checkCancelled.run(); + } } private boolean isEnabled() { From 0012e3ac6d9b5578bd434e39e5db1442b5d742cd Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Fri, 28 Feb 2020 14:09:14 +0100 Subject: [PATCH 11/26] Add unit test --- .../search/internal/ContextIndexSearcher.java | 154 +++++++++--------- .../search/query/QueryPhase.java | 5 +- .../search/SearchCancellationTests.java | 54 +++++- .../internal/ContextIndexSearcherTests.java | 2 +- 4 files changed, 134 insertions(+), 81 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index a53e456590512..9f4e5edc5bfe2 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -86,13 +86,11 @@ public class ContextIndexSearcher extends IndexSearcher { private AggregatedDfs aggregatedDfs; private QueryProfiler profiler; - private Runnable checkCancelled; - private Runnable checkTimeout; - private DirReaderCancellable dirReaderCancellable; + private Holder cancellable; public ContextIndexSearcher(IndexReader reader, Similarity similarity, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy) throws IOException { - this(reader, similarity, queryCache, queryCachingPolicy, new DirReaderCancellable()); + this(reader, similarity, queryCache, queryCachingPolicy, new Holder<>(null)); } // TODO: Make the 2nd constructor private so that the dirCancellable is never null and the IndexReader is always wrapped. @@ -100,9 +98,9 @@ public ContextIndexSearcher(IndexReader reader, Similarity similarity, // the use of searchAndReduce and the ShardSearcher sub-searchers. public ContextIndexSearcher(IndexReader reader, Similarity similarity, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, - DirReaderCancellable dirReaderCancellable) throws IOException { - super(dirReaderCancellable != null? new CancellableDirectoryReader((DirectoryReader) reader, dirReaderCancellable) : reader); - this.dirReaderCancellable = dirReaderCancellable; + Holder cancellable) throws IOException { + super(cancellable!= null? new CancellableDirectoryReader((DirectoryReader) reader, cancellable) : reader); + this.cancellable = cancellable; setSimilarity(similarity); setQueryCache(queryCache); setQueryCachingPolicy(queryCachingPolicy); @@ -114,24 +112,14 @@ public void setProfiler(QueryProfiler profiler) { /** * Set a {@link Runnable} that will be run on a regular basis while - * collecting documents and check for query cancellation. + * collecting documents and check for query cancellation or timeout */ - public void setCheckCancelled(Runnable checkCancelled) { - this.checkCancelled = checkCancelled; - this.dirReaderCancellable.setCheckCancelled(checkCancelled); - } - - /** - * Set a {@link Runnable} that will be run on a regular basis while - * collecting documents and check for query timeout. - */ - public void setCheckTimeout(Runnable checkTimeout) { - this.checkTimeout = checkTimeout; - this.dirReaderCancellable.setCheckTimeout(checkTimeout); + public void setCancellable(Cancellable cancellable) { + this.cancellable.set(cancellable); } public void unsetCheckTimeout() { - this.dirReaderCancellable.unsetCheckTimeout(); + this.cancellable.get().unsetCheckTimeout(); } public void setAggregatedDfs(AggregatedDfs aggregatedDfs) { @@ -175,15 +163,6 @@ public Weight createWeight(Query query, ScoreMode scoreMode, float boost) throws } } - private void checkCancelled() { - if (checkTimeout != null) { - checkTimeout.run(); - } - if (checkCancelled != null) { - checkCancelled.run(); - } - } - @SuppressWarnings({"unchecked", "rawtypes"}) public void search(List leaves, Weight weight, CollectorManager manager, QuerySearchResult result, DocValueFormat[] formats, TotalHits totalHits) throws IOException { @@ -219,7 +198,7 @@ protected void search(List leaves, Weight weight, Collector c * the provided ctx. */ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collector) throws IOException { - checkCancelled(); + cancellable.get().checkCancelled(); weight = wrapWeight(weight); final LeafCollector leafCollector; try { @@ -246,7 +225,7 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto Scorer scorer = weight.scorer(ctx); if (scorer != null) { try { - intersectScorerAndBitSet(scorer, liveDocsBitSet, leafCollector, this::checkCancelled); + intersectScorerAndBitSet(scorer, liveDocsBitSet, leafCollector, () -> cancellable.get().checkCancelled()); } catch (CollectionTerminatedException e) { // collection was terminated prematurely // continue with the following leaf @@ -256,7 +235,7 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto } private Weight wrapWeight(Weight weight) { - if (checkCancelled != null || checkTimeout != null) { + if (cancellable.get().isEnabled()) { return new Weight(weight.getQuery()) { @Override public void extractTerms(Set terms) { @@ -282,7 +261,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { BulkScorer in = weight.bulkScorer(context); if (in != null) { - return new CancellableBulkScorer(in, () -> checkCancelled()); + return new CancellableBulkScorer(in, () -> cancellable.get().checkCancelled()); } else { return null; } @@ -364,7 +343,7 @@ public DirectoryReader getDirectoryReader() { */ private static class CancellableDirectoryReader extends FilterDirectoryReader { - private CancellableDirectoryReader(DirectoryReader in, DirReaderCancellable cancellable) throws IOException { + private CancellableDirectoryReader(DirectoryReader in, Holder cancellable) throws IOException { super(in, new SubReaderWrapper() { @Override public LeafReader wrap(LeafReader reader) { @@ -389,9 +368,9 @@ public CacheHelper getReaderCacheHelper() { */ private static class CancellableLeafReader extends FilterLeafReader { - private final DirReaderCancellable cancellable; + private final Holder cancellable; - private CancellableLeafReader(LeafReader leafReader, DirReaderCancellable cancellable) { + private CancellableLeafReader(LeafReader leafReader, Holder cancellable) { super(leafReader); this.cancellable = cancellable; } @@ -402,7 +381,7 @@ public PointValues getPointValues(String field) throws IOException { if (pointValues == null) { return null; } - return (cancellable.isEnabled()) ? new ExitablePointValues(pointValues, cancellable) : pointValues; + return (cancellable.get().isEnabled()) ? new ExitablePointValues(pointValues, cancellable.get()) : pointValues; } @Override @@ -411,7 +390,8 @@ public Terms terms(String field) throws IOException { if (terms == null) { return null; } - return (cancellable.isEnabled() && terms instanceof CompletionTerms == false) ? new ExitableTerms(terms, cancellable) : terms; + return (cancellable.get().isEnabled() && terms instanceof CompletionTerms == false) ? + new ExitableTerms(terms, cancellable.get()) : terms; } @Override @@ -425,34 +405,60 @@ public CacheHelper getReaderCacheHelper() { } } - private static class DirReaderCancellable { + private static class Holder { + private T in; - private Runnable checkCancelled = null; - private Runnable checkTimeout = null; + public Holder(T in) { + this.in = in; + } - private void setCheckCancelled(Runnable checkCancelled) { - this.checkCancelled = checkCancelled; + private void set(T in) { + this.in = in; + } + + private T get() { + return in; } + } + + public interface Cancellable { + + boolean isEnabled(); + void checkCancelled(); + default void checkDirReaderCancelled() { + checkCancelled(); + } + void unsetCheckTimeout(); + } + + public static class CancellableImpl implements Cancellable { - private void setCheckTimeout(Runnable checkTimeout) { + private Runnable checkCancelled; + private Runnable checkTimeout; + + public CancellableImpl(Runnable checkTimeout, Runnable checkCancelled) { + this.checkCancelled = checkCancelled; this.checkTimeout = checkTimeout; } - private void unsetCheckTimeout() { - this.checkTimeout = null; + @Override + public boolean isEnabled() { + return checkCancelled != null || checkTimeout != null; } - void checkCancelled() { + @Override + public void checkCancelled() { if (checkTimeout != null) { checkTimeout.run(); } if (checkCancelled != null) { - checkCancelled.run(); + checkCancelled.run();; } } - private boolean isEnabled() { - return checkCancelled != null || checkTimeout != null; + @Override + public void unsetCheckTimeout() { + this.checkTimeout = null; } } @@ -461,10 +467,10 @@ private boolean isEnabled() { */ public static class ExitableTerms extends FilterLeafReader.FilterTerms { - private final DirReaderCancellable cancellable; + private final Cancellable cancellable; /** Constructor **/ - public ExitableTerms(Terms terms, DirReaderCancellable cancellable) { + public ExitableTerms(Terms terms, Cancellable cancellable) { super(terms); this.cancellable = cancellable; } @@ -486,19 +492,19 @@ public TermsEnum iterator() throws IOException { */ private static class ExitableTermsEnum extends FilterLeafReader.FilterTermsEnum { - private final DirReaderCancellable cancellable; + private final Cancellable cancellable; /** Constructor **/ - private ExitableTermsEnum(TermsEnum termsEnum, DirReaderCancellable cancellable) { + private ExitableTermsEnum(TermsEnum termsEnum, Cancellable cancellable) { super(termsEnum); this.cancellable = cancellable; - this.cancellable.checkCancelled(); + this.cancellable.checkDirReaderCancelled(); } @Override public BytesRef next() throws IOException { // Before every iteration, check if the iteration should exit - this.cancellable.checkCancelled(); + this.cancellable.checkDirReaderCancelled(); return in.next(); } } @@ -509,65 +515,65 @@ public BytesRef next() throws IOException { private static class ExitablePointValues extends PointValues { private final PointValues in; - private final DirReaderCancellable cancellable; + private final Cancellable cancellable; - private ExitablePointValues(PointValues in, DirReaderCancellable cancellable) { + private ExitablePointValues(PointValues in, Cancellable cancellable) { this.in = in; this.cancellable = cancellable; - this.cancellable.checkCancelled(); + this.cancellable.checkDirReaderCancelled(); } @Override public void intersect(IntersectVisitor visitor) throws IOException { - cancellable.checkCancelled(); + cancellable.checkDirReaderCancelled(); in.intersect(new ExitableIntersectVisitor(visitor, cancellable)); } @Override public long estimatePointCount(IntersectVisitor visitor) { - cancellable.checkCancelled(); + cancellable.checkDirReaderCancelled(); return in.estimatePointCount(visitor); } @Override public byte[] getMinPackedValue() throws IOException { - cancellable.checkCancelled(); + cancellable.checkDirReaderCancelled(); return in.getMinPackedValue(); } @Override public byte[] getMaxPackedValue() throws IOException { - cancellable.checkCancelled(); + cancellable.checkDirReaderCancelled(); return in.getMaxPackedValue(); } @Override public int getNumDimensions() throws IOException { - cancellable.checkCancelled(); + cancellable.checkDirReaderCancelled(); return in.getNumDimensions(); } @Override public int getNumIndexDimensions() throws IOException { - cancellable.checkCancelled(); + cancellable.checkDirReaderCancelled(); return in.getNumIndexDimensions(); } @Override public int getBytesPerDimension() throws IOException { - cancellable.checkCancelled(); + cancellable.checkDirReaderCancelled(); return in.getBytesPerDimension(); } @Override public long size() { - cancellable.checkCancelled(); + cancellable.checkDirReaderCancelled(); return in.size(); } @Override public int getDocCount() { - cancellable.checkCancelled(); + cancellable.checkDirReaderCancelled(); return in.getDocCount(); } } @@ -577,17 +583,17 @@ private static class ExitableIntersectVisitor implements PointValues.IntersectVi private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = 10; private final PointValues.IntersectVisitor in; - private final DirReaderCancellable cancellable; + private final Cancellable cancellable; private int calls; - private ExitableIntersectVisitor(PointValues.IntersectVisitor in, DirReaderCancellable cancellable) { + private ExitableIntersectVisitor(PointValues.IntersectVisitor in, Cancellable cancellable) { this.in = in; this.cancellable = cancellable; } private void checkAndThrowWithSampling() { if (calls++ % MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK == 0) { - cancellable.checkCancelled(); + cancellable.checkDirReaderCancelled(); } } @@ -605,13 +611,13 @@ public void visit(int docID, byte[] packedValue) throws IOException { @Override public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { - cancellable.checkCancelled(); + cancellable.checkDirReaderCancelled(); return in.compare(minPackedValue, maxPackedValue); } @Override public void grow(int count) { - cancellable.checkCancelled(); + cancellable.checkDirReaderCancelled(); in.grow(count); } } diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 167a56a19d494..2d1c6bc173b58 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -263,7 +263,6 @@ static boolean executeInternal(SearchContext searchContext) throws QueryPhaseExe } else { timeoutRunnable = null; } - searcher.setCheckTimeout(timeoutRunnable); final Runnable cancellationRunnable; if (searchContext.lowLevelCancellation()) { @@ -272,7 +271,8 @@ static boolean executeInternal(SearchContext searchContext) throws QueryPhaseExe } else { cancellationRunnable = null; } - searcher.setCheckCancelled(cancellationRunnable); + + searcher.setCancellable(new ContextIndexSearcher.CancellableImpl(timeoutRunnable, cancellationRunnable)); boolean shouldRescore; // if we are optimizing sort and there are no other collectors @@ -298,6 +298,7 @@ static boolean executeInternal(SearchContext searchContext) throws QueryPhaseExe queryResult.serviceTimeEWMA((long) rExecutor.getTaskExecutionEWMA()); } // Search phase has finished, no longer need to check for timeout + // otherwise aggregation phase might get cancelled. searcher.unsetCheckTimeout(); return shouldRescore; } catch (Exception e) { diff --git a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java index 0beeb06d37c58..8208a76df8313 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java @@ -24,8 +24,10 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.TotalHitCountCollector; import org.apache.lucene.store.Directory; import org.apache.lucene.util.TestUtil; @@ -46,6 +48,8 @@ public class SearchCancellationTests extends ESTestCase { static Directory dir; static IndexReader reader; + private static String FIELD_NAME = "foo"; + @BeforeClass public static void setup() throws IOException { dir = newDirectory(); @@ -63,7 +67,7 @@ public static void setup() throws IOException { private static void indexRandomDocuments(RandomIndexWriter w, int numDocs) throws IOException { for (int i = 0; i < numDocs; ++i) { Document doc = new Document(); - doc.add(new StringField("foo", "bar", Field.Store.NO)); + doc.add(new StringField(FIELD_NAME, "a".repeat(i), Field.Store.NO)); w.addDocument(doc); } } @@ -80,9 +84,25 @@ public void testCancellableCollector() throws IOException { AtomicBoolean cancelled = new AtomicBoolean(); ContextIndexSearcher searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); - searcher.setCheckCancelled(() -> { - if (cancelled.get()) { - throw new TaskCancelledException("cancelled"); + searcher.setCancellable(new ContextIndexSearcher.Cancellable() { + @Override + public boolean isEnabled() { + return true; + } + + @Override + public void checkCancelled() { + if (cancelled.get()) { + throw new TaskCancelledException("cancelled"); + } + } + + @Override + public void checkDirReaderCancelled() { + } + + @Override + public void unsetCheckTimeout() { } }); searcher.search(new MatchAllDocsQuery(), collector); @@ -92,4 +112,30 @@ public void testCancellableCollector() throws IOException { () -> searcher.search(new MatchAllDocsQuery(), collector)); } + public void testCancellableDirReader() throws IOException { + TotalHitCountCollector collector = new TotalHitCountCollector(); + ContextIndexSearcher searcher = new ContextIndexSearcher(reader, + IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); + searcher.setCancellable(new ContextIndexSearcher.Cancellable() { + @Override + public boolean isEnabled() { + return true; + } + + @Override + public void checkCancelled() { + } + + @Override + public void checkDirReaderCancelled() { + throw new TaskCancelledException("cancelled"); + } + + @Override + public void unsetCheckTimeout() { + } + }); + expectThrows(TaskCancelledException.class, () -> + searcher.search(new PrefixQuery(new Term(FIELD_NAME, "a")), collector)); + } } diff --git a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java index 186436e4bad3f..22be3839d5f8a 100644 --- a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java @@ -235,7 +235,7 @@ public void onRemoval(ShardId shardId, Accountable accountable) { ContextIndexSearcher searcher = new ContextIndexSearcher(filteredReader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); - searcher.setCheckCancelled(() -> {}); + searcher.setCancellable(new ContextIndexSearcher.CancellableImpl(null, null)); // Searching a non-existing term will trigger a null scorer assertEquals(0, searcher.count(new TermQuery(new Term("non_existing_field", "non_existing_value")))); From 4c3183f14eefbec43885c33b4b47839c6ed1bf95 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Fri, 28 Feb 2020 14:59:30 +0100 Subject: [PATCH 12/26] re-enable MultiReader Tests, fix cancellation tests --- .../search/internal/ContextIndexSearcher.java | 10 +- .../search/SearchCancellationTests.java | 27 +- .../metrics/AvgAggregatorTests.java | 1 - .../metrics/MaxAggregatorTests.java | 1 - .../metrics/MinAggregatorTests.java | 2 - test/framework/javac.20200228_143103.args | 237 ++++++++++++++++++ 6 files changed, 261 insertions(+), 17 deletions(-) create mode 100644 test/framework/javac.20200228_143103.args diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 9f4e5edc5bfe2..8b3f9db1fd91c 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -90,12 +90,13 @@ public class ContextIndexSearcher extends IndexSearcher { public ContextIndexSearcher(IndexReader reader, Similarity similarity, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy) throws IOException { - this(reader, similarity, queryCache, queryCachingPolicy, new Holder<>(null)); + this(reader, similarity, queryCache, queryCachingPolicy, new Holder<>(new CancellableImpl())); } // TODO: Make the 2nd constructor private so that the dirCancellable is never null and the IndexReader is always wrapped. - // Some issues must be fixed regarding tests deriving from AggregatorTestCase and more specifically - // the use of searchAndReduce and the ShardSearcher sub-searchers. + // Some issues must be fixed: + // - regarding tests deriving from AggregatorTestCase and more specifically the use of searchAndReduce and the ShardSearcher sub-searchers. + // - tests that use a MultiReader public ContextIndexSearcher(IndexReader reader, Similarity similarity, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, Holder cancellable) throws IOException { @@ -436,6 +437,9 @@ public static class CancellableImpl implements Cancellable { private Runnable checkCancelled; private Runnable checkTimeout; + private CancellableImpl() { + } + public CancellableImpl(Runnable checkTimeout, Runnable checkCancelled) { this.checkCancelled = checkCancelled; this.checkTimeout = checkTimeout; diff --git a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java index 8208a76df8313..064a5326f36b3 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java @@ -30,7 +30,6 @@ import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.TotalHitCountCollector; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.TestUtil; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.tasks.TaskCancelledException; @@ -41,14 +40,15 @@ import java.io.IOException; import java.util.concurrent.atomic.AtomicBoolean; +import static org.apache.lucene.util.TestUtil.nextInt; import static org.hamcrest.Matchers.equalTo; public class SearchCancellationTests extends ESTestCase { - static Directory dir; - static IndexReader reader; + private static final String FIELD_NAME = "foo"; - private static String FIELD_NAME = "foo"; + private static Directory dir; + private static IndexReader reader; @BeforeClass public static void setup() throws IOException { @@ -57,17 +57,18 @@ public static void setup() throws IOException { // we need at least 2 segments - so no merges should be allowed w.w.getConfig().setMergePolicy(NoMergePolicy.INSTANCE); w.setDoRandomForceMerge(false); - indexRandomDocuments(w, TestUtil.nextInt(random(), 2, 20)); + int numDocs = nextInt(random(), 2, 20); + indexRandomDocuments(w, numDocs, 0); w.flush(); - indexRandomDocuments(w, TestUtil.nextInt(random(), 1, 20)); + indexRandomDocuments(w, nextInt(random(), 1, 20), numDocs); reader = w.getReader(); w.close(); } - private static void indexRandomDocuments(RandomIndexWriter w, int numDocs) throws IOException { - for (int i = 0; i < numDocs; ++i) { + private static void indexRandomDocuments(RandomIndexWriter w, int numDocs, int repeatChar) throws IOException { + for (int i = 1; i <= numDocs; ++i) { Document doc = new Document(); - doc.add(new StringField(FIELD_NAME, "a".repeat(i), Field.Store.NO)); + doc.add(new StringField(FIELD_NAME, "a".repeat(i + repeatChar), Field.Store.NO)); w.addDocument(doc); } } @@ -114,6 +115,7 @@ public void unsetCheckTimeout() { public void testCancellableDirReader() throws IOException { TotalHitCountCollector collector = new TotalHitCountCollector(); + AtomicBoolean cancelled = new AtomicBoolean(); ContextIndexSearcher searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); searcher.setCancellable(new ContextIndexSearcher.Cancellable() { @@ -128,13 +130,18 @@ public void checkCancelled() { @Override public void checkDirReaderCancelled() { - throw new TaskCancelledException("cancelled"); + if (cancelled.get()) { + throw new TaskCancelledException("cancelled"); + } } @Override public void unsetCheckTimeout() { } }); + searcher.search(new PrefixQuery(new Term(FIELD_NAME, "a")), collector); + assertThat(collector.getTotalHits(), equalTo(reader.numDocs())); + cancelled.set(true); expectThrows(TaskCancelledException.class, () -> searcher.search(new PrefixQuery(new Term(FIELD_NAME, "a")), collector)); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java index 0464dc053d07b..52858cee25b83 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java @@ -261,7 +261,6 @@ private void verifyAvgOfDoubles(double[] values, double expected, double delta) ); } - @AwaitsFix(bugUrl = "Replace with integration test") public void testSingleValuedFieldPartiallyUnmapped() throws IOException { Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java index 48665b32a89dd..d43fafc22b2f7 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java @@ -483,7 +483,6 @@ public void testSingleValuedFieldGetProperty() throws IOException { directory.close(); } - @AwaitsFix(bugUrl = "Replace with integration test") public void testSingleValuedFieldPartiallyUnmapped() throws IOException { Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java index 52b23fe6a3acf..f1befc7291e39 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java @@ -393,7 +393,6 @@ public void testGetProperty() throws IOException { }, fieldType); } - @AwaitsFix(bugUrl = "Replace with integration test") public void testSingleValuedFieldPartiallyUnmapped() throws IOException { MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER); @@ -425,7 +424,6 @@ public void testSingleValuedFieldPartiallyUnmapped() throws IOException { } } - @AwaitsFix(bugUrl = "Replace with integration test") public void testSingleValuedFieldPartiallyUnmappedWithMissing() throws IOException { MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER); diff --git a/test/framework/javac.20200228_143103.args b/test/framework/javac.20200228_143103.args new file mode 100644 index 0000000000000..080cfaf01b106 --- /dev/null +++ b/test/framework/javac.20200228_143103.args @@ -0,0 +1,237 @@ +-d +/Users/matriv/elastic/elasticsearch/test/framework/build-idea/classes/java/main +-encoding +UTF-8 +-g +-sourcepath + +-proc:none +-s +/Users/matriv/elastic/elasticsearch/test/framework/build-idea/generated/sources/annotationProcessor/java/main +-XDuseUnsharedTable=true +-classpath +/Users/matriv/elastic/elasticsearch/client/sniffer/build-idea/distributions/elasticsearch-rest-client-sniffer-8.0.0-SNAPSHOT.jar:/Users/matriv/elastic/elasticsearch/client/rest/build-idea/distributions/elasticsearch-rest-client-8.0.0-SNAPSHOT.jar:/Users/matriv/elastic/elasticsearch/libs/nio/build-idea/distributions/elasticsearch-nio-8.0.0-SNAPSHOT.jar:/Users/matriv/elastic/elasticsearch/server/build-idea/distributions/elasticsearch-8.0.0-SNAPSHOT.jar:/Users/matriv/elastic/elasticsearch/libs/cli/build-idea/distributions/elasticsearch-cli-8.0.0-SNAPSHOT.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/com.carrotsearch.randomizedtesting/randomizedtesting-runner/2.7.4/afb7443bad2efc06c616c8a0b2aa6e5886266eb3/randomizedtesting-runner-2.7.4.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/junit/junit/4.12/2973d150c0dc1fefe998f834810d68f278ea58ec/junit-4.12.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.hamcrest/hamcrest/2.1/9420ba32c29217b54eebd26ff7f9234d31c3fbb2/hamcrest-2.1.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-test-framework/8.5.0-snapshot-b01d7cb/1939ee40d9d67356bfc6ee68c9c8fe90fc76e88d/lucene-test-framework-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-codecs/8.5.0-snapshot-b01d7cb/bd6dc6e715daab59fd28217b69d7fa4e944fa444/lucene-codecs-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/commons-logging/commons-logging/1.1.3/f6f66e966c70a83ffbdb6f17a0919eaf7c8aca7f/commons-logging-1.1.3.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/commons-codec/commons-codec/1.11/3acb4705652e16236558f0f4f2192cc33c3bd189/commons-codec-1.11.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.elasticsearch/securemock/1.2/98201d4ad5ac93f6b415ae9172d52b5e7cda490e/securemock-1.2.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.elasticsearch/mocksocket/1.2/190ad3f42fd6ab7367bae0c0e14d1dc4e4a3c361/mocksocket-1.2.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.httpcomponents/httpclient/4.5.10/7ca2e4276f4ef95e4db725a8cd4a1d1e7585b9e5/httpclient-4.5.10.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.httpcomponents/httpcore/4.4.12/21ebaf6d532bc350ba95bd81938fa5f0e511c132/httpcore-4.4.12.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.httpcomponents/httpasyncclient/4.1.4/f3a3240681faae3fa46b573a4c7e50cec9db0d86/httpasyncclient-4.1.4.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.httpcomponents/httpcore-nio/4.4.12/84cd29eca842f31db02987cfedea245af020198b/httpcore-nio-4.4.12.jar:/Users/matriv/elastic/elasticsearch/libs/x-content/build-idea/distributions/elasticsearch-x-content-8.0.0-SNAPSHOT.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/com.fasterxml.jackson.core/jackson-core/2.8.11/876ead1db19f0c9e79c9789273a3ef8c6fd6c29b/jackson-core-2.8.11.jar:/Users/matriv/elastic/elasticsearch/libs/core/build-idea/distributions/elasticsearch-core-8.0.0-SNAPSHOT.jar:/Users/matriv/elastic/elasticsearch/libs/secure-sm/build-idea/distributions/elasticsearch-secure-sm-8.0.0-SNAPSHOT.jar:/Users/matriv/elastic/elasticsearch/libs/geo/build-idea/distributions/elasticsearch-geo-8.0.0-SNAPSHOT.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-core/8.5.0-snapshot-b01d7cb/9c54e267bfa2cd1ef904dc6e35b38bbedda4c4b1/lucene-core-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-analyzers-common/8.5.0-snapshot-b01d7cb/4d358d392a9825c1502499d0a649730208f03777/lucene-analyzers-common-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-backward-codecs/8.5.0-snapshot-b01d7cb/17a0b6597c5bbe6cc2fc6d46003cdf9bc937c593/lucene-backward-codecs-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-grouping/8.5.0-snapshot-b01d7cb/a14fb545a10fcead07fbb0d801fdebce88192211/lucene-grouping-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-highlighter/8.5.0-snapshot-b01d7cb/25314c94f5cbcd8e360b74fb9980fc42e3641c94/lucene-highlighter-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-join/8.5.0-snapshot-b01d7cb/efe41313b6733a0d2764052972c4e27a7ca7636f/lucene-join-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-memory/8.5.0-snapshot-b01d7cb/f19fb11eb40f9d1bf77272de1cbd8c83e6463829/lucene-memory-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-misc/8.5.0-snapshot-b01d7cb/4295e60e1079c32dc19d1b070db3e62e04f07232/lucene-misc-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-queries/8.5.0-snapshot-b01d7cb/1555f90440c10250e084d36c27e0da0b760d0296/lucene-queries-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-queryparser/8.5.0-snapshot-b01d7cb/4fd374ffc3ec3c82d62d0811d78b044833a8dfa1/lucene-queryparser-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-sandbox/8.5.0-snapshot-b01d7cb/9e47dbe97818dd6caccf03e2ead6282d5e86bd3c/lucene-sandbox-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-spatial-extras/8.5.0-snapshot-b01d7cb/bb9fa551d6d0929d5fd37c524c8c284de3f6786e/lucene-spatial-extras-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-spatial3d/8.5.0-snapshot-b01d7cb/28c935b5eb84a85ac24432c8c630fa4b47f5fd54/lucene-spatial3d-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-suggest/8.5.0-snapshot-b01d7cb/25089b0f516df58e95f345504607853e7ba0a35d/lucene-suggest-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/com.carrotsearch/hppc/0.8.1/ffc7ba8f289428b9508ab484b8001dea944ae603/hppc-0.8.1.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/joda-time/joda-time/2.10.4/8c10bb8815109067ce3c91a8e547b5a52e8a1c1a/joda-time-2.10.4.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/com.tdunning/t-digest/3.2/2ab94758b0276a8a26102adf8d528cf6d0567b9a/t-digest-3.2.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.hdrhistogram/HdrHistogram/2.1.9/e4631ce165eb400edecfa32e03d3f1be53dee754/HdrHistogram-2.1.9.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.locationtech.spatial4j/spatial4j/0.7/faa8ba85d503da4ab872d17ba8c00da0098ab2f2/spatial4j-0.7.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.locationtech.jts/jts-core/1.15.0/705981b7e25d05a76a3654e597dab6ba423eb79e/jts-core-1.15.0.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.logging.log4j/log4j-api/2.11.1/268f0fe4df3eefe052b57c87ec48517d64fb2a10/log4j-api-2.11.1.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.logging.log4j/log4j-core/2.11.1/592a48674c926b01a9a747c7831bcd82a9e6d6e4/log4j-core-2.11.1.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.elasticsearch/jna/4.5.1/da10908ae23dc59b19dc258e63aea1c44621dc3a/jna-4.5.1.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/net.sf.jopt-simple/jopt-simple/5.0.2/98cafc6081d5632b61be2c9e60650b64ddbc637c/jopt-simple-5.0.2.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.yaml/snakeyaml/1.17/7a27ea250c5130b2922b86dea63cbb1cc10a660c/snakeyaml-1.17.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/com.fasterxml.jackson.dataformat/jackson-dataformat-smile/2.8.11/d9d1c49c5d9d5e46e2aee55f3cdd119286fe0fc1/jackson-dataformat-smile-2.8.11.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/2.8.11/2e77c6ff7342cd61ab1ae7cb14ed16aebfc8a72a/jackson-dataformat-yaml-2.8.11.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/com.fasterxml.jackson.dataformat/jackson-dataformat-cbor/2.8.11/8b9826e16c3366764bfb7ad7362554f0471046c3/jackson-dataformat-cbor-2.8.11.jar +-Xlint:-cast,-rawtypes,-unchecked +-Werror +-Xlint:all,-path,-serial,-options,-deprecation,-try +-Xdoclint:all +-Xdoclint:-missing +-proc:none +--release +11 +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cluster/coordination/LinearizabilityChecker.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cluster/coordination/DeterministicTaskQueue.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cluster/coordination/CoordinationStateTestCluster.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterService.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/indices/recovery/AsyncRecoveryTarget.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/transport/TestProfiles.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/transport/FakeTcpChannel.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransportPlugin.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/transport/nio/TestEventHandler.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/hamcrest/TupleMatchers.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/hamcrest/OptionalMatchers.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchMatchers.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/hamcrest/CollectionAssertions.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/hamcrest/CollectionMatchers.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/hamcrest/RegexMatcher.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/InternalMultiBucketAggregationTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetaData.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/MockHttpTransport.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/DummyShardLock.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManagerListener.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/IndexSettingsModule.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractBroadcastResponseTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/fixture/AbstractHttpFixture.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/junit/annotations/Network.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/junit/annotations/TestLogging.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/junit/annotations/TestIssueLogging.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/LoggingListener.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/NotEqualMessageBuilder.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/StreamsUtils.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/PosixPermissionsResetter.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractNamedWriteableTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/MockKeywordPlugin.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/DiffableTestUtils.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractDiffableSerializationTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractBootstrapCheckTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractDiffableWireSerializationTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/engine/MockInternalEngine.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineFactory.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/disruption/ServiceDisruptionScheme.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/disruption/BusyMasterServiceDisruption.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/disruption/NetworkDisruption.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockMasterServiceOnMaster.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/disruption/SingleNodeDisruption.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractSerializingTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/client/RandomizingClient.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/EqualsHashCodeTestUtils.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsTrueAssertion.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanAssertion.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ContainsAssertion.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/Assertion.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/VersionRange.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ApiCallSection.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LengthAssertion.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ParserUtils.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TransformAndSetSection.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetSection.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/package-info.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsFalseAssertion.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponseException.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestCandidate.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ObjectPath.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/BlacklistedPathPatternMatcher.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Stash.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApi.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParser.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/package-info.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/package-info.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/package-info.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/RestActionTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/gateway/TestGatewayAllocator.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/ESTokenStreamTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractWireTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/NodeConfigurationSource.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryFactory.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/BlobContainerWrapper.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/BlobStoreWrapper.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/plugins/PluginTestUtil.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/ingest/TestTemplateService.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/ingest/IngestDocumentMatcher.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/geo/GeometryTestUtils.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cli/CommandTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cli/MockTerminal.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/threadpool/TestThreadPool.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/script/ScoreAccessor.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/script/MockDeterministicScript.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/script/MockScriptService.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/script/MockScriptPlugin.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/search/aggregations/InternalSingleBucketAggregationTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/AbstractTermsTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/env/TestEnvironment.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/util/NamedFormatter.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/util/MockPageCacheRecycler.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/inject/ModuleTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/io/PathUtilsForTesting.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogsIntegTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogsStream.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/logging/TestThreadInfoPatternConverter.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogLine.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/action/support/ActionTestUtils.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/action/support/replication/TransportWriteActionTestHelper.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/http/NullDispatcher.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/MockEngineFactoryPlugin.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/alias/RandomAliasActionsGenerator.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseUtils.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/analysis/MyFilterTokenFilterFactory.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldFilterPlugin.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/engine/DocIdSeqNoAndSource.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/engine/InternalTestEngine.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/store/EsBaseDirectoryTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/node/MockNode.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/node/RecoverySettingsChunkSizePlugin.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/node/NodeMocksPlugin.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/client/RestClientBuilderTestCase.java +/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java From 8b38977ffd8682adad31a1063049c42f436b2cce Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Fri, 28 Feb 2020 15:24:26 +0100 Subject: [PATCH 13/26] fix checkstyle - enhance comments --- .../search/internal/ContextIndexSearcher.java | 125 ++++++++++-------- 1 file changed, 68 insertions(+), 57 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 8b3f9db1fd91c..0cb9ff8b9e296 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -74,6 +74,9 @@ import java.util.List; import java.util.Set; +import static org.apache.lucene.index.FilterLeafReader.FilterTerms; +import static org.apache.lucene.index.FilterLeafReader.FilterTermsEnum; + /** * Context-aware extension of {@link IndexSearcher}. */ @@ -95,7 +98,8 @@ public ContextIndexSearcher(IndexReader reader, Similarity similarity, // TODO: Make the 2nd constructor private so that the dirCancellable is never null and the IndexReader is always wrapped. // Some issues must be fixed: - // - regarding tests deriving from AggregatorTestCase and more specifically the use of searchAndReduce and the ShardSearcher sub-searchers. + // - regarding tests deriving from AggregatorTestCase and more specifically the use of searchAndReduce and + // the ShardSearcher sub-searchers. // - tests that use a MultiReader public ContextIndexSearcher(IndexReader reader, Similarity similarity, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, @@ -340,7 +344,54 @@ public DirectoryReader getDirectoryReader() { } /** - * Wraps an {@link IndexReader} with a cancellation Runnable task. + * iFace which implements the query timeout / cancellation logic + */ + public interface Cancellable { + + boolean isEnabled(); + void checkCancelled(); + default void checkDirReaderCancelled() { + checkCancelled(); + } + void unsetCheckTimeout(); + } + + public static class CancellableImpl implements Cancellable { + + private Runnable checkCancelled; + private Runnable checkTimeout; + + private CancellableImpl() { + } + + public CancellableImpl(Runnable checkTimeout, Runnable checkCancelled) { + this.checkCancelled = checkCancelled; + this.checkTimeout = checkTimeout; + } + + @Override + public boolean isEnabled() { + return checkCancelled != null || checkTimeout != null; + } + + @Override + public void checkCancelled() { + if (checkTimeout != null) { + checkTimeout.run(); + } + if (checkCancelled != null) { + checkCancelled.run(); + } + } + + @Override + public void unsetCheckTimeout() { + this.checkTimeout = null; + } + } + + /** + * Wraps an {@link IndexReader} with a {@link Cancellable}. */ private static class CancellableDirectoryReader extends FilterDirectoryReader { @@ -365,7 +416,7 @@ public CacheHelper getReaderCacheHelper() { } /** - * Wraps a leaf reader with a cancellable task + * Wraps a {@link FilterLeafReader} with a {@link Cancellable}. */ private static class CancellableLeafReader extends FilterLeafReader { @@ -406,10 +457,16 @@ public CacheHelper getReaderCacheHelper() { } } + /** + * Helper class to be used as an immutable reference so that the underlying + * {@link Cancellable} passed trough the hierarchy to the {@link Terms} and {@link PointValues} + * during construction can be set later with {@link ContextIndexSearcher#setCancellable} + */ private static class Holder { + private T in; - public Holder(T in) { + private Holder(T in) { this.in = in; } @@ -422,59 +479,14 @@ private T get() { } } - public interface Cancellable { - - boolean isEnabled(); - void checkCancelled(); - default void checkDirReaderCancelled() { - checkCancelled(); - } - void unsetCheckTimeout(); - } - - public static class CancellableImpl implements Cancellable { - - private Runnable checkCancelled; - private Runnable checkTimeout; - - private CancellableImpl() { - } - - public CancellableImpl(Runnable checkTimeout, Runnable checkCancelled) { - this.checkCancelled = checkCancelled; - this.checkTimeout = checkTimeout; - } - - @Override - public boolean isEnabled() { - return checkCancelled != null || checkTimeout != null; - } - - @Override - public void checkCancelled() { - if (checkTimeout != null) { - checkTimeout.run(); - } - if (checkCancelled != null) { - checkCancelled.run();; - } - } - - @Override - public void unsetCheckTimeout() { - this.checkTimeout = null; - } - } - /** - * Wrapper class for terms that check for query cancellation or timeout. + * Wrapper class for {@link FilterTerms} that check for query cancellation or timeout. */ - public static class ExitableTerms extends FilterLeafReader.FilterTerms { + private static class ExitableTerms extends FilterTerms { private final Cancellable cancellable; - /** Constructor **/ - public ExitableTerms(Terms terms, Cancellable cancellable) { + private ExitableTerms(Terms terms, Cancellable cancellable) { super(terms); this.cancellable = cancellable; } @@ -491,14 +503,13 @@ public TermsEnum iterator() throws IOException { } /** - * Wrapper class for TermsEnum that is used by ExitableTerms for implementing an - * exitable enumeration of terms. + * Wrapper class for {@link FilterTermsEnum} that is used by {@link ExitableTerms} for + * implementing an exitable enumeration of terms. */ - private static class ExitableTermsEnum extends FilterLeafReader.FilterTermsEnum { + private static class ExitableTermsEnum extends FilterTermsEnum { private final Cancellable cancellable; - /** Constructor **/ private ExitableTermsEnum(TermsEnum termsEnum, Cancellable cancellable) { super(termsEnum); this.cancellable = cancellable; @@ -514,7 +525,7 @@ public BytesRef next() throws IOException { } /** - * Wrapper class for PointValues that checks for cancellation and timeout. + * Wrapper class for {@link PointValues} that checks for query cancellation or timeout. */ private static class ExitablePointValues extends PointValues { From 9ebd847614c3c6c4f12ec2df7682c18fed6d67ff Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Fri, 28 Feb 2020 15:53:17 +0100 Subject: [PATCH 14/26] Fix NPE --- .../search/internal/ContextIndexSearcher.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 0cb9ff8b9e296..570dbebb6eaad 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -104,8 +104,12 @@ public ContextIndexSearcher(IndexReader reader, Similarity similarity, public ContextIndexSearcher(IndexReader reader, Similarity similarity, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, Holder cancellable) throws IOException { - super(cancellable!= null? new CancellableDirectoryReader((DirectoryReader) reader, cancellable) : reader); - this.cancellable = cancellable; + super(cancellable != null ? new CancellableDirectoryReader((DirectoryReader) reader, cancellable) : reader); + if (cancellable == null) { + this.cancellable = new Holder<>(new CancellableImpl()); + } else { + this.cancellable = cancellable; + } setSimilarity(similarity); setQueryCache(queryCache); setQueryCachingPolicy(queryCachingPolicy); From 0bf64f0efbde5c080fab3c51667a81fe113cbcd2 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Sat, 29 Feb 2020 12:42:57 +0100 Subject: [PATCH 15/26] address comments --- .../search/internal/ContextIndexSearcher.java | 50 ++-- .../search/SearchCancellationTests.java | 4 +- .../metrics/AvgAggregatorTests.java | 22 +- .../HDRPercentileRanksAggregatorTests.java | 33 +-- .../metrics/MaxAggregatorTests.java | 22 +- .../metrics/MinAggregatorTests.java | 111 ++++---- ...TDigestPercentileRanksAggregatorTests.java | 28 +-- test/framework/javac.20200228_143103.args | 237 ------------------ 8 files changed, 131 insertions(+), 376 deletions(-) delete mode 100644 test/framework/javac.20200228_143103.args diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 570dbebb6eaad..c1bf47c7571e1 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -72,6 +72,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.Set; import static org.apache.lucene.index.FilterLeafReader.FilterTerms; @@ -89,7 +90,7 @@ public class ContextIndexSearcher extends IndexSearcher { private AggregatedDfs aggregatedDfs; private QueryProfiler profiler; - private Holder cancellable; + private Holder cancellable; public ContextIndexSearcher(IndexReader reader, Similarity similarity, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy) throws IOException { @@ -103,7 +104,7 @@ public ContextIndexSearcher(IndexReader reader, Similarity similarity, // - tests that use a MultiReader public ContextIndexSearcher(IndexReader reader, Similarity similarity, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, - Holder cancellable) throws IOException { + Holder cancellable) throws IOException { super(cancellable != null ? new CancellableDirectoryReader((DirectoryReader) reader, cancellable) : reader); if (cancellable == null) { this.cancellable = new Holder<>(new CancellableImpl()); @@ -123,8 +124,8 @@ public void setProfiler(QueryProfiler profiler) { * Set a {@link Runnable} that will be run on a regular basis while * collecting documents and check for query cancellation or timeout */ - public void setCancellable(Cancellable cancellable) { - this.cancellable.set(cancellable); + public void setCancellable(QueryCancellable cancellable) { + this.cancellable.set(Objects.requireNonNull(cancellable, "queryCancellable should not be null")); } public void unsetCheckTimeout() { @@ -269,8 +270,9 @@ public Scorer scorer(LeafReaderContext context) throws IOException { @Override public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { BulkScorer in = weight.bulkScorer(context); + QueryCancellable checkCancelled = cancellable.get(); if (in != null) { - return new CancellableBulkScorer(in, () -> cancellable.get().checkCancelled()); + return new CancellableBulkScorer(in, () -> checkCancelled.checkCancelled()); } else { return null; } @@ -350,7 +352,7 @@ public DirectoryReader getDirectoryReader() { /** * iFace which implements the query timeout / cancellation logic */ - public interface Cancellable { + public interface QueryCancellable { boolean isEnabled(); void checkCancelled(); @@ -360,7 +362,7 @@ default void checkDirReaderCancelled() { void unsetCheckTimeout(); } - public static class CancellableImpl implements Cancellable { + public static class CancellableImpl implements QueryCancellable { private Runnable checkCancelled; private Runnable checkTimeout; @@ -395,11 +397,11 @@ public void unsetCheckTimeout() { } /** - * Wraps an {@link IndexReader} with a {@link Cancellable}. + * Wraps an {@link IndexReader} with a {@link QueryCancellable}. */ private static class CancellableDirectoryReader extends FilterDirectoryReader { - private CancellableDirectoryReader(DirectoryReader in, Holder cancellable) throws IOException { + private CancellableDirectoryReader(DirectoryReader in, Holder cancellable) throws IOException { super(in, new SubReaderWrapper() { @Override public LeafReader wrap(LeafReader reader) { @@ -410,7 +412,7 @@ public LeafReader wrap(LeafReader reader) { @Override protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) { - return in; + throw new UnsupportedOperationException("doWrapDirectoryReader() should never be invoked"); } @Override @@ -420,13 +422,13 @@ public CacheHelper getReaderCacheHelper() { } /** - * Wraps a {@link FilterLeafReader} with a {@link Cancellable}. + * Wraps a {@link FilterLeafReader} with a {@link QueryCancellable}. */ private static class CancellableLeafReader extends FilterLeafReader { - private final Holder cancellable; + private final Holder cancellable; - private CancellableLeafReader(LeafReader leafReader, Holder cancellable) { + private CancellableLeafReader(LeafReader leafReader, Holder cancellable) { super(leafReader); this.cancellable = cancellable; } @@ -463,7 +465,7 @@ public CacheHelper getReaderCacheHelper() { /** * Helper class to be used as an immutable reference so that the underlying - * {@link Cancellable} passed trough the hierarchy to the {@link Terms} and {@link PointValues} + * {@link QueryCancellable} passed trough the hierarchy to the {@link Terms} and {@link PointValues} * during construction can be set later with {@link ContextIndexSearcher#setCancellable} */ private static class Holder { @@ -488,9 +490,9 @@ private T get() { */ private static class ExitableTerms extends FilterTerms { - private final Cancellable cancellable; + private final QueryCancellable cancellable; - private ExitableTerms(Terms terms, Cancellable cancellable) { + private ExitableTerms(Terms terms, QueryCancellable cancellable) { super(terms); this.cancellable = cancellable; } @@ -512,9 +514,9 @@ public TermsEnum iterator() throws IOException { */ private static class ExitableTermsEnum extends FilterTermsEnum { - private final Cancellable cancellable; + private final QueryCancellable cancellable; - private ExitableTermsEnum(TermsEnum termsEnum, Cancellable cancellable) { + private ExitableTermsEnum(TermsEnum termsEnum, QueryCancellable cancellable) { super(termsEnum); this.cancellable = cancellable; this.cancellable.checkDirReaderCancelled(); @@ -534,9 +536,9 @@ public BytesRef next() throws IOException { private static class ExitablePointValues extends PointValues { private final PointValues in; - private final Cancellable cancellable; + private final QueryCancellable cancellable; - private ExitablePointValues(PointValues in, Cancellable cancellable) { + private ExitablePointValues(PointValues in, QueryCancellable cancellable) { this.in = in; this.cancellable = cancellable; this.cancellable.checkDirReaderCancelled(); @@ -599,19 +601,19 @@ public int getDocCount() { private static class ExitableIntersectVisitor implements PointValues.IntersectVisitor { - private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = 10; + private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = (1 << 4) - 1; // 15 private final PointValues.IntersectVisitor in; - private final Cancellable cancellable; + private final QueryCancellable cancellable; private int calls; - private ExitableIntersectVisitor(PointValues.IntersectVisitor in, Cancellable cancellable) { + private ExitableIntersectVisitor(PointValues.IntersectVisitor in, QueryCancellable cancellable) { this.in = in; this.cancellable = cancellable; } private void checkAndThrowWithSampling() { - if (calls++ % MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK == 0) { + if ((calls++ & MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK) == 0) { cancellable.checkDirReaderCancelled(); } } diff --git a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java index 064a5326f36b3..284560259acea 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java @@ -85,7 +85,7 @@ public void testCancellableCollector() throws IOException { AtomicBoolean cancelled = new AtomicBoolean(); ContextIndexSearcher searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); - searcher.setCancellable(new ContextIndexSearcher.Cancellable() { + searcher.setCancellable(new ContextIndexSearcher.QueryCancellable() { @Override public boolean isEnabled() { return true; @@ -118,7 +118,7 @@ public void testCancellableDirReader() throws IOException { AtomicBoolean cancelled = new AtomicBoolean(); ContextIndexSearcher searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); - searcher.setCancellable(new ContextIndexSearcher.Cancellable() { + searcher.setCancellable(new ContextIndexSearcher.QueryCancellable() { @Override public boolean isEnabled() { return true; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java index 52858cee25b83..46f08d786a856 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java @@ -611,8 +611,14 @@ public void testCacheAggregation() throws IOException { } indexWriter.close(); + Directory unmappedDirectory = newDirectory(); + RandomIndexWriter unmappedIndexWriter = new RandomIndexWriter(random(), unmappedDirectory); + unmappedIndexWriter.close(); + IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newIndexSearcher(indexReader); + IndexReader unamappedIndexReader = DirectoryReader.open(unmappedDirectory); + MultiReader multiReader = new MultiReader(indexReader, unamappedIndexReader); + IndexSearcher indexSearcher = newSearcher(multiReader, true, true); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER); fieldType.setName("value"); @@ -633,8 +639,9 @@ public void testCacheAggregation() throws IOException { // Test that an aggregation not using a script does get cached assertTrue(aggregator.context().getQueryShardContext().isCacheable()); - indexReader.close(); + multiReader.close(); directory.close(); + unmappedDirectory.close(); } /** @@ -650,8 +657,14 @@ public void testScriptCaching() throws IOException { } indexWriter.close(); + Directory unmappedDirectory = newDirectory(); + RandomIndexWriter unmappedIndexWriter = new RandomIndexWriter(random(), unmappedDirectory); + unmappedIndexWriter.close(); + IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newIndexSearcher(indexReader); + IndexReader unamappedIndexReader = DirectoryReader.open(unmappedDirectory); + MultiReader multiReader = new MultiReader(indexReader, unamappedIndexReader); + IndexSearcher indexSearcher = newSearcher(multiReader, true, true); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER); fieldType.setName("value"); @@ -692,7 +705,8 @@ public void testScriptCaching() throws IOException { // Test that an aggregation using a nondeterministic script does not get cached assertFalse(aggregator.context().getQueryShardContext().isCacheable()); - indexReader.close(); + multiReader.close(); directory.close(); + unmappedDirectory.close(); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java index 2b202de79f6c3..9d9c74f283b45 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java @@ -22,6 +22,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; @@ -29,51 +30,33 @@ import org.apache.lucene.util.NumericUtils; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; -import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.metrics.Percentile; +import org.elasticsearch.search.aggregations.metrics.PercentileRanks; +import org.elasticsearch.search.aggregations.metrics.PercentileRanksAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.PercentilesMethod; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; -import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.hamcrest.Matchers; import java.io.IOException; import java.util.Iterator; -import java.util.List; public class HDRPercentileRanksAggregatorTests extends AggregatorTestCase { - @Override - protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { - return new PercentileRanksAggregationBuilder("hdr_ranks", new double[]{0.1, 0.5, 12}) - .field(fieldName) - .percentilesConfig(new PercentilesConfig.Hdr()); - } - - @Override - protected List getSupportedValuesSourceTypes() { - return List.of(CoreValuesSourceType.NUMERIC); - } - public void testEmpty() throws IOException { PercentileRanksAggregationBuilder aggBuilder = new PercentileRanksAggregationBuilder("my_agg", new double[]{0.5}) .field("field") .method(PercentilesMethod.HDR); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE); fieldType.setName("field"); - - Directory directory = newDirectory(); - RandomIndexWriter unmappedIndexWriter = new RandomIndexWriter(random(), directory); - try (IndexReader reader = unmappedIndexWriter.getReader()) { + try (IndexReader reader = new MultiReader()) { IndexSearcher searcher = new IndexSearcher(reader); PercentileRanks ranks = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); Percentile rank = ranks.iterator().next(); assertEquals(Double.NaN, rank.getPercent(), 0d); assertEquals(0.5, rank.getValue(), 0d); assertFalse(AggregationInspectionHelper.hasValue((InternalHDRPercentileRanks)ranks)); - } finally { - unmappedIndexWriter.close(); - directory.close(); } } @@ -111,13 +94,13 @@ public void testSimple() throws IOException { } } - public void testNullValues() { + public void testNullValues() throws IOException { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new PercentileRanksAggregationBuilder("my_agg", null).field("field").method(PercentilesMethod.HDR)); assertThat(e.getMessage(), Matchers.equalTo("[values] must not be null: [my_agg]")); } - public void testEmptyValues() { + public void testEmptyValues() throws IOException { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new PercentileRanksAggregationBuilder("my_agg", new double[0]).field("field").method(PercentilesMethod.HDR)); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java index d43fafc22b2f7..598d19795db20 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java @@ -920,8 +920,14 @@ public void testCacheAggregation() throws IOException { } indexWriter.close(); + Directory unmappedDirectory = newDirectory(); + RandomIndexWriter unmappedIndexWriter = new RandomIndexWriter(random(), unmappedDirectory); + unmappedIndexWriter.close(); + IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newIndexSearcher(indexReader); + IndexReader unamappedIndexReader = DirectoryReader.open(unmappedDirectory); + MultiReader multiReader = new MultiReader(indexReader, unamappedIndexReader); + IndexSearcher indexSearcher = newSearcher(multiReader, true, true); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER); @@ -943,8 +949,9 @@ public void testCacheAggregation() throws IOException { // Test that an aggregation not using a script does get cached assertTrue(aggregator.context().getQueryShardContext().isCacheable()); - indexReader.close(); + multiReader.close(); directory.close(); + unmappedDirectory.close(); } /** @@ -960,8 +967,14 @@ public void testScriptCaching() throws Exception { } indexWriter.close(); + Directory unmappedDirectory = newDirectory(); + RandomIndexWriter unmappedIndexWriter = new RandomIndexWriter(random(), unmappedDirectory); + unmappedIndexWriter.close(); + IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newIndexSearcher(indexReader); + IndexReader unamappedIndexReader = DirectoryReader.open(unmappedDirectory); + MultiReader multiReader = new MultiReader(indexReader, unamappedIndexReader); + IndexSearcher indexSearcher = newSearcher(multiReader, true, true); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER); fieldType.setName("value"); @@ -1001,7 +1014,8 @@ public void testScriptCaching() throws Exception { // Test that an aggregation using a nondeterministic script does not get cached assertFalse(aggregator.context().getQueryShardContext().isCacheable()); - indexReader.close(); + multiReader.close(); directory.close(); + unmappedDirectory.close(); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java index f1befc7291e39..d6b65dfc62d74 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java @@ -46,17 +46,13 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.query.QueryShardContext; @@ -91,9 +87,9 @@ import org.elasticsearch.search.lookup.LeafDocLookup; import java.io.IOException; -import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -101,6 +97,7 @@ import java.util.Map; import java.util.function.BiFunction; import java.util.function.Consumer; +import java.util.function.DoubleConsumer; import java.util.function.Function; import java.util.function.Supplier; @@ -381,7 +378,7 @@ public void testGetProperty() throws IOException { iw.addDocument(singleton(new NumericDocValuesField("number", 7))); iw.addDocument(singleton(new NumericDocValuesField("number", 1))); }, (Consumer) global -> { - assertEquals(2, global.getDocCount()); + assertEquals(1.0, global.getDocCount(), 2); assertTrue(AggregationInspectionHelper.hasValue(global)); assertNotNull(global.getAggregations().asMap().get("min")); @@ -743,56 +740,41 @@ public void testShortcutIsApplicable() { ) ); } - for (DateFieldMapper.Resolution resolution : DateFieldMapper.Resolution.values()) { - assertNull( - MinAggregator.getPointReaderOrNull( - mockSearchContext(new MatchAllDocsQuery()), - mockAggregator(), - mockDateValuesSourceConfig("number", true, resolution) - ) - ); - assertNull( - MinAggregator.getPointReaderOrNull( - mockSearchContext(new TermQuery(new Term("foo", "bar"))), - null, - mockDateValuesSourceConfig("number", true, resolution) - ) - ); - assertNull( - MinAggregator.getPointReaderOrNull( - mockSearchContext(null), - mockAggregator(), - mockDateValuesSourceConfig("number", true, resolution) - ) - ); - assertNull( - MinAggregator.getPointReaderOrNull( - mockSearchContext(null), - null, - mockDateValuesSourceConfig("number", false, resolution) - ) - ); - } - // Check that we decode a dates "just like" the doc values instance. - Instant expected = Instant.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2020-01-01T00:00:00Z")); - byte[] scratch = new byte[8]; - LongPoint.encodeDimension(DateFieldMapper.Resolution.MILLISECONDS.convert(expected), scratch, 0); - assertThat( + assertNotNull( MinAggregator.getPointReaderOrNull( mockSearchContext(new MatchAllDocsQuery()), null, - mockDateValuesSourceConfig("number", true, DateFieldMapper.Resolution.MILLISECONDS) - ).apply(scratch), equalTo(expected.toEpochMilli()) + mockDateValuesSourceConfig("number", true) + ) ); - LongPoint.encodeDimension(DateFieldMapper.Resolution.NANOSECONDS.convert(expected), scratch, 0); - assertThat( + assertNull( MinAggregator.getPointReaderOrNull( mockSearchContext(new MatchAllDocsQuery()), + mockAggregator(), + mockDateValuesSourceConfig("number", true) + ) + ); + assertNull( + MinAggregator.getPointReaderOrNull( + mockSearchContext(new TermQuery(new Term("foo", "bar"))), null, - mockDateValuesSourceConfig("number", true, DateFieldMapper.Resolution.NANOSECONDS) - ).apply(scratch), equalTo(expected.toEpochMilli()) + mockDateValuesSourceConfig("number", true) + ) + ); + assertNull( + MinAggregator.getPointReaderOrNull( + mockSearchContext(null), + mockAggregator(), + mockDateValuesSourceConfig("number", true) + ) + ); + assertNull( + MinAggregator.getPointReaderOrNull( + mockSearchContext(null), + null, + mockDateValuesSourceConfig("number", false) + ) ); - } public void testMinShortcutRandom() throws Exception { @@ -817,6 +799,21 @@ public void testMinShortcutRandom() throws Exception { (v) -> DoublePoint.decodeDimension(v, 0)); } + private void testMinCase(IndexSearcher searcher, + AggregationBuilder aggregationBuilder, + MappedFieldType ft, + DoubleConsumer testResult) throws IOException { + Collection queries = Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(ft.name())); + for (Query query : queries) { + MinAggregator aggregator = createAggregator(query, aggregationBuilder, searcher, createIndexSettings(), ft); + aggregator.preCollection(); + searcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + InternalMin result = (InternalMin) aggregator.buildAggregation(0L); + testResult.accept(result.getValue()); + } + } + private void testMinShortcutCase(Supplier randomNumber, Function pointFieldFunc, Function pointConvertFunc) throws IOException { @@ -892,17 +889,12 @@ private ValuesSourceConfig mockNumericValuesSourceConfig(S return config; } - private ValuesSourceConfig mockDateValuesSourceConfig(String fieldName, boolean indexed, - DateFieldMapper.Resolution resolution) { + private ValuesSourceConfig mockDateValuesSourceConfig(String fieldName, boolean indexed) { ValuesSourceConfig config = mock(ValuesSourceConfig.class); - Mapper.BuilderContext builderContext = new Mapper.BuilderContext( - Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(), - new ContentPath()); - MappedFieldType ft = new DateFieldMapper.Builder(fieldName) - .index(indexed) - .withResolution(resolution) - .build(builderContext) - .fieldType(); + MappedFieldType ft = new DateFieldMapper.Builder(fieldName).fieldType(); + ft.setName(fieldName); + ft.setIndexOptions(indexed ? IndexOptions.DOCS : IndexOptions.NONE); + ft.freeze(); when(config.fieldContext()).thenReturn(new FieldContext(fieldName, null, ft)); return config; } @@ -929,7 +921,10 @@ private void testC V agg = searchAndReduce(indexSearcher, query, aggregationBuilder, fieldType); verify.accept(agg); + } } } + + } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java index 001736d49c9df..2541583e94580 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java @@ -22,6 +22,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; @@ -29,44 +30,27 @@ import org.apache.lucene.util.NumericUtils; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; -import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.metrics.Percentile; +import org.elasticsearch.search.aggregations.metrics.PercentileRanks; +import org.elasticsearch.search.aggregations.metrics.PercentileRanksAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.PercentilesMethod; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; -import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.hamcrest.Matchers; import java.io.IOException; import java.util.Iterator; -import java.util.List; public class TDigestPercentileRanksAggregatorTests extends AggregatorTestCase { - @Override - protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { - return new PercentileRanksAggregationBuilder("tdigest_ranks", new double[]{0.1, 0.5, 12}) - .field(fieldName) - .percentilesConfig(new PercentilesConfig.TDigest()); - } - - @Override - protected List getSupportedValuesSourceTypes() { - return List.of(CoreValuesSourceType.NUMERIC); - } - public void testEmpty() throws IOException { PercentileRanksAggregationBuilder aggBuilder = new PercentileRanksAggregationBuilder("my_agg", new double[]{0.5}) .field("field") .method(PercentilesMethod.TDIGEST); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE); fieldType.setName("field"); - - - - try (Directory directory = newDirectory(); - RandomIndexWriter unmappedIndexWriter = new RandomIndexWriter(random(), directory); - IndexReader reader = unmappedIndexWriter.getReader()) { + try (IndexReader reader = new MultiReader()) { IndexSearcher searcher = new IndexSearcher(reader); PercentileRanks ranks = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); Percentile rank = ranks.iterator().next(); diff --git a/test/framework/javac.20200228_143103.args b/test/framework/javac.20200228_143103.args deleted file mode 100644 index 080cfaf01b106..0000000000000 --- a/test/framework/javac.20200228_143103.args +++ /dev/null @@ -1,237 +0,0 @@ --d -/Users/matriv/elastic/elasticsearch/test/framework/build-idea/classes/java/main --encoding -UTF-8 --g --sourcepath - --proc:none --s -/Users/matriv/elastic/elasticsearch/test/framework/build-idea/generated/sources/annotationProcessor/java/main --XDuseUnsharedTable=true --classpath -/Users/matriv/elastic/elasticsearch/client/sniffer/build-idea/distributions/elasticsearch-rest-client-sniffer-8.0.0-SNAPSHOT.jar:/Users/matriv/elastic/elasticsearch/client/rest/build-idea/distributions/elasticsearch-rest-client-8.0.0-SNAPSHOT.jar:/Users/matriv/elastic/elasticsearch/libs/nio/build-idea/distributions/elasticsearch-nio-8.0.0-SNAPSHOT.jar:/Users/matriv/elastic/elasticsearch/server/build-idea/distributions/elasticsearch-8.0.0-SNAPSHOT.jar:/Users/matriv/elastic/elasticsearch/libs/cli/build-idea/distributions/elasticsearch-cli-8.0.0-SNAPSHOT.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/com.carrotsearch.randomizedtesting/randomizedtesting-runner/2.7.4/afb7443bad2efc06c616c8a0b2aa6e5886266eb3/randomizedtesting-runner-2.7.4.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/junit/junit/4.12/2973d150c0dc1fefe998f834810d68f278ea58ec/junit-4.12.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.hamcrest/hamcrest/2.1/9420ba32c29217b54eebd26ff7f9234d31c3fbb2/hamcrest-2.1.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-test-framework/8.5.0-snapshot-b01d7cb/1939ee40d9d67356bfc6ee68c9c8fe90fc76e88d/lucene-test-framework-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-codecs/8.5.0-snapshot-b01d7cb/bd6dc6e715daab59fd28217b69d7fa4e944fa444/lucene-codecs-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/commons-logging/commons-logging/1.1.3/f6f66e966c70a83ffbdb6f17a0919eaf7c8aca7f/commons-logging-1.1.3.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/commons-codec/commons-codec/1.11/3acb4705652e16236558f0f4f2192cc33c3bd189/commons-codec-1.11.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.elasticsearch/securemock/1.2/98201d4ad5ac93f6b415ae9172d52b5e7cda490e/securemock-1.2.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.elasticsearch/mocksocket/1.2/190ad3f42fd6ab7367bae0c0e14d1dc4e4a3c361/mocksocket-1.2.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.httpcomponents/httpclient/4.5.10/7ca2e4276f4ef95e4db725a8cd4a1d1e7585b9e5/httpclient-4.5.10.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.httpcomponents/httpcore/4.4.12/21ebaf6d532bc350ba95bd81938fa5f0e511c132/httpcore-4.4.12.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.httpcomponents/httpasyncclient/4.1.4/f3a3240681faae3fa46b573a4c7e50cec9db0d86/httpasyncclient-4.1.4.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.httpcomponents/httpcore-nio/4.4.12/84cd29eca842f31db02987cfedea245af020198b/httpcore-nio-4.4.12.jar:/Users/matriv/elastic/elasticsearch/libs/x-content/build-idea/distributions/elasticsearch-x-content-8.0.0-SNAPSHOT.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/com.fasterxml.jackson.core/jackson-core/2.8.11/876ead1db19f0c9e79c9789273a3ef8c6fd6c29b/jackson-core-2.8.11.jar:/Users/matriv/elastic/elasticsearch/libs/core/build-idea/distributions/elasticsearch-core-8.0.0-SNAPSHOT.jar:/Users/matriv/elastic/elasticsearch/libs/secure-sm/build-idea/distributions/elasticsearch-secure-sm-8.0.0-SNAPSHOT.jar:/Users/matriv/elastic/elasticsearch/libs/geo/build-idea/distributions/elasticsearch-geo-8.0.0-SNAPSHOT.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-core/8.5.0-snapshot-b01d7cb/9c54e267bfa2cd1ef904dc6e35b38bbedda4c4b1/lucene-core-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-analyzers-common/8.5.0-snapshot-b01d7cb/4d358d392a9825c1502499d0a649730208f03777/lucene-analyzers-common-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-backward-codecs/8.5.0-snapshot-b01d7cb/17a0b6597c5bbe6cc2fc6d46003cdf9bc937c593/lucene-backward-codecs-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-grouping/8.5.0-snapshot-b01d7cb/a14fb545a10fcead07fbb0d801fdebce88192211/lucene-grouping-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-highlighter/8.5.0-snapshot-b01d7cb/25314c94f5cbcd8e360b74fb9980fc42e3641c94/lucene-highlighter-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-join/8.5.0-snapshot-b01d7cb/efe41313b6733a0d2764052972c4e27a7ca7636f/lucene-join-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-memory/8.5.0-snapshot-b01d7cb/f19fb11eb40f9d1bf77272de1cbd8c83e6463829/lucene-memory-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-misc/8.5.0-snapshot-b01d7cb/4295e60e1079c32dc19d1b070db3e62e04f07232/lucene-misc-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-queries/8.5.0-snapshot-b01d7cb/1555f90440c10250e084d36c27e0da0b760d0296/lucene-queries-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-queryparser/8.5.0-snapshot-b01d7cb/4fd374ffc3ec3c82d62d0811d78b044833a8dfa1/lucene-queryparser-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-sandbox/8.5.0-snapshot-b01d7cb/9e47dbe97818dd6caccf03e2ead6282d5e86bd3c/lucene-sandbox-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-spatial-extras/8.5.0-snapshot-b01d7cb/bb9fa551d6d0929d5fd37c524c8c284de3f6786e/lucene-spatial-extras-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-spatial3d/8.5.0-snapshot-b01d7cb/28c935b5eb84a85ac24432c8c630fa4b47f5fd54/lucene-spatial3d-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.lucene/lucene-suggest/8.5.0-snapshot-b01d7cb/25089b0f516df58e95f345504607853e7ba0a35d/lucene-suggest-8.5.0-snapshot-b01d7cb.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/com.carrotsearch/hppc/0.8.1/ffc7ba8f289428b9508ab484b8001dea944ae603/hppc-0.8.1.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/joda-time/joda-time/2.10.4/8c10bb8815109067ce3c91a8e547b5a52e8a1c1a/joda-time-2.10.4.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/com.tdunning/t-digest/3.2/2ab94758b0276a8a26102adf8d528cf6d0567b9a/t-digest-3.2.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.hdrhistogram/HdrHistogram/2.1.9/e4631ce165eb400edecfa32e03d3f1be53dee754/HdrHistogram-2.1.9.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.locationtech.spatial4j/spatial4j/0.7/faa8ba85d503da4ab872d17ba8c00da0098ab2f2/spatial4j-0.7.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.locationtech.jts/jts-core/1.15.0/705981b7e25d05a76a3654e597dab6ba423eb79e/jts-core-1.15.0.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.logging.log4j/log4j-api/2.11.1/268f0fe4df3eefe052b57c87ec48517d64fb2a10/log4j-api-2.11.1.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.apache.logging.log4j/log4j-core/2.11.1/592a48674c926b01a9a747c7831bcd82a9e6d6e4/log4j-core-2.11.1.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.elasticsearch/jna/4.5.1/da10908ae23dc59b19dc258e63aea1c44621dc3a/jna-4.5.1.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/net.sf.jopt-simple/jopt-simple/5.0.2/98cafc6081d5632b61be2c9e60650b64ddbc637c/jopt-simple-5.0.2.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/org.yaml/snakeyaml/1.17/7a27ea250c5130b2922b86dea63cbb1cc10a660c/snakeyaml-1.17.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/com.fasterxml.jackson.dataformat/jackson-dataformat-smile/2.8.11/d9d1c49c5d9d5e46e2aee55f3cdd119286fe0fc1/jackson-dataformat-smile-2.8.11.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/2.8.11/2e77c6ff7342cd61ab1ae7cb14ed16aebfc8a72a/jackson-dataformat-yaml-2.8.11.jar:/Users/matriv/.gradle/caches/modules-2/files-2.1/com.fasterxml.jackson.dataformat/jackson-dataformat-cbor/2.8.11/8b9826e16c3366764bfb7ad7362554f0471046c3/jackson-dataformat-cbor-2.8.11.jar --Xlint:-cast,-rawtypes,-unchecked --Werror --Xlint:all,-path,-serial,-options,-deprecation,-try --Xdoclint:all --Xdoclint:-missing --proc:none ---release -11 -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cluster/coordination/LinearizabilityChecker.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cluster/coordination/DeterministicTaskQueue.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cluster/coordination/CoordinationStateTestCluster.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cluster/service/FakeThreadPoolMasterService.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/indices/recovery/AsyncRecoveryTarget.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/transport/TestProfiles.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/transport/FakeTcpChannel.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransportPlugin.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/transport/nio/TestEventHandler.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/hamcrest/TupleMatchers.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/hamcrest/OptionalMatchers.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchMatchers.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/hamcrest/CollectionAssertions.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/hamcrest/CollectionMatchers.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/hamcrest/RegexMatcher.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/InternalMultiBucketAggregationTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetaData.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/MockHttpTransport.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/DummyShardLock.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManagerListener.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableConnectionManager.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/IndexSettingsModule.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractBroadcastResponseTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/fixture/AbstractHttpFixture.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/junit/annotations/Network.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/junit/annotations/TestLogging.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/junit/annotations/TestIssueLogging.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/LoggingListener.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/NotEqualMessageBuilder.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/StreamsUtils.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/PosixPermissionsResetter.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractNamedWriteableTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/MockKeywordPlugin.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/DiffableTestUtils.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractDiffableSerializationTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractBootstrapCheckTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractDiffableWireSerializationTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/CorruptionUtils.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/engine/MockInternalEngine.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineFactory.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/disruption/ServiceDisruptionScheme.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/disruption/BusyMasterServiceDisruption.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/disruption/NetworkDisruption.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockMasterServiceOnMaster.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/disruption/SingleNodeDisruption.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractSerializingTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/client/RandomizingClient.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/EqualsHashCodeTestUtils.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsTrueAssertion.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanAssertion.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ContainsAssertion.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/Assertion.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/VersionRange.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ApiCallSection.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LengthAssertion.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ParserUtils.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TransformAndSetSection.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetSection.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/package-info.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsFalseAssertion.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponseException.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestCandidate.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ObjectPath.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/BlacklistedPathPatternMatcher.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Stash.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApi.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParser.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/package-info.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/package-info.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/package-info.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/rest/RestActionTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/gateway/TestGatewayAllocator.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/ESTokenStreamTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/AbstractWireTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/NodeConfigurationSource.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryFactory.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/BlobContainerWrapper.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/BlobStoreWrapper.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/plugins/PluginTestUtil.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/ingest/TestTemplateService.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/ingest/IngestDocumentMatcher.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/geo/GeometryTestUtils.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cli/CommandTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/cli/MockTerminal.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/threadpool/TestThreadPool.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/script/ScoreAccessor.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/script/MockDeterministicScript.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/script/MockScriptService.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/script/MockScriptPlugin.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/search/aggregations/InternalSingleBucketAggregationTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/AbstractTermsTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/env/TestEnvironment.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/util/NamedFormatter.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/util/MockPageCacheRecycler.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/inject/ModuleTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/io/PathUtilsForTesting.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogsIntegTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogsStream.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/logging/TestThreadInfoPatternConverter.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogLine.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/action/support/ActionTestUtils.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/action/support/replication/TransportWriteActionTestHelper.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/http/NullDispatcher.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/MockEngineFactoryPlugin.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/alias/RandomAliasActionsGenerator.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseUtils.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/analysis/MyFilterTokenFilterFactory.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldFilterPlugin.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/engine/DocIdSeqNoAndSource.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/engine/InternalTestEngine.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/index/store/EsBaseDirectoryTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/node/MockNode.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/node/RecoverySettingsChunkSizePlugin.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/node/NodeMocksPlugin.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/client/RestClientBuilderTestCase.java -/Users/matriv/elastic/elasticsearch/test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java From 3936a05c6adb5c38bb33d370d6e05db93dd3d872 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Sat, 29 Feb 2020 17:00:24 +0100 Subject: [PATCH 16/26] fix revert --- .../HDRPercentileRanksAggregatorTests.java | 20 +++++++++++++++---- ...TDigestPercentileRanksAggregatorTests.java | 20 +++++++++++++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java index 9d9c74f283b45..7ed434f3a202d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorTests.java @@ -30,20 +30,32 @@ import org.apache.lucene.util.NumericUtils; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; -import org.elasticsearch.search.aggregations.metrics.Percentile; -import org.elasticsearch.search.aggregations.metrics.PercentileRanks; -import org.elasticsearch.search.aggregations.metrics.PercentileRanksAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.PercentilesMethod; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.hamcrest.Matchers; import java.io.IOException; import java.util.Iterator; +import java.util.List; public class HDRPercentileRanksAggregatorTests extends AggregatorTestCase { + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return new PercentileRanksAggregationBuilder("hdr_ranks", new double[]{0.1, 0.5, 12}) + .field(fieldName) + .percentilesConfig(new PercentilesConfig.Hdr()); + } + + @Override + protected List getSupportedValuesSourceTypes() { + return List.of(CoreValuesSourceType.NUMERIC); + } + public void testEmpty() throws IOException { PercentileRanksAggregationBuilder aggBuilder = new PercentileRanksAggregationBuilder("my_agg", new double[]{0.5}) .field("field") diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java index 2541583e94580..420c9b8cecbcf 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorTests.java @@ -30,20 +30,32 @@ import org.apache.lucene.util.NumericUtils; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; -import org.elasticsearch.search.aggregations.metrics.Percentile; -import org.elasticsearch.search.aggregations.metrics.PercentileRanks; -import org.elasticsearch.search.aggregations.metrics.PercentileRanksAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.PercentilesMethod; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.hamcrest.Matchers; import java.io.IOException; import java.util.Iterator; +import java.util.List; public class TDigestPercentileRanksAggregatorTests extends AggregatorTestCase { + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return new PercentileRanksAggregationBuilder("tdigest_ranks", new double[]{0.1, 0.5, 12}) + .field(fieldName) + .percentilesConfig(new PercentilesConfig.TDigest()); + } + + @Override + protected List getSupportedValuesSourceTypes() { + return List.of(CoreValuesSourceType.NUMERIC); + } + public void testEmpty() throws IOException { PercentileRanksAggregationBuilder aggBuilder = new PercentileRanksAggregationBuilder("my_agg", new double[]{0.5}) .field("field") From 9bf0fe3d64f140fc2e534257a6eae073ea1b5c95 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Sat, 29 Feb 2020 17:01:31 +0100 Subject: [PATCH 17/26] fix revert --- .../metrics/MinAggregatorTests.java | 108 ++++++++++-------- 1 file changed, 58 insertions(+), 50 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java index d6b65dfc62d74..32c539e43348a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java @@ -46,13 +46,17 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.query.QueryShardContext; @@ -87,9 +91,9 @@ import org.elasticsearch.search.lookup.LeafDocLookup; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -97,7 +101,6 @@ import java.util.Map; import java.util.function.BiFunction; import java.util.function.Consumer; -import java.util.function.DoubleConsumer; import java.util.function.Function; import java.util.function.Supplier; @@ -378,7 +381,7 @@ public void testGetProperty() throws IOException { iw.addDocument(singleton(new NumericDocValuesField("number", 7))); iw.addDocument(singleton(new NumericDocValuesField("number", 1))); }, (Consumer) global -> { - assertEquals(1.0, global.getDocCount(), 2); + assertEquals(2, global.getDocCount()); assertTrue(AggregationInspectionHelper.hasValue(global)); assertNotNull(global.getAggregations().asMap().get("min")); @@ -740,41 +743,56 @@ public void testShortcutIsApplicable() { ) ); } - assertNotNull( + for (DateFieldMapper.Resolution resolution : DateFieldMapper.Resolution.values()) { + assertNull( + MinAggregator.getPointReaderOrNull( + mockSearchContext(new MatchAllDocsQuery()), + mockAggregator(), + mockDateValuesSourceConfig("number", true, resolution) + ) + ); + assertNull( + MinAggregator.getPointReaderOrNull( + mockSearchContext(new TermQuery(new Term("foo", "bar"))), + null, + mockDateValuesSourceConfig("number", true, resolution) + ) + ); + assertNull( + MinAggregator.getPointReaderOrNull( + mockSearchContext(null), + mockAggregator(), + mockDateValuesSourceConfig("number", true, resolution) + ) + ); + assertNull( + MinAggregator.getPointReaderOrNull( + mockSearchContext(null), + null, + mockDateValuesSourceConfig("number", false, resolution) + ) + ); + } + // Check that we decode a dates "just like" the doc values instance. + Instant expected = Instant.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2020-01-01T00:00:00Z")); + byte[] scratch = new byte[8]; + LongPoint.encodeDimension(DateFieldMapper.Resolution.MILLISECONDS.convert(expected), scratch, 0); + assertThat( MinAggregator.getPointReaderOrNull( mockSearchContext(new MatchAllDocsQuery()), null, - mockDateValuesSourceConfig("number", true) - ) + mockDateValuesSourceConfig("number", true, DateFieldMapper.Resolution.MILLISECONDS) + ).apply(scratch), equalTo(expected.toEpochMilli()) ); - assertNull( + LongPoint.encodeDimension(DateFieldMapper.Resolution.NANOSECONDS.convert(expected), scratch, 0); + assertThat( MinAggregator.getPointReaderOrNull( mockSearchContext(new MatchAllDocsQuery()), - mockAggregator(), - mockDateValuesSourceConfig("number", true) - ) - ); - assertNull( - MinAggregator.getPointReaderOrNull( - mockSearchContext(new TermQuery(new Term("foo", "bar"))), null, - mockDateValuesSourceConfig("number", true) - ) - ); - assertNull( - MinAggregator.getPointReaderOrNull( - mockSearchContext(null), - mockAggregator(), - mockDateValuesSourceConfig("number", true) - ) - ); - assertNull( - MinAggregator.getPointReaderOrNull( - mockSearchContext(null), - null, - mockDateValuesSourceConfig("number", false) - ) + mockDateValuesSourceConfig("number", true, DateFieldMapper.Resolution.NANOSECONDS) + ).apply(scratch), equalTo(expected.toEpochMilli()) ); + } public void testMinShortcutRandom() throws Exception { @@ -799,21 +817,6 @@ public void testMinShortcutRandom() throws Exception { (v) -> DoublePoint.decodeDimension(v, 0)); } - private void testMinCase(IndexSearcher searcher, - AggregationBuilder aggregationBuilder, - MappedFieldType ft, - DoubleConsumer testResult) throws IOException { - Collection queries = Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(ft.name())); - for (Query query : queries) { - MinAggregator aggregator = createAggregator(query, aggregationBuilder, searcher, createIndexSettings(), ft); - aggregator.preCollection(); - searcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - InternalMin result = (InternalMin) aggregator.buildAggregation(0L); - testResult.accept(result.getValue()); - } - } - private void testMinShortcutCase(Supplier randomNumber, Function pointFieldFunc, Function pointConvertFunc) throws IOException { @@ -889,12 +892,17 @@ private ValuesSourceConfig mockNumericValuesSourceConfig(S return config; } - private ValuesSourceConfig mockDateValuesSourceConfig(String fieldName, boolean indexed) { + private ValuesSourceConfig mockDateValuesSourceConfig(String fieldName, boolean indexed, + DateFieldMapper.Resolution resolution) { ValuesSourceConfig config = mock(ValuesSourceConfig.class); - MappedFieldType ft = new DateFieldMapper.Builder(fieldName).fieldType(); - ft.setName(fieldName); - ft.setIndexOptions(indexed ? IndexOptions.DOCS : IndexOptions.NONE); - ft.freeze(); + Mapper.BuilderContext builderContext = new Mapper.BuilderContext( + Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(), + new ContentPath()); + MappedFieldType ft = new DateFieldMapper.Builder(fieldName) + .index(indexed) + .withResolution(resolution) + .build(builderContext) + .fieldType(); when(config.fieldContext()).thenReturn(new FieldContext(fieldName, null, ft)); return config; } From ce51935a96c3d59af66205144924f3ff1b32d17e Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Sat, 29 Feb 2020 18:26:42 +0100 Subject: [PATCH 18/26] move impl to QueryPhase --- .../search/internal/ContextIndexSearcher.java | 82 ++++++------------- .../search/query/QueryPhase.java | 31 ++++++- .../search/SearchCancellationTests.java | 17 ---- .../internal/ContextIndexSearcherTests.java | 1 - .../aggregations/AggregatorTestCase.java | 2 +- 5 files changed, 56 insertions(+), 77 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index c1bf47c7571e1..d3bdabeaefa38 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -90,27 +90,22 @@ public class ContextIndexSearcher extends IndexSearcher { private AggregatedDfs aggregatedDfs; private QueryProfiler profiler; - private Holder cancellable; + private QueryCancellable cancellable; public ContextIndexSearcher(IndexReader reader, Similarity similarity, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy) throws IOException { - this(reader, similarity, queryCache, queryCachingPolicy, new Holder<>(new CancellableImpl())); + this(reader, similarity, queryCache, queryCachingPolicy, true); } - // TODO: Make the 2nd constructor private so that the dirCancellable is never null and the IndexReader is always wrapped. + // TODO: Remove the 2nd constructor so that the IndexReader is always wrapped. // Some issues must be fixed: // - regarding tests deriving from AggregatorTestCase and more specifically the use of searchAndReduce and // the ShardSearcher sub-searchers. // - tests that use a MultiReader public ContextIndexSearcher(IndexReader reader, Similarity similarity, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, - Holder cancellable) throws IOException { - super(cancellable != null ? new CancellableDirectoryReader((DirectoryReader) reader, cancellable) : reader); - if (cancellable == null) { - this.cancellable = new Holder<>(new CancellableImpl()); - } else { - this.cancellable = cancellable; - } + boolean shouldWrap) throws IOException { + super(shouldWrap ? new CancellableDirectoryReader((DirectoryReader) reader, new Holder<>(null)) : reader); setSimilarity(similarity); setQueryCache(queryCache); setQueryCachingPolicy(queryCachingPolicy); @@ -125,11 +120,14 @@ public void setProfiler(QueryProfiler profiler) { * collecting documents and check for query cancellation or timeout */ public void setCancellable(QueryCancellable cancellable) { - this.cancellable.set(Objects.requireNonNull(cancellable, "queryCancellable should not be null")); + this.cancellable = Objects.requireNonNull(cancellable, "queryCancellable should not be null"); + ((CancellableDirectoryReader) getIndexReader()).setCancellable(cancellable); } - public void unsetCheckTimeout() { - this.cancellable.get().unsetCheckTimeout(); + private void checkCancelled() { + if (cancellable != null) { + cancellable.checkCancelled(); + } } public void setAggregatedDfs(AggregatedDfs aggregatedDfs) { @@ -208,7 +206,7 @@ protected void search(List leaves, Weight weight, Collector c * the provided ctx. */ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collector) throws IOException { - cancellable.get().checkCancelled(); + checkCancelled(); weight = wrapWeight(weight); final LeafCollector leafCollector; try { @@ -235,7 +233,7 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto Scorer scorer = weight.scorer(ctx); if (scorer != null) { try { - intersectScorerAndBitSet(scorer, liveDocsBitSet, leafCollector, () -> cancellable.get().checkCancelled()); + intersectScorerAndBitSet(scorer, liveDocsBitSet, leafCollector, () -> checkCancelled()); } catch (CollectionTerminatedException e) { // collection was terminated prematurely // continue with the following leaf @@ -245,7 +243,7 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto } private Weight wrapWeight(Weight weight) { - if (cancellable.get().isEnabled()) { + if (cancellable != null) { return new Weight(weight.getQuery()) { @Override public void extractTerms(Set terms) { @@ -270,9 +268,8 @@ public Scorer scorer(LeafReaderContext context) throws IOException { @Override public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { BulkScorer in = weight.bulkScorer(context); - QueryCancellable checkCancelled = cancellable.get(); if (in != null) { - return new CancellableBulkScorer(in, () -> checkCancelled.checkCancelled()); + return new CancellableBulkScorer(in, () -> checkCancelled()); } else { return null; } @@ -354,60 +351,33 @@ public DirectoryReader getDirectoryReader() { */ public interface QueryCancellable { - boolean isEnabled(); void checkCancelled(); default void checkDirReaderCancelled() { checkCancelled(); } - void unsetCheckTimeout(); } - public static class CancellableImpl implements QueryCancellable { - - private Runnable checkCancelled; - private Runnable checkTimeout; - - private CancellableImpl() { - } - - public CancellableImpl(Runnable checkTimeout, Runnable checkCancelled) { - this.checkCancelled = checkCancelled; - this.checkTimeout = checkTimeout; - } - - @Override - public boolean isEnabled() { - return checkCancelled != null || checkTimeout != null; - } - - @Override - public void checkCancelled() { - if (checkTimeout != null) { - checkTimeout.run(); - } - if (checkCancelled != null) { - checkCancelled.run(); - } - } - @Override - public void unsetCheckTimeout() { - this.checkTimeout = null; - } - } /** * Wraps an {@link IndexReader} with a {@link QueryCancellable}. */ private static class CancellableDirectoryReader extends FilterDirectoryReader { - private CancellableDirectoryReader(DirectoryReader in, Holder cancellable) throws IOException { + private final Holder cancellableHolder; + + private CancellableDirectoryReader(DirectoryReader in, Holder cancellableHolder) throws IOException { super(in, new SubReaderWrapper() { @Override public LeafReader wrap(LeafReader reader) { - return new CancellableLeafReader(reader, cancellable); + return new CancellableLeafReader(reader, cancellableHolder); } }); + this.cancellableHolder = cancellableHolder; + } + + private void setCancellable(QueryCancellable cancellable) { + this.cancellableHolder.set(cancellable); } @Override @@ -439,7 +409,7 @@ public PointValues getPointValues(String field) throws IOException { if (pointValues == null) { return null; } - return (cancellable.get().isEnabled()) ? new ExitablePointValues(pointValues, cancellable.get()) : pointValues; + return (cancellable.get() != null) ? new ExitablePointValues(pointValues, cancellable.get()) : pointValues; } @Override @@ -448,7 +418,7 @@ public Terms terms(String field) throws IOException { if (terms == null) { return null; } - return (cancellable.get().isEnabled() && terms instanceof CompletionTerms == false) ? + return (cancellable.get() != null && terms instanceof CompletionTerms == false) ? new ExitableTerms(terms, cancellable.get()) : terms; } diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 2d1c6bc173b58..83cd38de17f1e 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -81,6 +81,7 @@ import java.util.List; import java.util.concurrent.ExecutorService; +import static org.elasticsearch.search.internal.ContextIndexSearcher.QueryCancellable; import static org.elasticsearch.search.query.QueryCollectorContext.createEarlyTerminationCollectorContext; import static org.elasticsearch.search.query.QueryCollectorContext.createFilteredCollectorContext; import static org.elasticsearch.search.query.QueryCollectorContext.createMinScoreCollectorContext; @@ -272,7 +273,8 @@ static boolean executeInternal(SearchContext searchContext) throws QueryPhaseExe cancellationRunnable = null; } - searcher.setCancellable(new ContextIndexSearcher.CancellableImpl(timeoutRunnable, cancellationRunnable)); + QueryCancellableImpl cancellable = new QueryCancellableImpl(timeoutRunnable, cancellationRunnable); + searcher.setCancellable(cancellable); boolean shouldRescore; // if we are optimizing sort and there are no other collectors @@ -299,7 +301,7 @@ static boolean executeInternal(SearchContext searchContext) throws QueryPhaseExe } // Search phase has finished, no longer need to check for timeout // otherwise aggregation phase might get cancelled. - searcher.unsetCheckTimeout(); + cancellable.unsetCheckTimeout(); return shouldRescore; } catch (Exception e) { throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Failed to execute main query", e); @@ -628,4 +630,29 @@ public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue } private static class TimeExceededException extends RuntimeException {} + + private static class QueryCancellableImpl implements QueryCancellable { + + private Runnable checkCancelled; + private Runnable checkTimeout; + + private QueryCancellableImpl(Runnable checkTimeout, Runnable checkCancelled) { + this.checkCancelled = checkCancelled; + this.checkTimeout = checkTimeout; + } + + @Override + public void checkCancelled() { + if (checkTimeout != null) { + checkTimeout.run(); + } + if (checkCancelled != null) { + checkCancelled.run(); + } + } + + private void unsetCheckTimeout() { + this.checkTimeout = null; + } + } } diff --git a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java index 284560259acea..915b52b16feb2 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java @@ -86,11 +86,6 @@ public void testCancellableCollector() throws IOException { ContextIndexSearcher searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); searcher.setCancellable(new ContextIndexSearcher.QueryCancellable() { - @Override - public boolean isEnabled() { - return true; - } - @Override public void checkCancelled() { if (cancelled.get()) { @@ -102,9 +97,6 @@ public void checkCancelled() { public void checkDirReaderCancelled() { } - @Override - public void unsetCheckTimeout() { - } }); searcher.search(new MatchAllDocsQuery(), collector); assertThat(collector.getTotalHits(), equalTo(reader.numDocs())); @@ -119,11 +111,6 @@ public void testCancellableDirReader() throws IOException { ContextIndexSearcher searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); searcher.setCancellable(new ContextIndexSearcher.QueryCancellable() { - @Override - public boolean isEnabled() { - return true; - } - @Override public void checkCancelled() { } @@ -134,10 +121,6 @@ public void checkDirReaderCancelled() { throw new TaskCancelledException("cancelled"); } } - - @Override - public void unsetCheckTimeout() { - } }); searcher.search(new PrefixQuery(new Term(FIELD_NAME, "a")), collector); assertThat(collector.getTotalHits(), equalTo(reader.numDocs())); diff --git a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java index 22be3839d5f8a..83da1a98b6f3d 100644 --- a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java @@ -235,7 +235,6 @@ public void onRemoval(ShardId shardId, Accountable accountable) { ContextIndexSearcher searcher = new ContextIndexSearcher(filteredReader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); - searcher.setCancellable(new ContextIndexSearcher.CancellableImpl(null, null)); // Searching a non-existing term will trigger a null scorer assertEquals(0, searcher.count(new TermQuery(new Term("non_existing_field", "non_existing_value")))); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 601a49019c686..32c8a949841a4 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -245,7 +245,7 @@ public boolean shouldCache(Query query) { } }; ContextIndexSearcher contextIndexSearcher = new ContextIndexSearcher(indexSearcher.getIndexReader(), - indexSearcher.getSimilarity(), queryCache, queryCachingPolicy, null); + indexSearcher.getSimilarity(), queryCache, queryCachingPolicy, false); SearchContext searchContext = mock(SearchContext.class); when(searchContext.numberOfShards()).thenReturn(1); From 9695114f3028f98ae59dc5ff7481aab3251bce9f Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Sun, 1 Mar 2020 13:09:29 +0100 Subject: [PATCH 19/26] unit test wrapping and exit --- .../search/internal/ContextIndexSearcher.java | 43 +++++++--------- .../search/SearchCancellationTests.java | 50 +++++++------------ .../internal/ContextIndexSearcherTests.java | 20 ++++++++ 3 files changed, 58 insertions(+), 55 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index d3bdabeaefa38..e7317af89205c 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -352,17 +352,12 @@ public DirectoryReader getDirectoryReader() { public interface QueryCancellable { void checkCancelled(); - default void checkDirReaderCancelled() { - checkCancelled(); - } } - - /** * Wraps an {@link IndexReader} with a {@link QueryCancellable}. */ - private static class CancellableDirectoryReader extends FilterDirectoryReader { + static class CancellableDirectoryReader extends FilterDirectoryReader { private final Holder cancellableHolder; @@ -394,7 +389,7 @@ public CacheHelper getReaderCacheHelper() { /** * Wraps a {@link FilterLeafReader} with a {@link QueryCancellable}. */ - private static class CancellableLeafReader extends FilterLeafReader { + static class CancellableLeafReader extends FilterLeafReader { private final Holder cancellable; @@ -458,7 +453,7 @@ private T get() { /** * Wrapper class for {@link FilterTerms} that check for query cancellation or timeout. */ - private static class ExitableTerms extends FilterTerms { + static class ExitableTerms extends FilterTerms { private final QueryCancellable cancellable; @@ -489,13 +484,13 @@ private static class ExitableTermsEnum extends FilterTermsEnum { private ExitableTermsEnum(TermsEnum termsEnum, QueryCancellable cancellable) { super(termsEnum); this.cancellable = cancellable; - this.cancellable.checkDirReaderCancelled(); + this.cancellable.checkCancelled(); } @Override public BytesRef next() throws IOException { // Before every iteration, check if the iteration should exit - this.cancellable.checkDirReaderCancelled(); + this.cancellable.checkCancelled(); return in.next(); } } @@ -503,7 +498,7 @@ public BytesRef next() throws IOException { /** * Wrapper class for {@link PointValues} that checks for query cancellation or timeout. */ - private static class ExitablePointValues extends PointValues { + static class ExitablePointValues extends PointValues { private final PointValues in; private final QueryCancellable cancellable; @@ -511,60 +506,60 @@ private static class ExitablePointValues extends PointValues { private ExitablePointValues(PointValues in, QueryCancellable cancellable) { this.in = in; this.cancellable = cancellable; - this.cancellable.checkDirReaderCancelled(); + this.cancellable.checkCancelled(); } @Override public void intersect(IntersectVisitor visitor) throws IOException { - cancellable.checkDirReaderCancelled(); + cancellable.checkCancelled(); in.intersect(new ExitableIntersectVisitor(visitor, cancellable)); } @Override public long estimatePointCount(IntersectVisitor visitor) { - cancellable.checkDirReaderCancelled(); + cancellable.checkCancelled(); return in.estimatePointCount(visitor); } @Override public byte[] getMinPackedValue() throws IOException { - cancellable.checkDirReaderCancelled(); + cancellable.checkCancelled(); return in.getMinPackedValue(); } @Override public byte[] getMaxPackedValue() throws IOException { - cancellable.checkDirReaderCancelled(); + cancellable.checkCancelled(); return in.getMaxPackedValue(); } @Override public int getNumDimensions() throws IOException { - cancellable.checkDirReaderCancelled(); + cancellable.checkCancelled(); return in.getNumDimensions(); } @Override public int getNumIndexDimensions() throws IOException { - cancellable.checkDirReaderCancelled(); + cancellable.checkCancelled(); return in.getNumIndexDimensions(); } @Override public int getBytesPerDimension() throws IOException { - cancellable.checkDirReaderCancelled(); + cancellable.checkCancelled(); return in.getBytesPerDimension(); } @Override public long size() { - cancellable.checkDirReaderCancelled(); + cancellable.checkCancelled(); return in.size(); } @Override public int getDocCount() { - cancellable.checkDirReaderCancelled(); + cancellable.checkCancelled(); return in.getDocCount(); } } @@ -584,7 +579,7 @@ private ExitableIntersectVisitor(PointValues.IntersectVisitor in, QueryCancellab private void checkAndThrowWithSampling() { if ((calls++ & MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK) == 0) { - cancellable.checkDirReaderCancelled(); + cancellable.checkCancelled(); } } @@ -602,13 +597,13 @@ public void visit(int docID, byte[] packedValue) throws IOException { @Override public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { - cancellable.checkDirReaderCancelled(); + cancellable.checkCancelled(); return in.compare(minPackedValue, maxPackedValue); } @Override public void grow(int count) { - cancellable.checkDirReaderCancelled(); + cancellable.checkCancelled(); in.grow(count); } } diff --git a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java index 915b52b16feb2..3f65da227aef4 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java @@ -20,14 +20,13 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.TotalHitCountCollector; import org.apache.lucene.store.Directory; import org.elasticsearch.core.internal.io.IOUtils; @@ -45,7 +44,8 @@ public class SearchCancellationTests extends ESTestCase { - private static final String FIELD_NAME = "foo"; + private static final String STRING_FIELD_NAME = "foo"; + private static final String POINT_FIELD_NAME = "point"; private static Directory dir; private static IndexReader reader; @@ -68,7 +68,8 @@ public static void setup() throws IOException { private static void indexRandomDocuments(RandomIndexWriter w, int numDocs, int repeatChar) throws IOException { for (int i = 1; i <= numDocs; ++i) { Document doc = new Document(); - doc.add(new StringField(FIELD_NAME, "a".repeat(i + repeatChar), Field.Store.NO)); + doc.add(new StringField(STRING_FIELD_NAME, "a".repeat(i + repeatChar), Field.Store.NO)); + doc.add(new IntPoint(POINT_FIELD_NAME, i, i + 1)); w.addDocument(doc); } } @@ -85,18 +86,10 @@ public void testCancellableCollector() throws IOException { AtomicBoolean cancelled = new AtomicBoolean(); ContextIndexSearcher searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); - searcher.setCancellable(new ContextIndexSearcher.QueryCancellable() { - @Override - public void checkCancelled() { - if (cancelled.get()) { - throw new TaskCancelledException("cancelled"); - } + searcher.setCancellable(() -> { + if (cancelled.get()) { + throw new TaskCancelledException("cancelled"); } - - @Override - public void checkDirReaderCancelled() { - } - }); searcher.search(new MatchAllDocsQuery(), collector); assertThat(collector.getTotalHits(), equalTo(reader.numDocs())); @@ -105,27 +98,22 @@ public void checkDirReaderCancelled() { () -> searcher.search(new MatchAllDocsQuery(), collector)); } - public void testCancellableDirReader() throws IOException { - TotalHitCountCollector collector = new TotalHitCountCollector(); + public void testCancellableDirectoryReader() throws IOException { AtomicBoolean cancelled = new AtomicBoolean(); ContextIndexSearcher searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); - searcher.setCancellable(new ContextIndexSearcher.QueryCancellable() { - @Override - public void checkCancelled() { - } - - @Override - public void checkDirReaderCancelled() { - if (cancelled.get()) { - throw new TaskCancelledException("cancelled"); - } + searcher.setCancellable(() -> { + if (cancelled.get()) { + throw new TaskCancelledException("cancelled"); } }); - searcher.search(new PrefixQuery(new Term(FIELD_NAME, "a")), collector); - assertThat(collector.getTotalHits(), equalTo(reader.numDocs())); + searcher.getIndexReader().leaves().get(0).reader().terms(STRING_FIELD_NAME).iterator(); + searcher.getIndexReader().leaves().get(0).reader().getPointValues(POINT_FIELD_NAME).getDocCount(); + cancelled.set(true); - expectThrows(TaskCancelledException.class, () -> - searcher.search(new PrefixQuery(new Term(FIELD_NAME, "a")), collector)); + expectThrows(TaskCancelledException.class, + () -> searcher.getIndexReader().leaves().get(0).reader().terms(STRING_FIELD_NAME).iterator()); + expectThrows(TaskCancelledException.class, + () -> searcher.getIndexReader().leaves().get(0).reader().getPointValues(POINT_FIELD_NAME).getDocCount()); } } diff --git a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java index 83da1a98b6f3d..5f181b2832ccb 100644 --- a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java @@ -22,6 +22,7 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FilterDirectoryReader; @@ -66,6 +67,9 @@ import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.aggregations.LeafBucketCollector; +import org.elasticsearch.search.internal.ContextIndexSearcher.CancellableDirectoryReader; +import org.elasticsearch.search.internal.ContextIndexSearcher.ExitablePointValues; +import org.elasticsearch.search.internal.ContextIndexSearcher.ExitableTerms; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; @@ -75,6 +79,7 @@ import java.util.IdentityHashMap; import java.util.Set; +import static org.elasticsearch.search.internal.ContextIndexSearcher.CancellableLeafReader; import static org.elasticsearch.search.internal.ContextIndexSearcher.intersectScorerAndBitSet; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -191,6 +196,8 @@ public void doTestContextIndexSearcher(boolean sparse, boolean deletions) throws doc.add(fooField); StringField deleteField = new StringField("delete", "no", Field.Store.NO); doc.add(deleteField); + IntPoint pointField = new IntPoint("point", 1, 2); + doc.add(pointField); w.addDocument(doc); if (deletions) { // add a document that matches foo:bar but will be deleted @@ -236,6 +243,19 @@ public void onRemoval(ShardId shardId, Accountable accountable) { ContextIndexSearcher searcher = new ContextIndexSearcher(filteredReader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); + // Assert wrapping + assertEquals(CancellableDirectoryReader.class, searcher.getIndexReader().getClass()); + for (LeafReaderContext lrc : searcher.getIndexReader().leaves()) { + assertEquals(CancellableLeafReader.class, lrc.reader().getClass()); + assertNotEquals(ExitableTerms.class, lrc.reader().terms("foo").getClass()); + assertNotEquals(ExitablePointValues.class, lrc.reader().getPointValues("point").getClass()); + } + searcher.setCancellable(() -> {}); + for (LeafReaderContext lrc : searcher.getIndexReader().leaves()) { + assertEquals(ExitableTerms.class, lrc.reader().terms("foo").getClass()); + assertEquals(ExitablePointValues.class, lrc.reader().getPointValues("point").getClass()); + } + // Searching a non-existing term will trigger a null scorer assertEquals(0, searcher.count(new TermQuery(new Term("non_existing_field", "non_existing_value")))); From d562cf1d4a23d023685095dfa18a1448574dd475 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Tue, 3 Mar 2020 13:32:42 +0100 Subject: [PATCH 20/26] address comments --- .../search/internal/ContextIndexSearcher.java | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index e7317af89205c..749f981f90511 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -74,6 +74,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.function.Supplier; import static org.apache.lucene.index.FilterLeafReader.FilterTerms; import static org.apache.lucene.index.FilterLeafReader.FilterTermsEnum; @@ -233,7 +234,8 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto Scorer scorer = weight.scorer(ctx); if (scorer != null) { try { - intersectScorerAndBitSet(scorer, liveDocsBitSet, leafCollector, () -> checkCancelled()); + intersectScorerAndBitSet(scorer, liveDocsBitSet, leafCollector, + this.cancellable == null ? () -> {} : this::checkCancelled); } catch (CollectionTerminatedException e) { // collection was terminated prematurely // continue with the following leaf @@ -269,7 +271,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { BulkScorer in = weight.bulkScorer(context); if (in != null) { - return new CancellableBulkScorer(in, () -> checkCancelled()); + return new CancellableBulkScorer(in, cancellable::checkCancelled); } else { return null; } @@ -391,9 +393,9 @@ public CacheHelper getReaderCacheHelper() { */ static class CancellableLeafReader extends FilterLeafReader { - private final Holder cancellable; + private final Supplier cancellable; - private CancellableLeafReader(LeafReader leafReader, Holder cancellable) { + private CancellableLeafReader(LeafReader leafReader, Supplier cancellable) { super(leafReader); this.cancellable = cancellable; } @@ -413,6 +415,9 @@ public Terms terms(String field) throws IOException { if (terms == null) { return null; } + // If we have a suggest CompletionQuery then the CompletionWeight#bulkScorer() will check that + // the terms are instanceof CompletionTerms (not generic FilterTerms) and will throw an exception + // if that's not the case. return (cancellable.get() != null && terms instanceof CompletionTerms == false) ? new ExitableTerms(terms, cancellable.get()) : terms; } @@ -433,7 +438,7 @@ public CacheHelper getReaderCacheHelper() { * {@link QueryCancellable} passed trough the hierarchy to the {@link Terms} and {@link PointValues} * during construction can be set later with {@link ContextIndexSearcher#setCancellable} */ - private static class Holder { + private static class Holder implements Supplier { private T in; @@ -445,7 +450,8 @@ private void set(T in) { this.in = in; } - private T get() { + @Override + public T get() { return in; } } @@ -479,6 +485,9 @@ public TermsEnum iterator() throws IOException { */ private static class ExitableTermsEnum extends FilterTermsEnum { + private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = (1 << 4) - 1; // 15 + + private int calls; private final QueryCancellable cancellable; private ExitableTermsEnum(TermsEnum termsEnum, QueryCancellable cancellable) { @@ -489,10 +498,15 @@ private ExitableTermsEnum(TermsEnum termsEnum, QueryCancellable cancellable) { @Override public BytesRef next() throws IOException { - // Before every iteration, check if the iteration should exit - this.cancellable.checkCancelled(); + checkAndThrowWithSampling(); return in.next(); } + + private void checkAndThrowWithSampling() { + if ((calls++ & MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK) == 0) { + cancellable.checkCancelled(); + } + } } /** From 19bdbdf07825ecdb51d3a8e90e30a84ae5f224a4 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Tue, 3 Mar 2020 20:58:40 +0100 Subject: [PATCH 21/26] improve tests --- .../search/internal/ContextIndexSearcher.java | 14 +++--- .../search/SearchCancellationTests.java | 48 +++++++++++++++---- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 749f981f90511..c5543b36a6145 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -235,7 +235,7 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto if (scorer != null) { try { intersectScorerAndBitSet(scorer, liveDocsBitSet, leafCollector, - this.cancellable == null ? () -> {} : this::checkCancelled); + this.cancellable == null ? () -> {} : cancellable::checkCancelled); } catch (CollectionTerminatedException e) { // collection was terminated prematurely // continue with the following leaf @@ -496,17 +496,17 @@ private ExitableTermsEnum(TermsEnum termsEnum, QueryCancellable cancellable) { this.cancellable.checkCancelled(); } - @Override - public BytesRef next() throws IOException { - checkAndThrowWithSampling(); - return in.next(); - } - private void checkAndThrowWithSampling() { if ((calls++ & MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK) == 0) { cancellable.checkCancelled(); } } + + @Override + public BytesRef next() throws IOException { + checkAndThrowWithSampling(); + return in.next(); + } } /** diff --git a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java index 3f65da227aef4..106f1e12a77da 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java @@ -24,11 +24,17 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.index.PointValues; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.TotalHitCountCollector; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.TestUtil; +import org.apache.lucene.util.automaton.CompiledAutomaton; +import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.tasks.TaskCancelledException; @@ -39,7 +45,6 @@ import java.io.IOException; import java.util.concurrent.atomic.AtomicBoolean; -import static org.apache.lucene.util.TestUtil.nextInt; import static org.hamcrest.Matchers.equalTo; public class SearchCancellationTests extends ESTestCase { @@ -57,10 +62,10 @@ public static void setup() throws IOException { // we need at least 2 segments - so no merges should be allowed w.w.getConfig().setMergePolicy(NoMergePolicy.INSTANCE); w.setDoRandomForceMerge(false); - int numDocs = nextInt(random(), 2, 20); + int numDocs = TestUtil.nextInt(random(), 2, 20); indexRandomDocuments(w, numDocs, 0); w.flush(); - indexRandomDocuments(w, nextInt(random(), 1, 20), numDocs); + indexRandomDocuments(w, TestUtil.nextInt(random(), 1, 20), numDocs); reader = w.getReader(); w.close(); } @@ -99,7 +104,7 @@ public void testCancellableCollector() throws IOException { } public void testCancellableDirectoryReader() throws IOException { - AtomicBoolean cancelled = new AtomicBoolean(); + AtomicBoolean cancelled = new AtomicBoolean(true); ContextIndexSearcher searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); searcher.setCancellable(() -> { @@ -107,13 +112,40 @@ public void testCancellableDirectoryReader() throws IOException { throw new TaskCancelledException("cancelled"); } }); - searcher.getIndexReader().leaves().get(0).reader().terms(STRING_FIELD_NAME).iterator(); - searcher.getIndexReader().leaves().get(0).reader().getPointValues(POINT_FIELD_NAME).getDocCount(); + CompiledAutomaton automaton = new CompiledAutomaton(new RegExp("a.*").toAutomaton()); - cancelled.set(true); expectThrows(TaskCancelledException.class, () -> searcher.getIndexReader().leaves().get(0).reader().terms(STRING_FIELD_NAME).iterator()); expectThrows(TaskCancelledException.class, - () -> searcher.getIndexReader().leaves().get(0).reader().getPointValues(POINT_FIELD_NAME).getDocCount()); + () -> searcher.getIndexReader().leaves().get(0).reader().terms(STRING_FIELD_NAME).intersect(automaton, null)); + expectThrows(TaskCancelledException.class, + () -> searcher.getIndexReader().leaves().get(0).reader().getPointValues(POINT_FIELD_NAME)); + expectThrows(TaskCancelledException.class, + () -> searcher.getIndexReader().leaves().get(0).reader().getPointValues(POINT_FIELD_NAME)); + + cancelled.set(false); // Avoid exception during construction of the wrapper objects + Terms terms = searcher.getIndexReader().leaves().get(0).reader().terms(STRING_FIELD_NAME); + TermsEnum termsIterator = terms.iterator(); + TermsEnum termsIntersect = terms.intersect(automaton, null); + PointValues pointValues = searcher.getIndexReader().leaves().get(0).reader().getPointValues(POINT_FIELD_NAME); + cancelled.set(true); + expectThrows(TaskCancelledException.class, termsIterator::next); + expectThrows(TaskCancelledException.class, termsIntersect::next); + expectThrows(TaskCancelledException.class, pointValues::getDocCount); + expectThrows(TaskCancelledException.class, pointValues::getNumDimensions); + expectThrows(TaskCancelledException.class, () -> pointValues.intersect(new PointValues.IntersectVisitor() { + @Override + public void visit(int docID) { + } + + @Override + public void visit(int docID, byte[] packedValue) { + } + + @Override + public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + return null; + } + })); } } From b446dfd16c6de533f1ccc2a552407e6df03f3376 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Wed, 4 Mar 2020 12:36:21 +0100 Subject: [PATCH 22/26] extract wrapper classes to another file, use more elegant approach for cancellable --- .../search/internal/ContextIndexSearcher.java | 323 ++---------------- .../internal/ExitableDirectoryReader.java | 272 +++++++++++++++ .../search/query/QueryPhase.java | 94 ++--- .../search/SearchCancellationTests.java | 50 +-- .../internal/ContextIndexSearcherTests.java | 13 +- .../aggregations/AggregatorTestCase.java | 2 +- 6 files changed, 379 insertions(+), 375 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index c5543b36a6145..139c48c543d90 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -20,15 +20,10 @@ package org.elasticsearch.search.internal; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.FilterDirectoryReader; -import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.QueryTimeout; import org.apache.lucene.index.Term; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.CollectionTerminatedException; @@ -50,14 +45,11 @@ import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.Weight; import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.search.suggest.document.CompletionTerms; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CombinedBitSet; import org.apache.lucene.util.SparseFixedBitSet; -import org.apache.lucene.util.automaton.CompiledAutomaton; import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.dfs.AggregatedDfs; @@ -71,13 +63,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; -import java.util.function.Supplier; - -import static org.apache.lucene.index.FilterLeafReader.FilterTerms; -import static org.apache.lucene.index.FilterLeafReader.FilterTermsEnum; /** * Context-aware extension of {@link IndexSearcher}. @@ -91,25 +80,26 @@ public class ContextIndexSearcher extends IndexSearcher { private AggregatedDfs aggregatedDfs; private QueryProfiler profiler; - private QueryCancellable cancellable; + private MutableQueryTimeout cancellable; public ContextIndexSearcher(IndexReader reader, Similarity similarity, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy) throws IOException { - this(reader, similarity, queryCache, queryCachingPolicy, true); + this(reader, similarity, queryCache, queryCachingPolicy, new MutableQueryTimeout()); } - // TODO: Remove the 2nd constructor so that the IndexReader is always wrapped. + // TODO: Make the 2nd constructor private so that the IndexReader is always wrapped. // Some issues must be fixed: // - regarding tests deriving from AggregatorTestCase and more specifically the use of searchAndReduce and // the ShardSearcher sub-searchers. // - tests that use a MultiReader public ContextIndexSearcher(IndexReader reader, Similarity similarity, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, - boolean shouldWrap) throws IOException { - super(shouldWrap ? new CancellableDirectoryReader((DirectoryReader) reader, new Holder<>(null)) : reader); + MutableQueryTimeout cancellable) throws IOException { + super(cancellable != null ? new ExitableDirectoryReader((DirectoryReader) reader, cancellable) : reader); setSimilarity(similarity); setQueryCache(queryCache); setQueryCachingPolicy(queryCachingPolicy); + this.cancellable = cancellable != null ? cancellable : new MutableQueryTimeout(); } public void setProfiler(QueryProfiler profiler) { @@ -117,18 +107,15 @@ public void setProfiler(QueryProfiler profiler) { } /** - * Set a {@link Runnable} that will be run on a regular basis while - * collecting documents and check for query cancellation or timeout + * Add a {@link Runnable} that will be run on a regular basis while + * collecting documents and check for query cancellation or timeout. */ - public void setCancellable(QueryCancellable cancellable) { - this.cancellable = Objects.requireNonNull(cancellable, "queryCancellable should not be null"); - ((CancellableDirectoryReader) getIndexReader()).setCancellable(cancellable); + public Runnable addQueryCancellation(Runnable action) { + return this.cancellable.add(action); } - private void checkCancelled() { - if (cancellable != null) { - cancellable.checkCancelled(); - } + public void removeQueryTimeout(Runnable action) { + this.cancellable.remove(action); } public void setAggregatedDfs(AggregatedDfs aggregatedDfs) { @@ -207,7 +194,7 @@ protected void search(List leaves, Weight weight, Collector c * the provided ctx. */ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collector) throws IOException { - checkCancelled(); + cancellable.shouldExit(); weight = wrapWeight(weight); final LeafCollector leafCollector; try { @@ -235,7 +222,7 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto if (scorer != null) { try { intersectScorerAndBitSet(scorer, liveDocsBitSet, leafCollector, - this.cancellable == null ? () -> {} : cancellable::checkCancelled); + this.cancellable.isTimeoutEnabled() ? cancellable::shouldExit : () -> {}); } catch (CollectionTerminatedException e) { // collection was terminated prematurely // continue with the following leaf @@ -245,7 +232,7 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto } private Weight wrapWeight(Weight weight) { - if (cancellable != null) { + if (cancellable.isTimeoutEnabled()) { return new Weight(weight.getQuery()) { @Override public void extractTerms(Set terms) { @@ -271,7 +258,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { BulkScorer in = weight.bulkScorer(context); if (in != null) { - return new CancellableBulkScorer(in, cancellable::checkCancelled); + return new CancellableBulkScorer(in, cancellable::shouldExit); } else { return null; } @@ -348,277 +335,33 @@ public DirectoryReader getDirectoryReader() { return (DirectoryReader) reader; } - /** - * iFace which implements the query timeout / cancellation logic - */ - public interface QueryCancellable { - - void checkCancelled(); - } - - /** - * Wraps an {@link IndexReader} with a {@link QueryCancellable}. - */ - static class CancellableDirectoryReader extends FilterDirectoryReader { - - private final Holder cancellableHolder; - - private CancellableDirectoryReader(DirectoryReader in, Holder cancellableHolder) throws IOException { - super(in, new SubReaderWrapper() { - @Override - public LeafReader wrap(LeafReader reader) { - return new CancellableLeafReader(reader, cancellableHolder); - } - }); - this.cancellableHolder = cancellableHolder; - } - - private void setCancellable(QueryCancellable cancellable) { - this.cancellableHolder.set(cancellable); - } - - @Override - protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) { - throw new UnsupportedOperationException("doWrapDirectoryReader() should never be invoked"); - } - - @Override - public CacheHelper getReaderCacheHelper() { - return in.getReaderCacheHelper(); - } - } - - /** - * Wraps a {@link FilterLeafReader} with a {@link QueryCancellable}. - */ - static class CancellableLeafReader extends FilterLeafReader { - - private final Supplier cancellable; - - private CancellableLeafReader(LeafReader leafReader, Supplier cancellable) { - super(leafReader); - this.cancellable = cancellable; - } - - @Override - public PointValues getPointValues(String field) throws IOException { - final PointValues pointValues = in.getPointValues(field); - if (pointValues == null) { - return null; - } - return (cancellable.get() != null) ? new ExitablePointValues(pointValues, cancellable.get()) : pointValues; - } - - @Override - public Terms terms(String field) throws IOException { - Terms terms = in.terms(field); - if (terms == null) { - return null; - } - // If we have a suggest CompletionQuery then the CompletionWeight#bulkScorer() will check that - // the terms are instanceof CompletionTerms (not generic FilterTerms) and will throw an exception - // if that's not the case. - return (cancellable.get() != null && terms instanceof CompletionTerms == false) ? - new ExitableTerms(terms, cancellable.get()) : terms; - } - - @Override - public CacheHelper getCoreCacheHelper() { - return in.getCoreCacheHelper(); - } - - @Override - public CacheHelper getReaderCacheHelper() { - return in.getReaderCacheHelper(); - } - } - - /** - * Helper class to be used as an immutable reference so that the underlying - * {@link QueryCancellable} passed trough the hierarchy to the {@link Terms} and {@link PointValues} - * during construction can be set later with {@link ContextIndexSearcher#setCancellable} - */ - private static class Holder implements Supplier { - - private T in; - - private Holder(T in) { - this.in = in; - } - - private void set(T in) { - this.in = in; - } - - @Override - public T get() { - return in; - } - } - - /** - * Wrapper class for {@link FilterTerms} that check for query cancellation or timeout. - */ - static class ExitableTerms extends FilterTerms { - - private final QueryCancellable cancellable; - - private ExitableTerms(Terms terms, QueryCancellable cancellable) { - super(terms); - this.cancellable = cancellable; - } - - @Override - public TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm) throws IOException { - return new ExitableTermsEnum(in.intersect(compiled, startTerm), cancellable); - } - - @Override - public TermsEnum iterator() throws IOException { - return new ExitableTermsEnum(in.iterator(), cancellable); - } - } - - /** - * Wrapper class for {@link FilterTermsEnum} that is used by {@link ExitableTerms} for - * implementing an exitable enumeration of terms. - */ - private static class ExitableTermsEnum extends FilterTermsEnum { - - private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = (1 << 4) - 1; // 15 - - private int calls; - private final QueryCancellable cancellable; + private static class MutableQueryTimeout implements QueryTimeout { - private ExitableTermsEnum(TermsEnum termsEnum, QueryCancellable cancellable) { - super(termsEnum); - this.cancellable = cancellable; - this.cancellable.checkCancelled(); - } + private final Set runnables = new HashSet<>(); - private void checkAndThrowWithSampling() { - if ((calls++ & MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK) == 0) { - cancellable.checkCancelled(); + private Runnable add(Runnable action) { + Objects.requireNonNull(action, "cancellation runnable should not be null"); + if (runnables.add(action) == false) { + throw new IllegalArgumentException("Cancellation runnable already added"); } + return action; } - @Override - public BytesRef next() throws IOException { - checkAndThrowWithSampling(); - return in.next(); - } - } - - /** - * Wrapper class for {@link PointValues} that checks for query cancellation or timeout. - */ - static class ExitablePointValues extends PointValues { - - private final PointValues in; - private final QueryCancellable cancellable; - - private ExitablePointValues(PointValues in, QueryCancellable cancellable) { - this.in = in; - this.cancellable = cancellable; - this.cancellable.checkCancelled(); - } - - @Override - public void intersect(IntersectVisitor visitor) throws IOException { - cancellable.checkCancelled(); - in.intersect(new ExitableIntersectVisitor(visitor, cancellable)); - } - - @Override - public long estimatePointCount(IntersectVisitor visitor) { - cancellable.checkCancelled(); - return in.estimatePointCount(visitor); - } - - @Override - public byte[] getMinPackedValue() throws IOException { - cancellable.checkCancelled(); - return in.getMinPackedValue(); - } - - @Override - public byte[] getMaxPackedValue() throws IOException { - cancellable.checkCancelled(); - return in.getMaxPackedValue(); - } - - @Override - public int getNumDimensions() throws IOException { - cancellable.checkCancelled(); - return in.getNumDimensions(); - } - - @Override - public int getNumIndexDimensions() throws IOException { - cancellable.checkCancelled(); - return in.getNumIndexDimensions(); - } - - @Override - public int getBytesPerDimension() throws IOException { - cancellable.checkCancelled(); - return in.getBytesPerDimension(); - } - - @Override - public long size() { - cancellable.checkCancelled(); - return in.size(); + private void remove(Runnable action) { + runnables.remove(action); } @Override - public int getDocCount() { - cancellable.checkCancelled(); - return in.getDocCount(); - } - } - - private static class ExitableIntersectVisitor implements PointValues.IntersectVisitor { - - private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = (1 << 4) - 1; // 15 - - private final PointValues.IntersectVisitor in; - private final QueryCancellable cancellable; - private int calls; - - private ExitableIntersectVisitor(PointValues.IntersectVisitor in, QueryCancellable cancellable) { - this.in = in; - this.cancellable = cancellable; - } - - private void checkAndThrowWithSampling() { - if ((calls++ & MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK) == 0) { - cancellable.checkCancelled(); + public boolean shouldExit() { + for (Runnable timeout : runnables) { + timeout.run(); } + return false; } @Override - public void visit(int docID) throws IOException { - checkAndThrowWithSampling(); - in.visit(docID); - } - - @Override - public void visit(int docID, byte[] packedValue) throws IOException { - checkAndThrowWithSampling(); - in.visit(docID, packedValue); - } - - @Override - public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { - cancellable.checkCancelled(); - return in.compare(minPackedValue, maxPackedValue); - } - - @Override - public void grow(int count) { - cancellable.checkCancelled(); - in.grow(count); + public boolean isTimeoutEnabled() { + return runnables.isEmpty() == false; } } } diff --git a/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java b/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java new file mode 100644 index 0000000000000..313d51befcdf5 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java @@ -0,0 +1,272 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.internal; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FilterDirectoryReader; +import org.apache.lucene.index.FilterLeafReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.QueryTimeout; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.suggest.document.CompletionTerms; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.CompiledAutomaton; + +import java.io.IOException; + +/** + * Wraps an {@link IndexReader} with a {@link QueryTimeout} + * which checks for cancelled or timed-out query. + */ +class ExitableDirectoryReader extends FilterDirectoryReader { + + ExitableDirectoryReader(DirectoryReader in, QueryTimeout queryTimeout) throws IOException { + super(in, new SubReaderWrapper() { + @Override + public LeafReader wrap(LeafReader reader) { + return new ExitableLeafReader(reader, queryTimeout); + } + }); + } + + @Override + protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) { + throw new UnsupportedOperationException("doWrapDirectoryReader() should never be invoked"); + } + + @Override + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); + } + + /** + * Wraps a {@link FilterLeafReader} with a {@link QueryTimeout}. + */ + static class ExitableLeafReader extends FilterLeafReader { + + private final QueryTimeout queryTimeout; + + private ExitableLeafReader(LeafReader leafReader, QueryTimeout queryTimeout) { + super(leafReader); + this.queryTimeout = queryTimeout; + } + + @Override + public PointValues getPointValues(String field) throws IOException { + final PointValues pointValues = in.getPointValues(field); + if (pointValues == null) { + return null; + } + return queryTimeout.isTimeoutEnabled() ? new ExitablePointValues(pointValues, queryTimeout) : pointValues; + } + + @Override + public Terms terms(String field) throws IOException { + Terms terms = in.terms(field); + if (terms == null) { + return null; + } + // If we have a suggest CompletionQuery then the CompletionWeight#bulkScorer() will check that + // the terms are instanceof CompletionTerms (not generic FilterTerms) and will throw an exception + // if that's not the case. + return (queryTimeout.isTimeoutEnabled() && terms instanceof CompletionTerms == false) ? + new ExitableTerms(terms, queryTimeout) : terms; + } + + @Override + public CacheHelper getCoreCacheHelper() { + return in.getCoreCacheHelper(); + } + + @Override + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); + } + } + + /** + * Wrapper class for {@link FilterLeafReader.FilterTerms} that check for query cancellation or timeout. + */ + static class ExitableTerms extends FilterLeafReader.FilterTerms { + + private final QueryTimeout queryTimeout; + + private ExitableTerms(Terms terms, QueryTimeout queryTimeout) { + super(terms); + this.queryTimeout = queryTimeout; + } + + @Override + public TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm) throws IOException { + return new ExitableTermsEnum(in.intersect(compiled, startTerm), queryTimeout); + } + + @Override + public TermsEnum iterator() throws IOException { + return new ExitableTermsEnum(in.iterator(), queryTimeout); + } + } + + /** + * Wrapper class for {@link FilterLeafReader.FilterTermsEnum} that is used by {@link ExitableTerms} for + * implementing an exitable enumeration of terms. + */ + private static class ExitableTermsEnum extends FilterLeafReader.FilterTermsEnum { + + private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = (1 << 4) - 1; // 15 + + private int calls; + private final QueryTimeout queryTimeout; + + private ExitableTermsEnum(TermsEnum termsEnum, QueryTimeout queryTimeout) { + super(termsEnum); + this.queryTimeout = queryTimeout; + this.queryTimeout.shouldExit(); + } + + private void checkAndThrowWithSampling() { + if ((calls++ & MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK) == 0) { + queryTimeout.shouldExit(); + } + } + + @Override + public BytesRef next() throws IOException { + checkAndThrowWithSampling(); + return in.next(); + } + } + + /** + * Wrapper class for {@link PointValues} that checks for query cancellation or timeout. + */ + static class ExitablePointValues extends PointValues { + + private final PointValues in; + private final QueryTimeout queryTimeout; + + private ExitablePointValues(PointValues in, QueryTimeout queryTimeout) { + this.in = in; + this.queryTimeout = queryTimeout; + this.queryTimeout.shouldExit(); + } + + @Override + public void intersect(IntersectVisitor visitor) throws IOException { + queryTimeout.shouldExit(); + in.intersect(new ExitableIntersectVisitor(visitor, queryTimeout)); + } + + @Override + public long estimatePointCount(IntersectVisitor visitor) { + queryTimeout.shouldExit(); + return in.estimatePointCount(visitor); + } + + @Override + public byte[] getMinPackedValue() throws IOException { + queryTimeout.shouldExit(); + return in.getMinPackedValue(); + } + + @Override + public byte[] getMaxPackedValue() throws IOException { + queryTimeout.shouldExit(); + return in.getMaxPackedValue(); + } + + @Override + public int getNumDimensions() throws IOException { + queryTimeout.shouldExit(); + return in.getNumDimensions(); + } + + @Override + public int getNumIndexDimensions() throws IOException { + queryTimeout.shouldExit(); + return in.getNumIndexDimensions(); + } + + @Override + public int getBytesPerDimension() throws IOException { + queryTimeout.shouldExit(); + return in.getBytesPerDimension(); + } + + @Override + public long size() { + queryTimeout.shouldExit(); + return in.size(); + } + + @Override + public int getDocCount() { + queryTimeout.shouldExit(); + return in.getDocCount(); + } + } + + private static class ExitableIntersectVisitor implements PointValues.IntersectVisitor { + + private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = (1 << 4) - 1; // 15 + + private final PointValues.IntersectVisitor in; + private final QueryTimeout queryTimeout; + private int calls; + + private ExitableIntersectVisitor(PointValues.IntersectVisitor in, QueryTimeout queryTimeout) { + this.in = in; + this.queryTimeout = queryTimeout; + } + + private void checkAndThrowWithSampling() { + if ((calls++ & MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK) == 0) { + queryTimeout.shouldExit(); + } + } + + @Override + public void visit(int docID) throws IOException { + checkAndThrowWithSampling(); + in.visit(docID); + } + + @Override + public void visit(int docID, byte[] packedValue) throws IOException { + checkAndThrowWithSampling(); + in.visit(docID, packedValue); + } + + @Override + public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + queryTimeout.shouldExit(); + return in.compare(minPackedValue, maxPackedValue); + } + + @Override + public void grow(int count) { + queryTimeout.shouldExit(); + in.grow(count); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 83cd38de17f1e..5691af384a92a 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -81,7 +81,6 @@ import java.util.List; import java.util.concurrent.ExecutorService; -import static org.elasticsearch.search.internal.ContextIndexSearcher.QueryCancellable; import static org.elasticsearch.search.query.QueryCollectorContext.createEarlyTerminationCollectorContext; import static org.elasticsearch.search.query.QueryCollectorContext.createFilteredCollectorContext; import static org.elasticsearch.search.query.QueryCollectorContext.createMinScoreCollectorContext; @@ -255,54 +254,58 @@ static boolean executeInternal(SearchContext searchContext) throws QueryPhaseExe final long startTime = searchContext.getRelativeTimeInMillis(); final long timeout = searchContext.timeout().millis(); final long maxTime = startTime + timeout; - timeoutRunnable = () -> { + timeoutRunnable = searcher.addQueryCancellation(() -> { final long time = searchContext.getRelativeTimeInMillis(); if (time > maxTime) { throw new TimeExceededException(); } - }; + }); } else { timeoutRunnable = null; } - final Runnable cancellationRunnable; if (searchContext.lowLevelCancellation()) { SearchShardTask task = searchContext.getTask(); - cancellationRunnable = () -> { if (task.isCancelled()) throw new TaskCancelledException("cancelled"); }; - } else { - cancellationRunnable = null; + searcher.addQueryCancellation(() -> { + if (task.isCancelled()) { + throw new TaskCancelledException("cancelled"); + } + }); } - QueryCancellableImpl cancellable = new QueryCancellableImpl(timeoutRunnable, cancellationRunnable); - searcher.setCancellable(cancellable); + try { + boolean shouldRescore; + // if we are optimizing sort and there are no other collectors + if (sortAndFormatsForRewrittenNumericSort != null && collectors.size() == 0 && searchContext.getProfilers() == null) { + shouldRescore = searchWithCollectorManager(searchContext, searcher, query, leafSorter, timeoutSet); + } else { + shouldRescore = searchWithCollector(searchContext, searcher, query, collectors, hasFilterCollector, timeoutSet); + } - boolean shouldRescore; - // if we are optimizing sort and there are no other collectors - if (sortAndFormatsForRewrittenNumericSort != null && collectors.size() == 0 && searchContext.getProfilers() == null) { - shouldRescore = searchWithCollectorManager(searchContext, searcher, query, leafSorter, timeoutSet); - } else { - shouldRescore = searchWithCollector(searchContext, searcher, query, collectors, hasFilterCollector, timeoutSet); - } + // if we rewrote numeric long or date sort, restore fieldDocs based on the original sort + if (sortAndFormatsForRewrittenNumericSort != null) { + searchContext.sort(sortAndFormatsForRewrittenNumericSort); // restore SortAndFormats + restoreTopFieldDocs(queryResult, sortAndFormatsForRewrittenNumericSort); + } - // if we rewrote numeric long or date sort, restore fieldDocs based on the original sort - if (sortAndFormatsForRewrittenNumericSort != null) { - searchContext.sort(sortAndFormatsForRewrittenNumericSort); // restore SortAndFormats - restoreTopFieldDocs(queryResult, sortAndFormatsForRewrittenNumericSort); - } + ExecutorService executor = searchContext.indexShard().getThreadPool().executor(ThreadPool.Names.SEARCH); + assert executor instanceof EWMATrackingEsThreadPoolExecutor || + (executor instanceof EsThreadPoolExecutor == false /* in case thread pool is mocked out in tests */) : + "SEARCH threadpool should have an executor that exposes EWMA metrics, but is of type " + executor.getClass(); + if (executor instanceof EWMATrackingEsThreadPoolExecutor) { + EWMATrackingEsThreadPoolExecutor rExecutor = (EWMATrackingEsThreadPoolExecutor) executor; + queryResult.nodeQueueSize(rExecutor.getCurrentQueueSize()); + queryResult.serviceTimeEWMA((long) rExecutor.getTaskExecutionEWMA()); + } - ExecutorService executor = searchContext.indexShard().getThreadPool().executor(ThreadPool.Names.SEARCH); - assert executor instanceof EWMATrackingEsThreadPoolExecutor || - (executor instanceof EsThreadPoolExecutor == false /* in case thread pool is mocked out in tests */) : - "SEARCH threadpool should have an executor that exposes EWMA metrics, but is of type " + executor.getClass(); - if (executor instanceof EWMATrackingEsThreadPoolExecutor) { - EWMATrackingEsThreadPoolExecutor rExecutor = (EWMATrackingEsThreadPoolExecutor) executor; - queryResult.nodeQueueSize(rExecutor.getCurrentQueueSize()); - queryResult.serviceTimeEWMA((long) rExecutor.getTaskExecutionEWMA()); + return shouldRescore; + } finally { + // Search phase has finished, no longer need to check for timeout + // otherwise aggregation phase might get cancelled. + if (timeoutRunnable != null) { + searcher.removeQueryTimeout(timeoutRunnable); + } } - // Search phase has finished, no longer need to check for timeout - // otherwise aggregation phase might get cancelled. - cancellable.unsetCheckTimeout(); - return shouldRescore; } catch (Exception e) { throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Failed to execute main query", e); } @@ -630,29 +633,4 @@ public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue } private static class TimeExceededException extends RuntimeException {} - - private static class QueryCancellableImpl implements QueryCancellable { - - private Runnable checkCancelled; - private Runnable checkTimeout; - - private QueryCancellableImpl(Runnable checkTimeout, Runnable checkCancelled) { - this.checkCancelled = checkCancelled; - this.checkTimeout = checkTimeout; - } - - @Override - public void checkCancelled() { - if (checkTimeout != null) { - checkTimeout.run(); - } - if (checkCancelled != null) { - checkCancelled.run(); - } - } - - private void unsetCheckTimeout() { - this.checkTimeout = null; - } - } } diff --git a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java index 106f1e12a77da..2ef269890a8bb 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java @@ -62,18 +62,17 @@ public static void setup() throws IOException { // we need at least 2 segments - so no merges should be allowed w.w.getConfig().setMergePolicy(NoMergePolicy.INSTANCE); w.setDoRandomForceMerge(false); - int numDocs = TestUtil.nextInt(random(), 2, 20); - indexRandomDocuments(w, numDocs, 0); + indexRandomDocuments(w, TestUtil.nextInt(random(), 2, 20)); w.flush(); - indexRandomDocuments(w, TestUtil.nextInt(random(), 1, 20), numDocs); + indexRandomDocuments(w, TestUtil.nextInt(random(), 1, 20)); reader = w.getReader(); w.close(); } - private static void indexRandomDocuments(RandomIndexWriter w, int numDocs, int repeatChar) throws IOException { + private static void indexRandomDocuments(RandomIndexWriter w, int numDocs) throws IOException { for (int i = 1; i <= numDocs; ++i) { Document doc = new Document(); - doc.add(new StringField(STRING_FIELD_NAME, "a".repeat(i + repeatChar), Field.Store.NO)); + doc.add(new StringField(STRING_FIELD_NAME, "a".repeat(i), Field.Store.NO)); doc.add(new IntPoint(POINT_FIELD_NAME, i, i + 1)); w.addDocument(doc); } @@ -86,32 +85,45 @@ public static void cleanup() throws IOException { reader = null; } + public void testAddingCancellationActions() throws IOException { + ContextIndexSearcher searcher = new ContextIndexSearcher(reader, + IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); + NullPointerException npe = expectThrows(NullPointerException.class, () -> searcher.addQueryCancellation(null)); + assertEquals("cancellation runnable should not be null", npe.getMessage()); + + Runnable r = () -> {}; + searcher.addQueryCancellation(r); + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> searcher.addQueryCancellation(r)); + assertEquals("Cancellation runnable already added", iae.getMessage()); + } + public void testCancellableCollector() throws IOException { - TotalHitCountCollector collector = new TotalHitCountCollector(); - AtomicBoolean cancelled = new AtomicBoolean(); + TotalHitCountCollector collector1 = new TotalHitCountCollector(); + Runnable cancellation = () -> { throw new TaskCancelledException("cancelled"); }; ContextIndexSearcher searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); - searcher.setCancellable(() -> { - if (cancelled.get()) { - throw new TaskCancelledException("cancelled"); - } - }); - searcher.search(new MatchAllDocsQuery(), collector); - assertThat(collector.getTotalHits(), equalTo(reader.numDocs())); - cancelled.set(true); + + searcher.search(new MatchAllDocsQuery(), collector1); + assertThat(collector1.getTotalHits(), equalTo(reader.numDocs())); + + searcher.addQueryCancellation(cancellation); expectThrows(TaskCancelledException.class, - () -> searcher.search(new MatchAllDocsQuery(), collector)); + () -> searcher.search(new MatchAllDocsQuery(), collector1)); + + searcher.removeQueryTimeout(cancellation); + TotalHitCountCollector collector2 = new TotalHitCountCollector(); + searcher.search(new MatchAllDocsQuery(), collector2); + assertThat(collector2.getTotalHits(), equalTo(reader.numDocs())); } public void testCancellableDirectoryReader() throws IOException { AtomicBoolean cancelled = new AtomicBoolean(true); ContextIndexSearcher searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); - searcher.setCancellable(() -> { + searcher.addQueryCancellation(() -> { if (cancelled.get()) { throw new TaskCancelledException("cancelled"); - } - }); + }}); CompiledAutomaton automaton = new CompiledAutomaton(new RegExp("a.*").toAutomaton()); expectThrows(TaskCancelledException.class, diff --git a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java index 5f181b2832ccb..61a75a0716fd2 100644 --- a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java @@ -67,9 +67,6 @@ import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.aggregations.LeafBucketCollector; -import org.elasticsearch.search.internal.ContextIndexSearcher.CancellableDirectoryReader; -import org.elasticsearch.search.internal.ContextIndexSearcher.ExitablePointValues; -import org.elasticsearch.search.internal.ContextIndexSearcher.ExitableTerms; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; @@ -79,8 +76,10 @@ import java.util.IdentityHashMap; import java.util.Set; -import static org.elasticsearch.search.internal.ContextIndexSearcher.CancellableLeafReader; import static org.elasticsearch.search.internal.ContextIndexSearcher.intersectScorerAndBitSet; +import static org.elasticsearch.search.internal.ExitableDirectoryReader.ExitableLeafReader; +import static org.elasticsearch.search.internal.ExitableDirectoryReader.ExitablePointValues; +import static org.elasticsearch.search.internal.ExitableDirectoryReader.ExitableTerms; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -244,13 +243,13 @@ public void onRemoval(ShardId shardId, Accountable accountable) { IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); // Assert wrapping - assertEquals(CancellableDirectoryReader.class, searcher.getIndexReader().getClass()); + assertEquals(ExitableDirectoryReader.class, searcher.getIndexReader().getClass()); for (LeafReaderContext lrc : searcher.getIndexReader().leaves()) { - assertEquals(CancellableLeafReader.class, lrc.reader().getClass()); + assertEquals(ExitableLeafReader.class, lrc.reader().getClass()); assertNotEquals(ExitableTerms.class, lrc.reader().terms("foo").getClass()); assertNotEquals(ExitablePointValues.class, lrc.reader().getPointValues("point").getClass()); } - searcher.setCancellable(() -> {}); + searcher.addQueryCancellation(() -> {}); for (LeafReaderContext lrc : searcher.getIndexReader().leaves()) { assertEquals(ExitableTerms.class, lrc.reader().terms("foo").getClass()); assertEquals(ExitablePointValues.class, lrc.reader().getPointValues("point").getClass()); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 32c8a949841a4..601a49019c686 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -245,7 +245,7 @@ public boolean shouldCache(Query query) { } }; ContextIndexSearcher contextIndexSearcher = new ContextIndexSearcher(indexSearcher.getIndexReader(), - indexSearcher.getSimilarity(), queryCache, queryCachingPolicy, false); + indexSearcher.getSimilarity(), queryCache, queryCachingPolicy, null); SearchContext searchContext = mock(SearchContext.class); when(searchContext.numberOfShards()).thenReturn(1); From a912fa348d07c8cc9b2d90a376f7b1f2aeb52280 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Wed, 4 Mar 2020 16:37:23 +0100 Subject: [PATCH 23/26] rename method, add more tests --- .../search/internal/ContextIndexSearcher.java | 10 ++- .../search/query/QueryPhase.java | 2 +- .../search/SearchCancellationTests.java | 62 ++++++++++++------- 3 files changed, 48 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 139c48c543d90..ecdb43bbcc6a5 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -107,14 +107,18 @@ public void setProfiler(QueryProfiler profiler) { } /** - * Add a {@link Runnable} that will be run on a regular basis while - * collecting documents and check for query cancellation or timeout. + * Add a {@link Runnable} that will be run on a regular basis while fetching document from the + * DirectoryReader but also while collecting them and check for query cancellation or timeout. */ public Runnable addQueryCancellation(Runnable action) { return this.cancellable.add(action); } - public void removeQueryTimeout(Runnable action) { + /** + * Remove a {@link Runnable} that checks for query cancellation or timeout + * which is called while fetching documents from the DirectoryReader but also while collecting them. + */ + public void removeQueryCancellation(Runnable action) { this.cancellable.remove(action); } diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 5691af384a92a..4383f0eb54501 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -303,7 +303,7 @@ static boolean executeInternal(SearchContext searchContext) throws QueryPhaseExe // Search phase has finished, no longer need to check for timeout // otherwise aggregation phase might get cancelled. if (timeoutRunnable != null) { - searcher.removeQueryTimeout(timeoutRunnable); + searcher.removeQueryCancellation(timeoutRunnable); } } } catch (Exception e) { diff --git a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java index 2ef269890a8bb..3aafdcc575230 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java @@ -110,7 +110,7 @@ public void testCancellableCollector() throws IOException { expectThrows(TaskCancelledException.class, () -> searcher.search(new MatchAllDocsQuery(), collector1)); - searcher.removeQueryTimeout(cancellation); + searcher.removeQueryCancellation(cancellation); TotalHitCountCollector collector2 = new TotalHitCountCollector(); searcher.search(new MatchAllDocsQuery(), collector2); assertThat(collector2.getTotalHits(), equalTo(reader.numDocs())); @@ -118,12 +118,13 @@ public void testCancellableCollector() throws IOException { public void testCancellableDirectoryReader() throws IOException { AtomicBoolean cancelled = new AtomicBoolean(true); - ContextIndexSearcher searcher = new ContextIndexSearcher(reader, - IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); - searcher.addQueryCancellation(() -> { + Runnable cancellation = () -> { if (cancelled.get()) { throw new TaskCancelledException("cancelled"); - }}); + }}; + ContextIndexSearcher searcher = new ContextIndexSearcher(reader, + IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); + searcher.addQueryCancellation(cancellation); CompiledAutomaton automaton = new CompiledAutomaton(new RegExp("a.*").toAutomaton()); expectThrows(TaskCancelledException.class, @@ -139,25 +140,42 @@ public void testCancellableDirectoryReader() throws IOException { Terms terms = searcher.getIndexReader().leaves().get(0).reader().terms(STRING_FIELD_NAME); TermsEnum termsIterator = terms.iterator(); TermsEnum termsIntersect = terms.intersect(automaton, null); - PointValues pointValues = searcher.getIndexReader().leaves().get(0).reader().getPointValues(POINT_FIELD_NAME); + PointValues pointValues1 = searcher.getIndexReader().leaves().get(0).reader().getPointValues(POINT_FIELD_NAME); cancelled.set(true); expectThrows(TaskCancelledException.class, termsIterator::next); expectThrows(TaskCancelledException.class, termsIntersect::next); - expectThrows(TaskCancelledException.class, pointValues::getDocCount); - expectThrows(TaskCancelledException.class, pointValues::getNumDimensions); - expectThrows(TaskCancelledException.class, () -> pointValues.intersect(new PointValues.IntersectVisitor() { - @Override - public void visit(int docID) { - } - - @Override - public void visit(int docID, byte[] packedValue) { - } - - @Override - public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { - return null; - } - })); + expectThrows(TaskCancelledException.class, pointValues1::getDocCount); + expectThrows(TaskCancelledException.class, pointValues1::getNumIndexDimensions); + expectThrows(TaskCancelledException.class, () -> pointValues1.intersect(new PointValuesIntersectVisitor())); + + cancelled.set(false); // Avoid exception during construction of the wrapper objects + // Re-initialize objects so that we reset the `calls` counter used to avoid cancellation check + // on every iteration and assure that cancellation would normally happen if we hadn't removed the + // cancellation runnable. + termsIterator = terms.iterator(); + termsIntersect = terms.intersect(automaton, null); + PointValues pointValues2 = searcher.getIndexReader().leaves().get(0).reader().getPointValues(POINT_FIELD_NAME); + cancelled.set(true); + searcher.removeQueryCancellation(cancellation); + termsIterator.next(); + termsIntersect.next(); + pointValues2.getDocCount(); + pointValues2.getNumIndexDimensions(); + pointValues2.intersect(new PointValuesIntersectVisitor()); + } + + private static class PointValuesIntersectVisitor implements PointValues.IntersectVisitor { + @Override + public void visit(int docID) { + } + + @Override + public void visit(int docID, byte[] packedValue) { + } + + @Override + public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + return PointValues.Relation.CELL_CROSSES_QUERY; + } } } From 23c3adc7129b3ca912e5c19a4828dd01c26364bb Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Wed, 4 Mar 2020 18:10:55 +0100 Subject: [PATCH 24/26] replace QueryTimeout with our own iface --- .../search/internal/ContextIndexSearcher.java | 16 ++-- .../internal/ExitableDirectoryReader.java | 89 ++++++++++--------- 2 files changed, 54 insertions(+), 51 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index ecdb43bbcc6a5..839e6844fb71f 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -22,7 +22,6 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.QueryTimeout; import org.apache.lucene.index.Term; import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.CollectionStatistics; @@ -198,7 +197,7 @@ protected void search(List leaves, Weight weight, Collector c * the provided ctx. */ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collector) throws IOException { - cancellable.shouldExit(); + cancellable.checkCancelled(); weight = wrapWeight(weight); final LeafCollector leafCollector; try { @@ -226,7 +225,7 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto if (scorer != null) { try { intersectScorerAndBitSet(scorer, liveDocsBitSet, leafCollector, - this.cancellable.isTimeoutEnabled() ? cancellable::shouldExit : () -> {}); + this.cancellable.isEnabled() ? cancellable::checkCancelled: () -> {}); } catch (CollectionTerminatedException e) { // collection was terminated prematurely // continue with the following leaf @@ -236,7 +235,7 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto } private Weight wrapWeight(Weight weight) { - if (cancellable.isTimeoutEnabled()) { + if (cancellable.isEnabled()) { return new Weight(weight.getQuery()) { @Override public void extractTerms(Set terms) { @@ -262,7 +261,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { BulkScorer in = weight.bulkScorer(context); if (in != null) { - return new CancellableBulkScorer(in, cancellable::shouldExit); + return new CancellableBulkScorer(in, cancellable::checkCancelled); } else { return null; } @@ -339,7 +338,7 @@ public DirectoryReader getDirectoryReader() { return (DirectoryReader) reader; } - private static class MutableQueryTimeout implements QueryTimeout { + private static class MutableQueryTimeout implements ExitableDirectoryReader.QueryCancellation { private final Set runnables = new HashSet<>(); @@ -356,15 +355,14 @@ private void remove(Runnable action) { } @Override - public boolean shouldExit() { + public void checkCancelled() { for (Runnable timeout : runnables) { timeout.run(); } - return false; } @Override - public boolean isTimeoutEnabled() { + public boolean isEnabled() { return runnables.isEmpty() == false; } } diff --git a/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java b/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java index 313d51befcdf5..d79c16bb720fe 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java @@ -25,7 +25,6 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.PointValues; -import org.apache.lucene.index.QueryTimeout; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.suggest.document.CompletionTerms; @@ -35,16 +34,23 @@ import java.io.IOException; /** - * Wraps an {@link IndexReader} with a {@link QueryTimeout} + * Wraps an {@link IndexReader} with a {@link QueryCancellation} * which checks for cancelled or timed-out query. */ class ExitableDirectoryReader extends FilterDirectoryReader { - ExitableDirectoryReader(DirectoryReader in, QueryTimeout queryTimeout) throws IOException { + interface QueryCancellation { + + boolean isEnabled(); + + void checkCancelled(); + } + + ExitableDirectoryReader(DirectoryReader in, QueryCancellation queryCancellation) throws IOException { super(in, new SubReaderWrapper() { @Override public LeafReader wrap(LeafReader reader) { - return new ExitableLeafReader(reader, queryTimeout); + return new ExitableLeafReader(reader, queryCancellation); } }); } @@ -58,17 +64,16 @@ protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) { public CacheHelper getReaderCacheHelper() { return in.getReaderCacheHelper(); } - /** - * Wraps a {@link FilterLeafReader} with a {@link QueryTimeout}. + * Wraps a {@link FilterLeafReader} with a {@link QueryCancellation}. */ static class ExitableLeafReader extends FilterLeafReader { - private final QueryTimeout queryTimeout; + private final QueryCancellation queryCancellation; - private ExitableLeafReader(LeafReader leafReader, QueryTimeout queryTimeout) { + private ExitableLeafReader(LeafReader leafReader, QueryCancellation queryCancellation) { super(leafReader); - this.queryTimeout = queryTimeout; + this.queryCancellation = queryCancellation; } @Override @@ -77,7 +82,7 @@ public PointValues getPointValues(String field) throws IOException { if (pointValues == null) { return null; } - return queryTimeout.isTimeoutEnabled() ? new ExitablePointValues(pointValues, queryTimeout) : pointValues; + return queryCancellation.isEnabled() ? new ExitablePointValues(pointValues, queryCancellation) : pointValues; } @Override @@ -89,8 +94,8 @@ public Terms terms(String field) throws IOException { // If we have a suggest CompletionQuery then the CompletionWeight#bulkScorer() will check that // the terms are instanceof CompletionTerms (not generic FilterTerms) and will throw an exception // if that's not the case. - return (queryTimeout.isTimeoutEnabled() && terms instanceof CompletionTerms == false) ? - new ExitableTerms(terms, queryTimeout) : terms; + return (queryCancellation.isEnabled() && terms instanceof CompletionTerms == false) ? + new ExitableTerms(terms, queryCancellation) : terms; } @Override @@ -109,21 +114,21 @@ public CacheHelper getReaderCacheHelper() { */ static class ExitableTerms extends FilterLeafReader.FilterTerms { - private final QueryTimeout queryTimeout; + private final QueryCancellation queryCancellation; - private ExitableTerms(Terms terms, QueryTimeout queryTimeout) { + private ExitableTerms(Terms terms, QueryCancellation queryCancellation) { super(terms); - this.queryTimeout = queryTimeout; + this.queryCancellation = queryCancellation; } @Override public TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm) throws IOException { - return new ExitableTermsEnum(in.intersect(compiled, startTerm), queryTimeout); + return new ExitableTermsEnum(in.intersect(compiled, startTerm), queryCancellation); } @Override public TermsEnum iterator() throws IOException { - return new ExitableTermsEnum(in.iterator(), queryTimeout); + return new ExitableTermsEnum(in.iterator(), queryCancellation); } } @@ -136,17 +141,17 @@ private static class ExitableTermsEnum extends FilterLeafReader.FilterTermsEnum private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = (1 << 4) - 1; // 15 private int calls; - private final QueryTimeout queryTimeout; + private final QueryCancellation queryCancellation; - private ExitableTermsEnum(TermsEnum termsEnum, QueryTimeout queryTimeout) { + private ExitableTermsEnum(TermsEnum termsEnum, QueryCancellation queryCancellation) { super(termsEnum); - this.queryTimeout = queryTimeout; - this.queryTimeout.shouldExit(); + this.queryCancellation = queryCancellation; + this.queryCancellation.checkCancelled(); } private void checkAndThrowWithSampling() { if ((calls++ & MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK) == 0) { - queryTimeout.shouldExit(); + queryCancellation.checkCancelled(); } } @@ -163,65 +168,65 @@ public BytesRef next() throws IOException { static class ExitablePointValues extends PointValues { private final PointValues in; - private final QueryTimeout queryTimeout; + private final QueryCancellation queryCancellation; - private ExitablePointValues(PointValues in, QueryTimeout queryTimeout) { + private ExitablePointValues(PointValues in, QueryCancellation queryCancellation) { this.in = in; - this.queryTimeout = queryTimeout; - this.queryTimeout.shouldExit(); + this.queryCancellation = queryCancellation; + this.queryCancellation.checkCancelled(); } @Override public void intersect(IntersectVisitor visitor) throws IOException { - queryTimeout.shouldExit(); - in.intersect(new ExitableIntersectVisitor(visitor, queryTimeout)); + queryCancellation.checkCancelled(); + in.intersect(new ExitableIntersectVisitor(visitor, queryCancellation)); } @Override public long estimatePointCount(IntersectVisitor visitor) { - queryTimeout.shouldExit(); + queryCancellation.checkCancelled(); return in.estimatePointCount(visitor); } @Override public byte[] getMinPackedValue() throws IOException { - queryTimeout.shouldExit(); + queryCancellation.checkCancelled(); return in.getMinPackedValue(); } @Override public byte[] getMaxPackedValue() throws IOException { - queryTimeout.shouldExit(); + queryCancellation.checkCancelled(); return in.getMaxPackedValue(); } @Override public int getNumDimensions() throws IOException { - queryTimeout.shouldExit(); + queryCancellation.checkCancelled(); return in.getNumDimensions(); } @Override public int getNumIndexDimensions() throws IOException { - queryTimeout.shouldExit(); + queryCancellation.checkCancelled(); return in.getNumIndexDimensions(); } @Override public int getBytesPerDimension() throws IOException { - queryTimeout.shouldExit(); + queryCancellation.checkCancelled(); return in.getBytesPerDimension(); } @Override public long size() { - queryTimeout.shouldExit(); + queryCancellation.checkCancelled(); return in.size(); } @Override public int getDocCount() { - queryTimeout.shouldExit(); + queryCancellation.checkCancelled(); return in.getDocCount(); } } @@ -231,17 +236,17 @@ private static class ExitableIntersectVisitor implements PointValues.IntersectVi private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = (1 << 4) - 1; // 15 private final PointValues.IntersectVisitor in; - private final QueryTimeout queryTimeout; + private final QueryCancellation queryCancellation; private int calls; - private ExitableIntersectVisitor(PointValues.IntersectVisitor in, QueryTimeout queryTimeout) { + private ExitableIntersectVisitor(PointValues.IntersectVisitor in, QueryCancellation queryCancellation) { this.in = in; - this.queryTimeout = queryTimeout; + this.queryCancellation = queryCancellation; } private void checkAndThrowWithSampling() { if ((calls++ & MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK) == 0) { - queryTimeout.shouldExit(); + queryCancellation.checkCancelled(); } } @@ -259,13 +264,13 @@ public void visit(int docID, byte[] packedValue) throws IOException { @Override public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { - queryTimeout.shouldExit(); + queryCancellation.checkCancelled(); return in.compare(minPackedValue, maxPackedValue); } @Override public void grow(int count) { - queryTimeout.shouldExit(); + queryCancellation.checkCancelled(); in.grow(count); } } From 087f2ad9a9d9cedaaae0ef718ec123858a08dd95 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Wed, 4 Mar 2020 18:19:38 +0100 Subject: [PATCH 25/26] document iface --- .../search/internal/ExitableDirectoryReader.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java b/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java index d79c16bb720fe..b66532bbd093e 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java @@ -39,10 +39,22 @@ */ class ExitableDirectoryReader extends FilterDirectoryReader { + /** + * Used to check if query cancellation is actually enabled + * and if so use it to check if the query is cancelled or timed-out. + */ interface QueryCancellation { + /** + * Used to prevent unnecessary checks for cancellation + * @return true if query cancellation is enabled + */ boolean isEnabled(); + /** + * Call to check if the query is cancelled or timed-out. + * If so a {@link RuntimeException} is thrown + */ void checkCancelled(); } From eb158e54a363eda92f3e7830e6158f5e8da555eb Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Thu, 5 Mar 2020 11:40:42 +0100 Subject: [PATCH 26/26] fix comment --- .../elasticsearch/search/internal/ContextIndexSearcher.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 839e6844fb71f..d3c4f5641dc23 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -106,7 +106,7 @@ public void setProfiler(QueryProfiler profiler) { } /** - * Add a {@link Runnable} that will be run on a regular basis while fetching document from the + * Add a {@link Runnable} that will be run on a regular basis while accessing documents in the * DirectoryReader but also while collecting them and check for query cancellation or timeout. */ public Runnable addQueryCancellation(Runnable action) { @@ -115,7 +115,7 @@ public Runnable addQueryCancellation(Runnable action) { /** * Remove a {@link Runnable} that checks for query cancellation or timeout - * which is called while fetching documents from the DirectoryReader but also while collecting them. + * which is called while accessing documents in the DirectoryReader but also while collecting them. */ public void removeQueryCancellation(Runnable action) { this.cancellable.remove(action);