From e23d11b2cf83377ff4272f76f55ce2db0ccfb217 Mon Sep 17 00:00:00 2001 From: Zach Chen Date: Tue, 10 Aug 2021 19:45:50 -0700 Subject: [PATCH 01/14] LUCENE-10002: Deprecate IndexSearch#search(Query, Collector) in favor of IndexSearcher#search(Query, CollectorManager) Refactor out TopScoreDocCollectorManager, TopFieldCollectorManager, TotalHitCountCollectorManager and FixedBitSetCollector Refactor some tests to use the above collector manager --- .../benchmark/byTask/tasks/ReadTask.java | 20 ++- .../apache/lucene/search/IndexSearcher.java | 74 ++------ .../apache/lucene/search/SortRescorer.java | 3 +- .../org/apache/lucene/search/TopDocs.java | 2 +- .../lucene/search/TopFieldCollector.java | 103 ++--------- .../search/TopFieldCollectorManager.java | 141 +++++++++++++++ .../lucene/search/TopScoreDocCollector.java | 70 ++------ .../search/TopScoreDocCollectorManager.java | 97 ++++++++++ .../search/TotalHitCountCollectorManager.java | 37 ++++ .../lucene/document/BaseSpatialTestCase.java | 41 +---- .../TestLatLonPointDistanceFeatureQuery.java | 32 ++-- .../TestLongDistanceFeatureQuery.java | 37 ++-- .../apache/lucene/index/TestIndexSorting.java | 14 +- .../lucene/index/TestIndexWriterMaxDocs.java | 8 +- .../apache/lucene/search/TestBoolean2.java | 53 +++--- .../search/TestBooleanMinShouldMatch.java | 7 +- .../lucene/search/TestConstantScoreQuery.java | 88 +++++---- .../search/TestConstantScoreScorer.java | 16 +- .../search/TestFuzzyTermOnShortTerms.java | 5 +- .../lucene/search/TestMatchAllDocsQuery.java | 17 +- .../apache/lucene/search/TestPhraseQuery.java | 43 +++-- .../lucene/search/TestPointQueries.java | 49 +---- .../lucene/search/TestReqOptSumScorer.java | 37 ++-- .../lucene/search/TestSearchWithThreads.java | 9 +- .../lucene/search/TestSortOptimization.java | 155 ++++++++-------- .../lucene/search/TestSynonymQuery.java | 38 ++-- .../apache/lucene/search/TestTermQuery.java | 15 +- .../apache/lucene/search/TestTermScorer.java | 23 +-- .../lucene/search/TestTopDocsCollector.java | 78 ++++---- .../lucene/search/TestTopFieldCollector.java | 167 ++++++++++-------- ...TestTopFieldCollectorEarlyTermination.java | 25 +-- .../search/TestTotalHitCountCollector.java | 5 +- .../lucene/luke/models/search/SearchImpl.java | 8 +- .../lucene/queries/spans/TestBasics.java | 21 ++- .../search/TestCombinedFieldQuery.java | 30 ++-- .../analyzing/AnalyzingInfixSuggester.java | 8 +- .../lucene/geo/BaseGeoPointTestCase.java | 28 +-- .../lucene/geo/BaseXYPointTestCase.java | 28 +-- .../search/BaseRangeFieldQueryTestCase.java | 25 +-- .../org/apache/lucene/search/CheckHits.java | 13 +- .../lucene/util/FixedBitSetCollector.java | 68 +++++++ 41 files changed, 908 insertions(+), 830 deletions(-) create mode 100644 lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java create mode 100644 lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java create mode 100644 lucene/core/src/java/org/apache/lucene/search/TotalHitCountCollectorManager.java create mode 100644 lucene/test-framework/src/java/org/apache/lucene/util/FixedBitSetCollector.java diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java index ee7f00029007..0a4adeb9614f 100644 --- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java +++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java @@ -29,8 +29,9 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TopFieldCollector; +import org.apache.lucene.search.TopFieldCollectorManager; import org.apache.lucene.search.TopScoreDocCollector; +import org.apache.lucene.search.TopScoreDocCollectorManager; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; @@ -108,17 +109,19 @@ public int doLogic() throws Exception { // the IndexSearcher search methods that take // Weight public again, we can go back to // pulling the Weight ourselves: - TopFieldCollector collector = - TopFieldCollector.create(sort, numHits, withTotalHits() ? Integer.MAX_VALUE : 1); - searcher.search(q, collector); - hits = collector.topDocs(); + TopFieldCollectorManager collectorManager = + TopFieldCollectorManager.create( + sort, numHits, withTotalHits() ? Integer.MAX_VALUE : 1); + hits = searcher.search(q, collectorManager); } else { hits = searcher.search(q, numHits); } } else { - Collector collector = createCollector(); - searcher.search(q, collector); - // hits = collector.topDocs(); + TopScoreDocCollectorManager collectorManager = + TopScoreDocCollectorManager.create( + numHits(), withTotalHits() ? Integer.MAX_VALUE : 1); + searcher.search(q, collectorManager); + // hits = collectorManager.topDocs(); } if (hits != null) { @@ -180,6 +183,7 @@ protected int withTopDocs(IndexSearcher searcher, Query q, TopDocs hits) throws return res; } + @Deprecated protected Collector createCollector() throws Exception { return TopScoreDocCollector.create(numHits(), withTotalHits() ? Integer.MAX_VALUE : 1); } diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java index d7937a5cba95..268897c665f3 100644 --- a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java +++ b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java @@ -66,9 +66,9 @@ * match lots of documents, counting the number of hits may take much longer than computing the top * hits so this trade-off allows to get some minimal information about the hit count without slowing * down search too much. The {@link TopDocs#scoreDocs} array is always accurate however. If this - * behavior doesn't suit your needs, you should create collectors manually with either {@link - * TopScoreDocCollector#create} or {@link TopFieldCollector#create} and call {@link #search(Query, - * Collector)}. + * behavior doesn't suit your needs, you should create collectorManagers manually with either {@link + * TopScoreDocCollectorManager#create} or {@link TopFieldCollectorManager#create} and call {@link + * #search(Query, CollectorManager)}. * *

* @@ -478,34 +478,8 @@ public TopDocs searchAfter(ScoreDoc after, Query query, int numHits) throws IOEx } final int cappedNumHits = Math.min(numHits, limit); - - final CollectorManager manager = - new CollectorManager() { - - private final HitsThresholdChecker hitsThresholdChecker = - (executor == null || leafSlices.length <= 1) - ? HitsThresholdChecker.create(Math.max(TOTAL_HITS_THRESHOLD, numHits)) - : HitsThresholdChecker.createShared(Math.max(TOTAL_HITS_THRESHOLD, numHits)); - - private final MaxScoreAccumulator minScoreAcc = - (executor == null || leafSlices.length <= 1) ? null : new MaxScoreAccumulator(); - - @Override - public TopScoreDocCollector newCollector() throws IOException { - return TopScoreDocCollector.create( - cappedNumHits, after, hitsThresholdChecker, minScoreAcc); - } - - @Override - public TopDocs reduce(Collection collectors) throws IOException { - final TopDocs[] topDocs = new TopDocs[collectors.size()]; - int i = 0; - for (TopScoreDocCollector collector : collectors) { - topDocs[i++] = collector.topDocs(); - } - return TopDocs.merge(0, cappedNumHits, topDocs); - } - }; + CollectorManager manager = + new TopScoreDocCollectorManager(cappedNumHits, after, TOTAL_HITS_THRESHOLD); return search(query, manager); } @@ -527,7 +501,10 @@ public TopDocs search(Query query, int n) throws IOException { * * @throws TooManyClauses If a query would exceed {@link IndexSearcher#getMaxClauseCount()} * clauses. + * @deprecated This method is being deprecated in favor of {@link IndexSearcher#search(Query, + * CollectorManager)} due to its support for concurrency in IndexSearcher */ + @Deprecated public void search(Query query, Collector results) throws IOException { query = rewrite(query); search(leafContexts, createWeight(query, results.scoreMode(), 1), results); @@ -614,33 +591,7 @@ private TopFieldDocs searchAfter( final Sort rewrittenSort = sort.rewrite(this); final CollectorManager manager = - new CollectorManager<>() { - - private final HitsThresholdChecker hitsThresholdChecker = - (executor == null || leafSlices.length <= 1) - ? HitsThresholdChecker.create(Math.max(TOTAL_HITS_THRESHOLD, numHits)) - : HitsThresholdChecker.createShared(Math.max(TOTAL_HITS_THRESHOLD, numHits)); - - private final MaxScoreAccumulator minScoreAcc = - (executor == null || leafSlices.length <= 1) ? null : new MaxScoreAccumulator(); - - @Override - public TopFieldCollector newCollector() throws IOException { - // TODO: don't pay the price for accurate hit counts by default - return TopFieldCollector.create( - rewrittenSort, cappedNumHits, after, hitsThresholdChecker, minScoreAcc); - } - - @Override - public TopFieldDocs reduce(Collection collectors) throws IOException { - final TopFieldDocs[] topDocs = new TopFieldDocs[collectors.size()]; - int i = 0; - for (TopFieldCollector collector : collectors) { - topDocs[i++] = collector.topDocs(); - } - return TopDocs.merge(rewrittenSort, 0, cappedNumHits, topDocs); - } - }; + new TopFieldCollectorManager(rewrittenSort, cappedNumHits, after, TOTAL_HITS_THRESHOLD); TopFieldDocs topDocs = search(query, manager); if (doDocScores) { @@ -659,9 +610,12 @@ public TopFieldDocs reduce(Collection collectors) throws IOEx */ public T search(Query query, CollectorManager collectorManager) throws IOException { - if (executor == null || leafSlices.length <= 1) { + if (executor == null || leafSlices.length == 0) { final C collector = collectorManager.newCollector(); - search(query, collector); + final Query rewrittenQuery = rewrite(query); + + search(leafContexts, createWeight(rewrittenQuery, collector.scoreMode(), 1), collector); + return collectorManager.reduce(Collections.singletonList(collector)); } else { final List collectors = new ArrayList<>(leafSlices.length); diff --git a/lucene/core/src/java/org/apache/lucene/search/SortRescorer.java b/lucene/core/src/java/org/apache/lucene/search/SortRescorer.java index 5b334c2f4d6f..095f32695eac 100644 --- a/lucene/core/src/java/org/apache/lucene/search/SortRescorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/SortRescorer.java @@ -45,7 +45,8 @@ public TopDocs rescore(IndexSearcher searcher, TopDocs firstPassTopDocs, int top List leaves = searcher.getIndexReader().leaves(); - TopFieldCollector collector = TopFieldCollector.create(sort, topN, Integer.MAX_VALUE); + TopFieldCollector collector = + TopFieldCollectorManager.create(sort, topN, Integer.MAX_VALUE).newCollector(); // Now merge sort docIDs from hits, with reader's leaves: int hitUpto = 0; diff --git a/lucene/core/src/java/org/apache/lucene/search/TopDocs.java b/lucene/core/src/java/org/apache/lucene/search/TopDocs.java index 0b468a4434ce..66c6a266e8f8 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopDocs.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopDocs.java @@ -233,7 +233,7 @@ public static TopDocs merge( * Returns a new TopFieldDocs, containing topN results across the provided TopFieldDocs, sorting * by the specified {@link Sort}. Each of the TopDocs must have been sorted by the same Sort, and * sort field values must have been filled (ie, fillFields=true must be passed to - * {@link TopFieldCollector#create}). + * {@link TopFieldCollectorManager#create}). * * @see #merge(Sort, int, int, TopFieldDocs[]) * @lucene.experimental diff --git a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java index 6796ff911131..32601d935d1b 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java @@ -18,7 +18,6 @@ import java.io.IOException; import java.util.Arrays; -import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.Objects; @@ -174,7 +173,7 @@ private static boolean canEarlyTerminateOnPrefix(Sort searchSort, Sort indexSort * Implements a TopFieldCollector over one SortField criteria, with tracking * document scores and maxScore. */ - private static class SimpleFieldCollector extends TopFieldCollector { + static class SimpleFieldCollector extends TopFieldCollector { final Sort sort; final FieldValueHitQueue queue; @@ -214,7 +213,7 @@ public void collect(int doc) throws IOException { /* * Implements a TopFieldCollector when after != null. */ - private static final class PagingFieldCollector extends TopFieldCollector { + static final class PagingFieldCollector extends TopFieldCollector { final Sort sort; int collectedHits; @@ -383,9 +382,13 @@ protected void updateMinCompetitiveScore(Scorable scorer) throws IOException { * count of the result will be accurate. {@link Integer#MAX_VALUE} may be used to make the hit * count accurate, but this will also make query processing slower. * @return a {@link TopFieldCollector} instance which will sort the results by the sort criteria. + * @deprecated This method is being deprecated in favor of {@link + * TopFieldCollectorManager#create(Sort, int, int)} due to its support for concurrency in + * IndexSearcher */ + @Deprecated public static TopFieldCollector create(Sort sort, int numHits, int totalHitsThreshold) { - return create(sort, numHits, null, totalHitsThreshold); + return TopFieldCollectorManager.create(sort, numHits, totalHitsThreshold).newCollector(); } /** @@ -407,97 +410,13 @@ public static TopFieldCollector create(Sort sort, int numHits, int totalHitsThre * field is indexed both with doc values and points. In this case, there is an assumption that * the same data is stored in these points and doc values. * @return a {@link TopFieldCollector} instance which will sort the results by the sort criteria. + * @deprecated This method is being deprecated in favor of using the constructor of {@link + * TopFieldCollectorManager} due to its support for concurrency in IndexSearcher */ + @Deprecated public static TopFieldCollector create( Sort sort, int numHits, FieldDoc after, int totalHitsThreshold) { - if (totalHitsThreshold < 0) { - throw new IllegalArgumentException( - "totalHitsThreshold must be >= 0, got " + totalHitsThreshold); - } - - return create( - sort, - numHits, - after, - HitsThresholdChecker.create(Math.max(totalHitsThreshold, numHits)), - null /* bottomValueChecker */); - } - - /** - * Same as above with additional parameters to allow passing in the threshold checker and the max - * score accumulator. - */ - static TopFieldCollector create( - Sort sort, - int numHits, - FieldDoc after, - HitsThresholdChecker hitsThresholdChecker, - MaxScoreAccumulator minScoreAcc) { - - if (sort.fields.length == 0) { - throw new IllegalArgumentException("Sort must contain at least one field"); - } - - if (numHits <= 0) { - throw new IllegalArgumentException( - "numHits must be > 0; please use TotalHitCountCollector if you just need the total hit count"); - } - - if (hitsThresholdChecker == null) { - throw new IllegalArgumentException("hitsThresholdChecker should not be null"); - } - - FieldValueHitQueue queue = FieldValueHitQueue.create(sort.fields, numHits); - - if (after == null) { - return new SimpleFieldCollector(sort, queue, numHits, hitsThresholdChecker, minScoreAcc); - } else { - if (after.fields == null) { - throw new IllegalArgumentException( - "after.fields wasn't set; you must pass fillFields=true for the previous search"); - } - - if (after.fields.length != sort.getSort().length) { - throw new IllegalArgumentException( - "after.fields has " - + after.fields.length - + " values but sort has " - + sort.getSort().length); - } - - return new PagingFieldCollector( - sort, queue, after, numHits, hitsThresholdChecker, minScoreAcc); - } - } - - /** - * Create a CollectorManager which uses a shared hit counter to maintain number of hits and a - * shared {@link MaxScoreAccumulator} to propagate the minimum score accross segments if the - * primary sort is by relevancy. - */ - public static CollectorManager createSharedManager( - Sort sort, int numHits, FieldDoc after, int totalHitsThreshold) { - return new CollectorManager<>() { - - private final HitsThresholdChecker hitsThresholdChecker = - HitsThresholdChecker.createShared(Math.max(totalHitsThreshold, numHits)); - private final MaxScoreAccumulator minScoreAcc = new MaxScoreAccumulator(); - - @Override - public TopFieldCollector newCollector() throws IOException { - return create(sort, numHits, after, hitsThresholdChecker, minScoreAcc); - } - - @Override - public TopFieldDocs reduce(Collection collectors) throws IOException { - final TopFieldDocs[] topDocs = new TopFieldDocs[collectors.size()]; - int i = 0; - for (TopFieldCollector collector : collectors) { - topDocs[i++] = collector.topDocs(); - } - return TopDocs.merge(sort, 0, numHits, topDocs); - } - }; + return new TopFieldCollectorManager(sort, numHits, after, totalHitsThreshold).newCollector(); } /** diff --git a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java new file mode 100644 index 000000000000..2f648a4d6b32 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Create a TopFieldCollectorManager which uses a shared hit counter to maintain number of hits and + * a shared {@link MaxScoreAccumulator} to propagate the minimum score across segments if the + * primary sort is by relevancy. + * + *

Note that a new collectorManager should be created for each search due to its internal states. + */ +public class TopFieldCollectorManager implements CollectorManager { + private final Sort sort; + private final int numHits; + private final FieldDoc after; + private final HitsThresholdChecker hitsThresholdChecker; + private final MaxScoreAccumulator minScoreAcc; + private final List collectors; + + public TopFieldCollectorManager(Sort sort, int numHits, FieldDoc after, int totalHitsThreshold) { + if (totalHitsThreshold < 0) { + throw new IllegalArgumentException( + "totalHitsThreshold must be >= 0, got " + totalHitsThreshold); + } + + this.sort = sort; + this.numHits = numHits; + this.after = after; + /* + nocommit + Should the following two be passed in instead? Possible custom initialization based on executor status and slices? + On the other hand, in a single-threaded environment, shared HitsThresholdChecker and MaxScoreAccumulator should be fast without lock contention anyway? + + final HitsThresholdChecker hitsThresholdChecker = + (executor == null || leafSlices.length <= 1) + ? HitsThresholdChecker.create(Math.max(TOTAL_HITS_THRESHOLD, numHits)) + : HitsThresholdChecker.createShared(Math.max(TOTAL_HITS_THRESHOLD, numHits)); + + final MaxScoreAccumulator minScoreAcc = (executor == null || leafSlices.length <= 1) ? null : new MaxScoreAccumulator(); + */ + this.hitsThresholdChecker = + HitsThresholdChecker.createShared(Math.max(totalHitsThreshold, numHits)); + this.minScoreAcc = new MaxScoreAccumulator(); + this.collectors = new ArrayList<>(); + } + + /** + * Creates a new {@link TopFieldCollectorManager} from the given arguments. + * + *

NOTE: The instances returned by this method pre-allocate a full array of length + * numHits. + * + * @param sort the sort criteria (SortFields). + * @param numHits the number of results to collect. + * @param totalHitsThreshold the number of docs to count accurately. If the query matches more + * than {@code totalHitsThreshold} hits then its hit count will be a lower bound. On the other + * hand if the query matches less than or exactly {@code totalHitsThreshold} hits then the hit + * count of the result will be accurate. {@link Integer#MAX_VALUE} may be used to make the hit + * count accurate, but this will also make query processing slower. + * @return a {@link TopFieldCollectorManager} instance which will be used to create {@link + * TopFieldCollector} to sort the results by the sort criteria. + */ + public static TopFieldCollectorManager create(Sort sort, int numHits, int totalHitsThreshold) { + return new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); + } + + @Override + public TopFieldCollector newCollector() { + if (sort.fields.length == 0) { + throw new IllegalArgumentException("Sort must contain at least one field"); + } + + if (numHits <= 0) { + throw new IllegalArgumentException( + "numHits must be > 0; please use TotalHitCountCollector if you just need the total hit count"); + } + + FieldValueHitQueue queue = + FieldValueHitQueue.create(sort.fields, numHits); + + TopFieldCollector collector; + if (after == null) { + collector = + new TopFieldCollector.SimpleFieldCollector( + sort, queue, numHits, hitsThresholdChecker, minScoreAcc); + } else { + if (after.fields == null) { + throw new IllegalArgumentException( + "after.fields wasn't set; you must pass fillFields=true for the previous search"); + } + + if (after.fields.length != sort.getSort().length) { + throw new IllegalArgumentException( + "after.fields has " + + after.fields.length + + " values but sort has " + + sort.getSort().length); + } + + collector = + new TopFieldCollector.PagingFieldCollector( + sort, queue, after, numHits, hitsThresholdChecker, minScoreAcc); + } + + collectors.add(collector); + return collector; + } + + @Override + public TopFieldDocs reduce(Collection collectors) throws IOException { + final TopFieldDocs[] topDocs = new TopFieldDocs[collectors.size()]; + int i = 0; + for (TopFieldCollector collector : collectors) { + topDocs[i++] = collector.topDocs(); + } + return TopDocs.merge(sort, 0, numHits, topDocs); + } + + public List getCollectors() { + return collectors; + } +} diff --git a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java index 87d2a2710f30..f684fab7557e 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java @@ -17,7 +17,6 @@ package org.apache.lucene.search; import java.io.IOException; -import java.util.Collection; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.MaxScoreAccumulator.DocAndScore; @@ -44,7 +43,7 @@ public void setScorer(Scorable scorer) throws IOException { } } - private static class SimpleTopScoreDocCollector extends TopScoreDocCollector { + static class SimpleTopScoreDocCollector extends TopScoreDocCollector { SimpleTopScoreDocCollector( int numHits, HitsThresholdChecker hitsThresholdChecker, MaxScoreAccumulator minScoreAcc) { @@ -101,7 +100,7 @@ public void collect(int doc) throws IOException { } } - private static class PagingTopScoreDocCollector extends TopScoreDocCollector { + static class PagingTopScoreDocCollector extends TopScoreDocCollector { private final ScoreDoc after; private int collectedHits; @@ -192,9 +191,14 @@ public void collect(int doc) throws IOException { * *

NOTE: The instances returned by this method pre-allocate a full array of length * numHits, and fill the array with sentinel objects. + * + * @deprecated This method is being deprecated in favor of {@link + * TopScoreDocCollectorManager#create(int, int)} due to its support for concurrency in + * IndexSearcher */ + @Deprecated public static TopScoreDocCollector create(int numHits, int totalHitsThreshold) { - return create(numHits, null, totalHitsThreshold); + return TopScoreDocCollectorManager.create(numHits, totalHitsThreshold).newCollector(); } /** @@ -209,61 +213,13 @@ public static TopScoreDocCollector create(int numHits, int totalHitsThreshold) { * *

NOTE: The instances returned by this method pre-allocate a full array of length * numHits, and fill the array with sentinel objects. + * + * @deprecated This method is being deprecated in favor of using the constructor of {@link + * TopScoreDocCollectorManager} due to its support for concurrency in IndexSearcher */ + @Deprecated public static TopScoreDocCollector create(int numHits, ScoreDoc after, int totalHitsThreshold) { - return create( - numHits, after, HitsThresholdChecker.create(Math.max(totalHitsThreshold, numHits)), null); - } - - static TopScoreDocCollector create( - int numHits, - ScoreDoc after, - HitsThresholdChecker hitsThresholdChecker, - MaxScoreAccumulator minScoreAcc) { - - if (numHits <= 0) { - throw new IllegalArgumentException( - "numHits must be > 0; please use TotalHitCountCollector if you just need the total hit count"); - } - - if (hitsThresholdChecker == null) { - throw new IllegalArgumentException("hitsThresholdChecker must be non null"); - } - - if (after == null) { - return new SimpleTopScoreDocCollector(numHits, hitsThresholdChecker, minScoreAcc); - } else { - return new PagingTopScoreDocCollector(numHits, after, hitsThresholdChecker, minScoreAcc); - } - } - - /** - * Create a CollectorManager which uses a shared hit counter to maintain number of hits and a - * shared {@link MaxScoreAccumulator} to propagate the minimum score accross segments - */ - public static CollectorManager createSharedManager( - int numHits, FieldDoc after, int totalHitsThreshold) { - return new CollectorManager<>() { - - private final HitsThresholdChecker hitsThresholdChecker = - HitsThresholdChecker.createShared(Math.max(totalHitsThreshold, numHits)); - private final MaxScoreAccumulator minScoreAcc = new MaxScoreAccumulator(); - - @Override - public TopScoreDocCollector newCollector() throws IOException { - return TopScoreDocCollector.create(numHits, after, hitsThresholdChecker, minScoreAcc); - } - - @Override - public TopDocs reduce(Collection collectors) throws IOException { - final TopDocs[] topDocs = new TopDocs[collectors.size()]; - int i = 0; - for (TopScoreDocCollector collector : collectors) { - topDocs[i++] = collector.topDocs(); - } - return TopDocs.merge(0, numHits, topDocs); - } - }; + return new TopScoreDocCollectorManager(numHits, after, totalHitsThreshold).newCollector(); } int docBase; diff --git a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java new file mode 100644 index 000000000000..f089371ae2c2 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.search; + +import java.io.IOException; +import java.util.Collection; + +/** + * Create a TopScoreDocCollectorManager which uses a shared hit counter to maintain number of hits + * and a shared {@link MaxScoreAccumulator} to propagate the minimum score across segments + * + *

Note that a new collectorManager should be created for each search due to its internal states. + */ +public class TopScoreDocCollectorManager + implements CollectorManager { + private final int numHits; + private final ScoreDoc after; + private final HitsThresholdChecker hitsThresholdChecker; + private final MaxScoreAccumulator minScoreAcc; + + public TopScoreDocCollectorManager(int numHits, ScoreDoc after, int totalHitsThreshold) { + if (numHits <= 0) { + throw new IllegalArgumentException( + "numHits must be > 0; please use TotalHitCountCollectorManager if you just need the total hit count"); + } + + this.numHits = numHits; + this.after = after; + /* + nocommit + Should the following two be passed in instead? Possible custom initialization based on executor status and slices? + On the other hand, in a single-threaded environment, shared HitsThresholdChecker and MaxScoreAccumulator should be fast without lock contention anyway? + + final HitsThresholdChecker hitsThresholdChecker = + (executor == null || leafSlices.length <= 1) + ? HitsThresholdChecker.create(Math.max(TOTAL_HITS_THRESHOLD, numHits)) + : HitsThresholdChecker.createShared(Math.max(TOTAL_HITS_THRESHOLD, numHits)); + + final MaxScoreAccumulator minScoreAcc = (executor == null || leafSlices.length <= 1) ? null : new MaxScoreAccumulator(); + */ + this.hitsThresholdChecker = + HitsThresholdChecker.createShared(Math.max(totalHitsThreshold, numHits)); + this.minScoreAcc = new MaxScoreAccumulator(); + } + + /** + * Creates a new {@link TopScoreDocCollectorManager} given the number of hits to collect and the + * number of hits to count accurately. + * + *

NOTE: If the total hit count of the top docs is less than or exactly {@code + * totalHitsThreshold} then this value is accurate. On the other hand, if the {@link + * TopDocs#totalHits} value is greater than {@code totalHitsThreshold} then its value is a lower + * bound of the hit count. A value of {@link Integer#MAX_VALUE} will make the hit count accurate + * but will also likely make query processing slower. + * + *

NOTE: The instances returned by this method pre-allocate a full array of length + * numHits, and fill the array with sentinel objects. + */ + public static TopScoreDocCollectorManager create(int numHits, int totalHitsThreshold) { + return new TopScoreDocCollectorManager(numHits, null, totalHitsThreshold); + } + + @Override + public TopScoreDocCollector newCollector() { + if (after == null) { + return new TopScoreDocCollector.SimpleTopScoreDocCollector( + numHits, hitsThresholdChecker, minScoreAcc); + } else { + return new TopScoreDocCollector.PagingTopScoreDocCollector( + numHits, after, hitsThresholdChecker, minScoreAcc); + } + } + + @Override + public TopDocs reduce(Collection collectors) throws IOException { + final TopDocs[] topDocs = new TopDocs[collectors.size()]; + int i = 0; + for (TopScoreDocCollector collector : collectors) { + topDocs[i++] = collector.topDocs(); + } + return TopDocs.merge(0, numHits, topDocs); + } +} diff --git a/lucene/core/src/java/org/apache/lucene/search/TotalHitCountCollectorManager.java b/lucene/core/src/java/org/apache/lucene/search/TotalHitCountCollectorManager.java new file mode 100644 index 000000000000..523fc753619c --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/search/TotalHitCountCollectorManager.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.search; + +import java.util.Collection; + +/** CollectorManager that aggregates counts to get the total number of hits. */ +public class TotalHitCountCollectorManager + implements CollectorManager { + @Override + public TotalHitCountCollector newCollector() { + return new TotalHitCountCollector(); + } + + @Override + public Integer reduce(Collection collectors) { + int total = 0; + for (TotalHitCountCollector collector : collectors) { + total += collector.getTotalHits(); + } + return total; + } +} diff --git a/lucene/core/src/test/org/apache/lucene/document/BaseSpatialTestCase.java b/lucene/core/src/test/org/apache/lucene/document/BaseSpatialTestCase.java index 2d2da74e2800..122e8e970618 100644 --- a/lucene/core/src/test/org/apache/lucene/document/BaseSpatialTestCase.java +++ b/lucene/core/src/test/org/apache/lucene/document/BaseSpatialTestCase.java @@ -20,7 +20,6 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween; import com.carrotsearch.randomizedtesting.generators.RandomPicks; -import java.io.IOException; import java.util.Arrays; import java.util.HashSet; import java.util.Set; @@ -31,7 +30,6 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiBits; import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.NumericDocValues; @@ -39,11 +37,10 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.SimpleCollector; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.FixedBitSetCollector; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; @@ -323,7 +320,7 @@ protected void verifyRandomBBoxQueries(IndexReader reader, Object... shapes) thr System.out.println(" query=" + query + ", relation=" + queryRelation); } - final FixedBitSet hits = searchIndex(s, query, maxDoc); + final FixedBitSet hits = s.search(query, FixedBitSetCollector.create(maxDoc)); boolean fail = false; NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id"); @@ -420,7 +417,7 @@ protected void verifyRandomLineQueries(IndexReader reader, Object... shapes) thr System.out.println(" query=" + query + ", relation=" + queryRelation); } - final FixedBitSet hits = searchIndex(s, query, maxDoc); + final FixedBitSet hits = s.search(query, FixedBitSetCollector.create(maxDoc)); boolean fail = false; NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id"); @@ -493,7 +490,7 @@ protected void verifyRandomPolygonQueries(IndexReader reader, Object... shapes) System.out.println(" query=" + query + ", relation=" + queryRelation); } - final FixedBitSet hits = searchIndex(s, query, maxDoc); + final FixedBitSet hits = s.search(query, FixedBitSetCollector.create(maxDoc)); boolean fail = false; NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id"); @@ -572,7 +569,7 @@ protected void verifyRandomPointQueries(IndexReader reader, Object... shapes) th System.out.println(" query=" + query + ", relation=" + queryRelation); } - final FixedBitSet hits = searchIndex(s, query, maxDoc); + final FixedBitSet hits = s.search(query, FixedBitSetCollector.create(maxDoc)); boolean fail = false; NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id"); @@ -647,7 +644,7 @@ protected void verifyRandomDistanceQueries(IndexReader reader, Object... shapes) System.out.println(" query=" + query + ", relation=" + queryRelation); } - final FixedBitSet hits = searchIndex(s, query, maxDoc); + final FixedBitSet hits = s.search(query, FixedBitSetCollector.create(maxDoc)); boolean fail = false; NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id"); @@ -696,32 +693,6 @@ protected void verifyRandomDistanceQueries(IndexReader reader, Object... shapes) } } - private FixedBitSet searchIndex(IndexSearcher s, Query query, int maxDoc) throws IOException { - final FixedBitSet hits = new FixedBitSet(maxDoc); - s.search( - query, - new SimpleCollector() { - - private int docBase; - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE_NO_SCORES; - } - - @Override - protected void doSetNextReader(LeafReaderContext context) { - docBase = context.docBase; - } - - @Override - public void collect(int doc) { - hits.set(docBase + doc); - } - }); - return hits; - } - protected abstract Validator getValidator(); protected abstract static class Encoder { diff --git a/lucene/core/src/test/org/apache/lucene/document/TestLatLonPointDistanceFeatureQuery.java b/lucene/core/src/test/org/apache/lucene/document/TestLatLonPointDistanceFeatureQuery.java index f5bfb9ead306..0a2957bc596c 100644 --- a/lucene/core/src/test/org/apache/lucene/document/TestLatLonPointDistanceFeatureQuery.java +++ b/lucene/core/src/test/org/apache/lucene/document/TestLatLonPointDistanceFeatureQuery.java @@ -33,7 +33,7 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TopScoreDocCollector; +import org.apache.lucene.search.TopScoreDocCollectorManager; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.SloppyMath; @@ -100,9 +100,8 @@ public void testBasics() throws IOException { IndexSearcher searcher = newSearcher(reader); Query q = LatLonPoint.newDistanceFeatureQuery("foo", 3, 10, 10, pivotDistance); - TopScoreDocCollector collector = TopScoreDocCollector.create(2, null, 1); - searcher.search(q, collector); - TopDocs topHits = collector.topDocs(); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, null, 1); + TopDocs topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); double distance1 = @@ -140,9 +139,8 @@ public void testBasics() throws IOException { 9); q = LatLonPoint.newDistanceFeatureQuery("foo", 3, 9, 9, pivotDistance); - collector = TopScoreDocCollector.create(2, null, 1); - searcher.search(q, collector); - topHits = collector.topDocs(); + collectorManager = new TopScoreDocCollectorManager(2, null, 1); + topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); CheckHits.checkExplanations(q, "", searcher); @@ -198,9 +196,8 @@ public void testCrossesDateLine() throws IOException { IndexSearcher searcher = newSearcher(reader); Query q = LatLonPoint.newDistanceFeatureQuery("foo", 3, 0, 179, pivotDistance); - TopScoreDocCollector collector = TopScoreDocCollector.create(2, null, 1); - searcher.search(q, collector); - TopDocs topHits = collector.topDocs(); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, null, 1); + TopDocs topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); double distance1 = @@ -265,9 +262,8 @@ public void testMissingValue() throws IOException { IndexSearcher searcher = newSearcher(reader); Query q = LatLonPoint.newDistanceFeatureQuery("foo", 3, 10, 10, 5); - TopScoreDocCollector collector = TopScoreDocCollector.create(3, null, 1); - searcher.search(q, collector); - TopDocs topHits = collector.topDocs(); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(3, null, 1); + TopDocs topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); double distance1 = @@ -345,9 +341,8 @@ public void testMultiValued() throws IOException { IndexSearcher searcher = newSearcher(reader); Query q = LatLonPoint.newDistanceFeatureQuery("foo", 3, 0, 0, 200); - TopScoreDocCollector collector = TopScoreDocCollector.create(2, null, 1); - searcher.search(q, collector); - TopDocs topHits = collector.topDocs(); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, null, 1); + TopDocs topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); double distance1 = @@ -372,9 +367,8 @@ public void testMultiValued() throws IOException { topHits.scoreDocs); q = LatLonPoint.newDistanceFeatureQuery("foo", 3, -90, 0, 10000.); - collector = TopScoreDocCollector.create(2, null, 1); - searcher.search(q, collector); - topHits = collector.topDocs(); + collectorManager = new TopScoreDocCollectorManager(2, null, 1); + topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); CheckHits.checkExplanations(q, "", searcher); diff --git a/lucene/core/src/test/org/apache/lucene/document/TestLongDistanceFeatureQuery.java b/lucene/core/src/test/org/apache/lucene/document/TestLongDistanceFeatureQuery.java index 2e6db55f0288..702422fa2a83 100644 --- a/lucene/core/src/test/org/apache/lucene/document/TestLongDistanceFeatureQuery.java +++ b/lucene/core/src/test/org/apache/lucene/document/TestLongDistanceFeatureQuery.java @@ -28,7 +28,7 @@ import org.apache.lucene.search.QueryUtils; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TopScoreDocCollector; +import org.apache.lucene.search.TopScoreDocCollectorManager; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; @@ -89,9 +89,8 @@ public void testBasics() throws IOException { IndexSearcher searcher = newSearcher(reader); Query q = LongPoint.newDistanceFeatureQuery("foo", 3, 10, 5); - TopScoreDocCollector collector = TopScoreDocCollector.create(2, null, 1); - searcher.search(q, collector); - TopDocs topHits = collector.topDocs(); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, null, 1); + TopDocs topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); CheckHits.checkEqual( @@ -103,9 +102,8 @@ public void testBasics() throws IOException { topHits.scoreDocs); q = LongPoint.newDistanceFeatureQuery("foo", 3, 7, 5); - collector = TopScoreDocCollector.create(2, null, 1); - searcher.search(q, collector); - topHits = collector.topDocs(); + collectorManager = new TopScoreDocCollectorManager(2, null, 1); + topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); CheckHits.checkExplanations(q, "", searcher); @@ -159,9 +157,8 @@ public void testOverUnderFlow() throws IOException { IndexSearcher searcher = newSearcher(reader); Query q = LongPoint.newDistanceFeatureQuery("foo", 3, Long.MAX_VALUE - 1, 100); - TopScoreDocCollector collector = TopScoreDocCollector.create(2, null, 1); - searcher.search(q, collector); - TopDocs topHits = collector.topDocs(); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, null, 1); + TopDocs topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); CheckHits.checkEqual( @@ -181,9 +178,8 @@ public void testOverUnderFlow() throws IOException { topHits.scoreDocs); q = LongPoint.newDistanceFeatureQuery("foo", 3, Long.MIN_VALUE + 1, 100); - collector = TopScoreDocCollector.create(2, null, 1); - searcher.search(q, collector); - topHits = collector.topDocs(); + collectorManager = new TopScoreDocCollectorManager(2, null, 1); + topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); CheckHits.checkExplanations(q, "", searcher); @@ -244,9 +240,8 @@ public void testMissingValue() throws IOException { IndexSearcher searcher = newSearcher(reader); Query q = LongPoint.newDistanceFeatureQuery("foo", 3, 10, 5); - TopScoreDocCollector collector = TopScoreDocCollector.create(3, null, 1); - searcher.search(q, collector); - TopDocs topHits = collector.topDocs(); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(3, null, 1); + TopDocs topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); CheckHits.checkEqual( @@ -311,9 +306,8 @@ public void testMultiValued() throws IOException { IndexSearcher searcher = newSearcher(reader); Query q = LongPoint.newDistanceFeatureQuery("foo", 3, 10, 5); - TopScoreDocCollector collector = TopScoreDocCollector.create(2, null, 1); - searcher.search(q, collector); - TopDocs topHits = collector.topDocs(); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, null, 1); + TopDocs topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); CheckHits.checkEqual( @@ -325,9 +319,8 @@ public void testMultiValued() throws IOException { topHits.scoreDocs); q = LongPoint.newDistanceFeatureQuery("foo", 3, 7, 5); - collector = TopScoreDocCollector.create(2, null, 1); - searcher.search(q, collector); - topHits = collector.topDocs(); + collectorManager = new TopScoreDocCollectorManager(2, null, 1); + topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); CheckHits.checkExplanations(q, "", searcher); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java index ae10636c9f01..827e36474d90 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java @@ -76,7 +76,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermStatistics; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TopFieldCollector; +import org.apache.lucene.search.TopFieldCollectorManager; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; @@ -2643,14 +2643,12 @@ public void testRandom3() throws Exception { System.out.println("TEST: iter=" + iter + " numHits=" + numHits); } - TopFieldCollector c1 = TopFieldCollector.create(sort, numHits, Integer.MAX_VALUE); - s1.search(new MatchAllDocsQuery(), c1); - TopDocs hits1 = c1.topDocs(); + TopFieldCollectorManager c1 = + TopFieldCollectorManager.create(sort, numHits, Integer.MAX_VALUE); + TopDocs hits1 = s1.search(new MatchAllDocsQuery(), c1); - TopFieldCollector c2 = TopFieldCollector.create(sort, numHits, 1); - s2.search(new MatchAllDocsQuery(), c2); - - TopDocs hits2 = c2.topDocs(); + TopFieldCollectorManager c2 = TopFieldCollectorManager.create(sort, numHits, 1); + TopDocs hits2 = s2.search(new MatchAllDocsQuery(), c2); if (VERBOSE) { System.out.println(" topDocs query-time sort: totalHits=" + hits1.totalHits.value); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java index 7a68585fbb0a..35dde49b6eb3 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java @@ -28,7 +28,7 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TopScoreDocCollector; +import org.apache.lucene.search.TopScoreDocCollectorManager; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FilterDirectory; import org.apache.lucene.store.MockDirectoryWrapper; @@ -66,9 +66,9 @@ public void testExactlyAtTrueLimit() throws Exception { assertEquals(IndexWriter.MAX_DOCS, ir.maxDoc()); assertEquals(IndexWriter.MAX_DOCS, ir.numDocs()); IndexSearcher searcher = new IndexSearcher(ir); - TopScoreDocCollector collector = TopScoreDocCollector.create(10, Integer.MAX_VALUE); - searcher.search(new TermQuery(new Term("field", "text")), collector); - TopDocs hits = collector.topDocs(); + TopScoreDocCollectorManager collectorManager = + TopScoreDocCollectorManager.create(10, Integer.MAX_VALUE); + TopDocs hits = searcher.search(new TermQuery(new Term("field", "text")), collectorManager); assertEquals(IndexWriter.MAX_DOCS, hits.totalHits.value); // Sort by docID reversed: diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java b/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java index 5181c67a9309..48fc95a067e8 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java @@ -234,32 +234,29 @@ public void queriesTest(Query query, int[] expDocNrs) throws Exception { // The asserting searcher will sometimes return the bulk scorer and // sometimes return a default impl around the scorer so that we can // compare BS1 and BS2 - TopScoreDocCollector collector = TopScoreDocCollector.create(topDocsToCheck, Integer.MAX_VALUE); - searcher.search(query, collector); - ScoreDoc[] hits1 = collector.topDocs().scoreDocs; - collector = TopScoreDocCollector.create(topDocsToCheck, Integer.MAX_VALUE); - searcher.search(query, collector); - ScoreDoc[] hits2 = collector.topDocs().scoreDocs; + TopScoreDocCollectorManager collectorManager = + TopScoreDocCollectorManager.create(topDocsToCheck, Integer.MAX_VALUE); + ScoreDoc[] hits1 = searcher.search(query, collectorManager).scoreDocs; + collectorManager = TopScoreDocCollectorManager.create(topDocsToCheck, Integer.MAX_VALUE); + ScoreDoc[] hits2 = searcher.search(query, collectorManager).scoreDocs; CheckHits.checkHitsQuery(query, hits1, hits2, expDocNrs); // Since we have no deleted docs, we should also be able to verify identical matches & // scores against an single segment copy of our index - collector = TopScoreDocCollector.create(topDocsToCheck, Integer.MAX_VALUE); - singleSegmentSearcher.search(query, collector); - hits2 = collector.topDocs().scoreDocs; + collectorManager = TopScoreDocCollectorManager.create(topDocsToCheck, Integer.MAX_VALUE); + TopDocs topDocs2 = singleSegmentSearcher.search(query, collectorManager); + hits2 = topDocs2.scoreDocs; CheckHits.checkHitsQuery(query, hits1, hits2, expDocNrs); // sanity check expected num matches in bigSearcher - assertEquals(mulFactor * collector.totalHits, bigSearcher.count(query)); + assertEquals(mulFactor * topDocs2.totalHits.value, bigSearcher.count(query)); // now check 2 diff scorers from the bigSearcher as well - collector = TopScoreDocCollector.create(topDocsToCheck, Integer.MAX_VALUE); - bigSearcher.search(query, collector); - hits1 = collector.topDocs().scoreDocs; - collector = TopScoreDocCollector.create(topDocsToCheck, Integer.MAX_VALUE); - bigSearcher.search(query, collector); - hits2 = collector.topDocs().scoreDocs; + collectorManager = TopScoreDocCollectorManager.create(topDocsToCheck, Integer.MAX_VALUE); + hits1 = bigSearcher.search(query, collectorManager).scoreDocs; + collectorManager = TopScoreDocCollectorManager.create(topDocsToCheck, Integer.MAX_VALUE); + hits2 = bigSearcher.search(query, collectorManager).scoreDocs; // NOTE: just comparing results, not vetting against expDocNrs // since we have dups in bigSearcher @@ -387,27 +384,25 @@ public void testRandomQueries() throws Exception { } // check diff (randomized) scorers (from AssertingSearcher) produce the same results - TopFieldCollector collector = TopFieldCollector.create(sort, 1000, 1); - searcher.search(q1, collector); - ScoreDoc[] hits1 = collector.topDocs().scoreDocs; - collector = TopFieldCollector.create(sort, 1000, 1); - searcher.search(q1, collector); - ScoreDoc[] hits2 = collector.topDocs().scoreDocs; + TopFieldCollectorManager collectorManager = TopFieldCollectorManager.create(sort, 1000, 1); + ScoreDoc[] hits1 = searcher.search(q1, collectorManager).scoreDocs; + collectorManager = TopFieldCollectorManager.create(sort, 1000, 1); + TopDocs topDocs = searcher.search(q1, collectorManager); + ScoreDoc[] hits2 = topDocs.scoreDocs; CheckHits.checkEqual(q1, hits1, hits2); BooleanQuery.Builder q3 = new BooleanQuery.Builder(); q3.add(q1, BooleanClause.Occur.SHOULD); q3.add(new PrefixQuery(new Term("field2", "b")), BooleanClause.Occur.SHOULD); assertEquals( - mulFactor * collector.totalHits + NUM_EXTRA_DOCS / 2, bigSearcher.count(q3.build())); + mulFactor * topDocs.totalHits.value + NUM_EXTRA_DOCS / 2, + bigSearcher.count(q3.build())); // test diff (randomized) scorers produce the same results on bigSearcher as well - collector = TopFieldCollector.create(sort, 1000 * mulFactor, 1); - bigSearcher.search(q1, collector); - hits1 = collector.topDocs().scoreDocs; - collector = TopFieldCollector.create(sort, 1000 * mulFactor, 1); - bigSearcher.search(q1, collector); - hits2 = collector.topDocs().scoreDocs; + collectorManager = TopFieldCollectorManager.create(sort, 1000 * mulFactor, 1); + hits1 = bigSearcher.search(q1, collectorManager).scoreDocs; + collectorManager = TopFieldCollectorManager.create(sort, 1000 * mulFactor, 1); + hits2 = bigSearcher.search(q1, collectorManager).scoreDocs; CheckHits.checkEqual(q1, hits1, hits2); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java index fee63f3d60ed..131b10672b05 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java @@ -90,9 +90,10 @@ public void verifyNrHits(Query q, int expected) throws Exception { assertEquals("result count", expected, h.length); // System.out.println("TEST: now check"); // bs2 - TopScoreDocCollector collector = TopScoreDocCollector.create(1000, Integer.MAX_VALUE); - s.search(q, collector); - ScoreDoc[] h2 = collector.topDocs().scoreDocs; + TopScoreDocCollectorManager collectorManager = + TopScoreDocCollectorManager.create(1000, Integer.MAX_VALUE); + TopDocs topDocs = s.search(q, collectorManager); + ScoreDoc[] h2 = topDocs.scoreDocs; if (expected != h2.length) { printHits(getTestName(), h2, s); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java index fbbf51731a1d..e9e3b846d3b4 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java @@ -57,36 +57,25 @@ private void checkHits( final float expectedScore, final Class innerScorerClass) throws IOException { - final int[] count = new int[1]; - searcher.search( - q, - new SimpleCollector() { - private Scorable scorer; - - @Override - public void setScorer(Scorable scorer) { - this.scorer = scorer; - if (innerScorerClass != null) { - Scorable innerScorer = rootScorer(scorer); - assertEquals( - "inner Scorer is implemented by wrong class", - innerScorerClass, - innerScorer.getClass()); - } - } - - @Override - public void collect(int doc) throws IOException { - assertEquals("Score differs from expected", expectedScore, this.scorer.score(), 0); - count[0]++; - } - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE; - } - }); - assertEquals("invalid number of results", 1, count[0]); + int count = + searcher.search( + q, + new CollectorManager() { + @Override + public CountingSimpleCollector newCollector() { + return new CountingSimpleCollector(innerScorerClass, expectedScore); + } + + @Override + public Integer reduce(Collection collectors) { + return collectors.stream() + .map(CountingSimpleCollector::getCount) + .mapToInt(Integer::intValue) + .sum(); + } + }); + + assertEquals("invalid number of results", 1, count); } private Scorable rootScorer(Scorable s) { @@ -242,4 +231,43 @@ public void testPropagatesApproximations() throws IOException { w.close(); dir.close(); } + + private class CountingSimpleCollector extends SimpleCollector { + private final Class innerScorerClass; + private final float expectedScore; + private int count; + private Scorable scorer; + + public CountingSimpleCollector( + Class innerScorerClass, float expectedScore) { + this.innerScorerClass = innerScorerClass; + this.expectedScore = expectedScore; + count = 0; + } + + @Override + public void setScorer(Scorable scorer) { + this.scorer = scorer; + if (innerScorerClass != null) { + Scorable innerScorer = rootScorer(scorer); + assertEquals( + "inner Scorer is implemented by wrong class", innerScorerClass, innerScorer.getClass()); + } + } + + @Override + public void collect(int doc) throws IOException { + assertEquals("Score differs from expected", expectedScore, this.scorer.score(), 0); + count++; + } + + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE; + } + + public int getCount() { + return count; + } + } } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreScorer.java index d43aed3ac854..952c4d6a17a8 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreScorer.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreScorer.java @@ -240,20 +240,20 @@ public void testEarlyTermination() throws IOException { IndexSearcher is = newSearcher(ir); - TopScoreDocCollector c = TopScoreDocCollector.create(10, null, 10); - is.search(new ConstantScoreQuery(new TermQuery(new Term("key", "foo"))), c); - assertEquals(11, c.totalHits); - assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, c.totalHitsRelation); + TopScoreDocCollectorManager c = new TopScoreDocCollectorManager(10, null, 10); + TopDocs topDocs = is.search(new ConstantScoreQuery(new TermQuery(new Term("key", "foo"))), c); + assertEquals(11, topDocs.totalHits.value); + assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, topDocs.totalHits.relation); - c = TopScoreDocCollector.create(10, null, 10); + c = new TopScoreDocCollectorManager(10, null, 10); Query query = new BooleanQuery.Builder() .add(new ConstantScoreQuery(new TermQuery(new Term("key", "foo"))), Occur.SHOULD) .add(new ConstantScoreQuery(new TermQuery(new Term("key", "bar"))), Occur.FILTER) .build(); - is.search(query, c); - assertEquals(11, c.totalHits); - assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, c.totalHitsRelation); + topDocs = is.search(query, c); + assertEquals(11, topDocs.totalHits.value); + assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, topDocs.totalHits.relation); iw.close(); ir.close(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestFuzzyTermOnShortTerms.java b/lucene/core/src/test/org/apache/lucene/search/TestFuzzyTermOnShortTerms.java index b60d70c77ffc..57399120b2cd 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestFuzzyTermOnShortTerms.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestFuzzyTermOnShortTerms.java @@ -62,9 +62,8 @@ private void countHits(Analyzer analyzer, String[] docs, Query q, int expected) Directory d = getDirectory(analyzer, docs); IndexReader r = DirectoryReader.open(d); IndexSearcher s = new IndexSearcher(r); - TotalHitCountCollector c = new TotalHitCountCollector(); - s.search(q, c); - assertEquals(q.toString(), expected, c.getTotalHits()); + int totalHits = s.search(q, new TotalHitCountCollectorManager()); + assertEquals(q.toString(), expected, totalHits); r.close(); d.close(); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java index 37a7e5a406a6..95b08b09788b 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java @@ -118,17 +118,18 @@ public void testEarlyTermination() throws IOException { IndexSearcher is = newSearcher(ir); final int totalHitsThreshold = 200; - TopScoreDocCollector c = TopScoreDocCollector.create(10, null, totalHitsThreshold); + TopScoreDocCollectorManager collectorManager = + new TopScoreDocCollectorManager(10, null, totalHitsThreshold); - is.search(new MatchAllDocsQuery(), c); - assertEquals(totalHitsThreshold + 1, c.totalHits); - assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, c.totalHitsRelation); + TopDocs topDocs = is.search(new MatchAllDocsQuery(), collectorManager); + assertEquals(totalHitsThreshold + 1, topDocs.totalHits.value); + assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, topDocs.totalHits.relation); - TopScoreDocCollector c1 = TopScoreDocCollector.create(10, null, numDocs); + collectorManager = new TopScoreDocCollectorManager(10, null, numDocs); - is.search(new MatchAllDocsQuery(), c1); - assertEquals(numDocs, c1.totalHits); - assertEquals(TotalHits.Relation.EQUAL_TO, c1.totalHitsRelation); + topDocs = is.search(new MatchAllDocsQuery(), collectorManager); + assertEquals(numDocs, topDocs.totalHits.value); + assertEquals(TotalHits.Relation.EQUAL_TO, topDocs.totalHits.relation); iw.close(); ir.close(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java index 8deaacffc61c..66e4b25cc020 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java @@ -770,13 +770,16 @@ public void testTopPhrases() throws IOException { new PhraseQuery("f", "d", "d") // repeated term )) { for (int topN = 1; topN <= 2; ++topN) { - TopScoreDocCollector collector1 = - TopScoreDocCollector.create(topN, null, Integer.MAX_VALUE); - searcher.search(query, collector1); - ScoreDoc[] hits1 = collector1.topDocs().scoreDocs; - TopScoreDocCollector collector2 = TopScoreDocCollector.create(topN, null, 1); - searcher.search(query, collector2); - ScoreDoc[] hits2 = collector2.topDocs().scoreDocs; + TopScoreDocCollectorManager collectorManager1 = + new TopScoreDocCollectorManager(topN, null, Integer.MAX_VALUE); + TopDocs topDocs1 = searcher.search(query, collectorManager1); + ScoreDoc[] hits1 = topDocs1.scoreDocs; + + TopScoreDocCollectorManager collectorManager2 = + new TopScoreDocCollectorManager(topN, null, 1); + TopDocs topDocs2 = searcher.search(query, collectorManager2); + ScoreDoc[] hits2 = topDocs2.scoreDocs; + assertTrue("" + query, hits1.length > 0); CheckHits.checkEqual(query, hits1, hits2); } @@ -1027,13 +1030,15 @@ public void testRandomTopDocs() throws IOException { for (String secondTerm : new String[] {"a", "b", "c"}) { Query query = new PhraseQuery("foo", new BytesRef(firstTerm), new BytesRef(secondTerm)); - TopScoreDocCollector collector1 = - TopScoreDocCollector.create(10, null, Integer.MAX_VALUE); // COMPLETE - TopScoreDocCollector collector2 = TopScoreDocCollector.create(10, null, 10); // TOP_SCORES + TopScoreDocCollectorManager collectorManager1 = + new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE); // COMPLETE + TopDocs topDocs1 = searcher.search(query, collectorManager1); - searcher.search(query, collector1); - searcher.search(query, collector2); - CheckHits.checkEqual(query, collector1.topDocs().scoreDocs, collector2.topDocs().scoreDocs); + TopScoreDocCollectorManager collectorManager2 = + new TopScoreDocCollectorManager(10, null, 10); // TOP_SCORES + TopDocs topDocs2 = searcher.search(query, collectorManager2); + + CheckHits.checkEqual(query, topDocs1.scoreDocs, topDocs2.scoreDocs); Query filteredQuery = new BooleanQuery.Builder() @@ -1041,11 +1046,13 @@ public void testRandomTopDocs() throws IOException { .add(new TermQuery(new Term("foo", "b")), Occur.FILTER) .build(); - collector1 = TopScoreDocCollector.create(10, null, Integer.MAX_VALUE); // COMPLETE - collector2 = TopScoreDocCollector.create(10, null, 10); // TOP_SCORES - searcher.search(filteredQuery, collector1); - searcher.search(filteredQuery, collector2); - CheckHits.checkEqual(query, collector1.topDocs().scoreDocs, collector2.topDocs().scoreDocs); + collectorManager1 = + new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE); // COMPLETE + topDocs1 = searcher.search(filteredQuery, collectorManager1); + + collectorManager2 = new TopScoreDocCollectorManager(10, null, 10); // TOP_SCORES + topDocs2 = searcher.search(filteredQuery, collectorManager2); + CheckHits.checkEqual(query, topDocs1.scoreDocs, topDocs2.scoreDocs); } } reader.close(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java index 54134948ab04..62341d9ea3aa 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java @@ -52,7 +52,6 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.PointValues; @@ -62,6 +61,8 @@ import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.FixedBitSetCollector; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.NumericUtils; @@ -572,28 +573,7 @@ private void _run() throws Exception { System.out.println(Thread.currentThread().getName() + ": using query: " + query); } - final BitSet hits = new BitSet(); - s.search( - query, - new SimpleCollector() { - - private int docBase; - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE_NO_SCORES; - } - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - docBase = context.docBase; - } - - @Override - public void collect(int doc) { - hits.set(docBase + doc); - } - }); + final FixedBitSet hits = s.search(query, FixedBitSetCollector.create(r.maxDoc())); if (VERBOSE) { System.out.println( @@ -870,28 +850,7 @@ private void _run() throws Exception { System.out.println(Thread.currentThread().getName() + ": using query: " + query); } - final BitSet hits = new BitSet(); - s.search( - query, - new SimpleCollector() { - - private int docBase; - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE_NO_SCORES; - } - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - docBase = context.docBase; - } - - @Override - public void collect(int doc) { - hits.set(docBase + doc); - } - }); + final FixedBitSet hits = s.search(query, FixedBitSetCollector.create(r.maxDoc())); if (VERBOSE) { System.out.println( diff --git a/lucene/core/src/test/org/apache/lucene/search/TestReqOptSumScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestReqOptSumScorer.java index 677b83ab3f7d..2e9ed9929b47 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestReqOptSumScorer.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestReqOptSumScorer.java @@ -263,9 +263,10 @@ private void doTestRandom(double optFreq) throws IOException { Query query = new BooleanQuery.Builder().add(mustTerm, Occur.MUST).add(shouldTerm, Occur.SHOULD).build(); - TopScoreDocCollector coll = TopScoreDocCollector.create(10, null, Integer.MAX_VALUE); - searcher.search(query, coll); - ScoreDoc[] expected = coll.topDocs().scoreDocs; + TopScoreDocCollectorManager collectorManager = + new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE); + TopDocs topDocs = searcher.search(query, collectorManager); + ScoreDoc[] expected = topDocs.scoreDocs; // Also test a filtered query, since it does not compute the score on all // matches. @@ -275,9 +276,9 @@ private void doTestRandom(double optFreq) throws IOException { .add(new TermQuery(new Term("f", "C")), Occur.FILTER) .build(); - coll = TopScoreDocCollector.create(10, null, Integer.MAX_VALUE); - searcher.search(query, coll); - ScoreDoc[] expectedFiltered = coll.topDocs().scoreDocs; + collectorManager = new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE); + topDocs = searcher.search(query, collectorManager); + ScoreDoc[] expectedFiltered = topDocs.scoreDocs; CheckHits.checkTopScores(random(), query, searcher); @@ -288,9 +289,9 @@ private void doTestRandom(double optFreq) throws IOException { .add(shouldTerm, Occur.SHOULD) .build(); - coll = TopScoreDocCollector.create(10, null, 1); - searcher.search(q, coll); - ScoreDoc[] actual = coll.topDocs().scoreDocs; + collectorManager = new TopScoreDocCollectorManager(10, null, 1); + topDocs = searcher.search(q, collectorManager); + ScoreDoc[] actual = topDocs.scoreDocs; CheckHits.checkEqual(query, expected, actual); q = @@ -298,9 +299,9 @@ private void doTestRandom(double optFreq) throws IOException { .add(mustTerm, Occur.MUST) .add(new RandomApproximationQuery(shouldTerm, random()), Occur.SHOULD) .build(); - coll = TopScoreDocCollector.create(10, null, 1); - searcher.search(q, coll); - actual = coll.topDocs().scoreDocs; + collectorManager = new TopScoreDocCollectorManager(10, null, 1); + topDocs = searcher.search(q, collectorManager); + actual = topDocs.scoreDocs; CheckHits.checkEqual(q, expected, actual); q = @@ -308,9 +309,9 @@ private void doTestRandom(double optFreq) throws IOException { .add(new RandomApproximationQuery(mustTerm, random()), Occur.MUST) .add(new RandomApproximationQuery(shouldTerm, random()), Occur.SHOULD) .build(); - coll = TopScoreDocCollector.create(10, null, 1); - searcher.search(q, coll); - actual = coll.topDocs().scoreDocs; + collectorManager = new TopScoreDocCollectorManager(10, null, 1); + topDocs = searcher.search(q, collectorManager); + actual = topDocs.scoreDocs; CheckHits.checkEqual(q, expected, actual); } @@ -330,9 +331,9 @@ private void doTestRandom(double optFreq) throws IOException { Occur.FILTER) .build(); - coll = TopScoreDocCollector.create(10, null, 1); - searcher.search(nestedQ, coll); - ScoreDoc[] actualFiltered = coll.topDocs().scoreDocs; + collectorManager = new TopScoreDocCollectorManager(10, null, 1); + topDocs = searcher.search(nestedQ, collectorManager); + ScoreDoc[] actualFiltered = topDocs.scoreDocs; CheckHits.checkEqual(nestedQ, expectedFiltered, actualFiltered); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSearchWithThreads.java b/lucene/core/src/test/org/apache/lucene/search/TestSearchWithThreads.java index 901f8761b39d..3e86561ada06 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSearchWithThreads.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSearchWithThreads.java @@ -54,6 +54,7 @@ public void test() throws Exception { w.close(); final IndexSearcher s = newSearcher(r); + final TotalHitCountCollectorManager collectorManager = new TotalHitCountCollectorManager(); final AtomicBoolean failed = new AtomicBoolean(); final AtomicLong netSearch = new AtomicLong(); @@ -62,18 +63,14 @@ public void test() throws Exception { for (int threadID = 0; threadID < numThreads; threadID++) { threads[threadID] = new Thread() { - TotalHitCountCollector col = new TotalHitCountCollector(); - @Override public void run() { try { long totHits = 0; long totSearch = 0; for (; totSearch < numSearches & !failed.get(); totSearch++) { - s.search(new TermQuery(new Term("body", "aaa")), col); - totHits += col.getTotalHits(); - s.search(new TermQuery(new Term("body", "bbb")), col); - totHits += col.getTotalHits(); + totHits += s.search(new TermQuery(new Term("body", "aaa")), collectorManager); + totHits += s.search(new TermQuery(new Term("body", "bbb")), collectorManager); } assertTrue(totSearch > 0 && totHits > 0); netSearch.addAndGet(totSearch); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java b/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java index 88d53d281e05..31d86865abc5 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java @@ -60,41 +60,42 @@ public void testLongSortOptimization() throws IOException { final int totalHitsThreshold = 3; { // simple sort - final TopFieldCollector collector = - TopFieldCollector.create(sort, numHits, null, totalHitsThreshold); - searcher.search(new MatchAllDocsQuery(), collector); - TopDocs topDocs = collector.topDocs(); + final TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); + + TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); + assertEquals(topDocs.scoreDocs.length, numHits); for (int i = 0; i < numHits; i++) { FieldDoc fieldDoc = (FieldDoc) topDocs.scoreDocs[i]; assertEquals(i, ((Long) fieldDoc.fields[0]).intValue()); } - assertTrue(collector.isEarlyTerminated()); + assertTrue( + collectorManager.getCollectors().stream().anyMatch(TopFieldCollector::isEarlyTerminated)); assertTrue(topDocs.totalHits.value < numDocs); } { // paging sort with after long afterValue = 2; FieldDoc after = new FieldDoc(2, Float.NaN, new Long[] {afterValue}); - final TopFieldCollector collector = - TopFieldCollector.create(sort, numHits, after, totalHitsThreshold); - searcher.search(new MatchAllDocsQuery(), collector); - TopDocs topDocs = collector.topDocs(); + final TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, numHits, after, totalHitsThreshold); + TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); assertEquals(topDocs.scoreDocs.length, numHits); for (int i = 0; i < numHits; i++) { FieldDoc fieldDoc = (FieldDoc) topDocs.scoreDocs[i]; assertEquals(afterValue + 1 + i, fieldDoc.fields[0]); } - assertTrue(collector.isEarlyTerminated()); + assertTrue( + collectorManager.getCollectors().stream().anyMatch(TopFieldCollector::isEarlyTerminated)); assertTrue(topDocs.totalHits.value < numDocs); } { // test that if there is the secondary sort on _score, scores are filled correctly - final TopFieldCollector collector = - TopFieldCollector.create( + final TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager( new Sort(sortField, FIELD_SCORE), numHits, null, totalHitsThreshold); - searcher.search(new MatchAllDocsQuery(), collector); - TopDocs topDocs = collector.topDocs(); + TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); assertEquals(topDocs.scoreDocs.length, numHits); for (int i = 0; i < numHits; i++) { FieldDoc fieldDoc = (FieldDoc) topDocs.scoreDocs[i]; @@ -102,16 +103,16 @@ public void testLongSortOptimization() throws IOException { float score = (float) fieldDoc.fields[1]; assertEquals(1.0, score, 0.001); } - assertTrue(collector.isEarlyTerminated()); + assertTrue( + collectorManager.getCollectors().stream().anyMatch(TopFieldCollector::isEarlyTerminated)); assertTrue(topDocs.totalHits.value < numDocs); } { // test that if numeric field is a secondary sort, no optimization is run - final TopFieldCollector collector = - TopFieldCollector.create( + final TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager( new Sort(FIELD_SCORE, sortField), numHits, null, totalHitsThreshold); - searcher.search(new MatchAllDocsQuery(), collector); - TopDocs topDocs = collector.topDocs(); + TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); assertEquals(topDocs.scoreDocs.length, numHits); assertEquals( topDocs.totalHits.value, @@ -144,10 +145,9 @@ public void testLongSortOptimizationOnFieldNotIndexedWithPoints() throws IOExcep final int numHits = 3; final int totalHitsThreshold = 3; - final TopFieldCollector collector = - TopFieldCollector.create(sort, numHits, null, totalHitsThreshold); - searcher.search(new MatchAllDocsQuery(), collector); - TopDocs topDocs = collector.topDocs(); + final TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); + TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); assertEquals( topDocs.scoreDocs.length, numHits); // sort still works and returns expected number of docs for (int i = 0; i < numHits; i++) { @@ -185,10 +185,9 @@ public void testSortOptimizationWithMissingValues() throws IOException { final SortField sortField = new SortField("my_field", SortField.Type.LONG); sortField.setMissingValue(0L); // set a competitive missing value final Sort sort = new Sort(sortField); - final TopFieldCollector collector = - TopFieldCollector.create(sort, numHits, null, totalHitsThreshold); - searcher.search(new MatchAllDocsQuery(), collector); - TopDocs topDocs = collector.topDocs(); + final TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); + TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); assertEquals(topDocs.scoreDocs.length, numHits); assertEquals( topDocs.totalHits.value, @@ -198,10 +197,9 @@ public void testSortOptimizationWithMissingValues() throws IOException { final SortField sortField = new SortField("my_field", SortField.Type.LONG); sortField.setMissingValue(100L); // set a NON competitive missing value final Sort sort = new Sort(sortField); - final TopFieldCollector collector = - TopFieldCollector.create(sort, numHits, null, totalHitsThreshold); - searcher.search(new MatchAllDocsQuery(), collector); - TopDocs topDocs = collector.topDocs(); + final TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); + TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); assertEquals(topDocs.scoreDocs.length, numHits); assertTrue( topDocs.totalHits.value @@ -236,10 +234,9 @@ public void testSortOptimizationEqualValues() throws IOException { { // test that sorting on a single field with equal values uses the optimization final SortField sortField = new SortField("my_field1", SortField.Type.INT); final Sort sort = new Sort(sortField); - final TopFieldCollector collector = - TopFieldCollector.create(sort, numHits, null, totalHitsThreshold); - searcher.search(new MatchAllDocsQuery(), collector); - TopDocs topDocs = collector.topDocs(); + final TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); + TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); assertEquals(topDocs.scoreDocs.length, numHits); for (int i = 0; i < numHits; i++) { FieldDoc fieldDoc = (FieldDoc) topDocs.scoreDocs[i]; @@ -256,10 +253,9 @@ public void testSortOptimizationEqualValues() throws IOException { final SortField sortField = new SortField("my_field1", SortField.Type.INT); final Sort sort = new Sort(sortField); FieldDoc after = new FieldDoc(10, Float.NaN, new Integer[] {afterValue}); - final TopFieldCollector collector = - TopFieldCollector.create(sort, numHits, after, totalHitsThreshold); - searcher.search(new MatchAllDocsQuery(), collector); - TopDocs topDocs = collector.topDocs(); + final TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, numHits, after, totalHitsThreshold); + TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); assertEquals(topDocs.scoreDocs.length, numHits); for (int i = 0; i < numHits; i++) { FieldDoc fieldDoc = (FieldDoc) topDocs.scoreDocs[i]; @@ -275,10 +271,9 @@ public void testSortOptimizationEqualValues() throws IOException { final SortField sortField1 = new SortField("my_field1", SortField.Type.INT); final SortField sortField2 = new SortField("my_field2", SortField.Type.INT); final Sort sort = new Sort(sortField1, sortField2); - final TopFieldCollector collector = - TopFieldCollector.create(sort, numHits, null, totalHitsThreshold); - searcher.search(new MatchAllDocsQuery(), collector); - TopDocs topDocs = collector.topDocs(); + final TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); + TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); assertEquals(topDocs.scoreDocs.length, numHits); for (int i = 0; i < numHits; i++) { FieldDoc fieldDoc = (FieldDoc) topDocs.scoreDocs[i]; @@ -315,16 +310,17 @@ public void testFloatSortOptimization() throws IOException { final int totalHitsThreshold = 3; { // simple sort - final TopFieldCollector collector = - TopFieldCollector.create(sort, numHits, null, totalHitsThreshold); - searcher.search(new MatchAllDocsQuery(), collector); - TopDocs topDocs = collector.topDocs(); + final TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); + + TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); assertEquals(topDocs.scoreDocs.length, numHits); for (int i = 0; i < numHits; i++) { FieldDoc fieldDoc = (FieldDoc) topDocs.scoreDocs[i]; assertEquals(1f * i, fieldDoc.fields[0]); } - assertTrue(collector.isEarlyTerminated()); + assertTrue( + collectorManager.getCollectors().stream().anyMatch(TopFieldCollector::isEarlyTerminated)); assertTrue(topDocs.totalHits.value < numDocs); } @@ -366,10 +362,9 @@ public void testDocSortOptimizationMultipleIndices() throws IOException { do { for (int i = 0; i < numIndices; i++) { IndexSearcher searcher = newSearcher(readers[i]); - final TopFieldCollector collector = - TopFieldCollector.create(sort, size, after, totalHitsThreshold); - searcher.search(new MatchAllDocsQuery(), collector); - topDocs[i] = collector.topDocs(); + final TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, size, after, totalHitsThreshold); + topDocs[i] = searcher.search(new MatchAllDocsQuery(), collectorManager); for (int docID = 0; docID < topDocs[i].scoreDocs.length; docID++) { topDocs[i].scoreDocs[docID].shardIndex = i; } @@ -418,10 +413,9 @@ public void testDocSortOptimizationWithAfter() throws IOException { { final Sort sort = new Sort(FIELD_DOC); FieldDoc after = new FieldDoc(searchAfter, Float.NaN, new Integer[] {searchAfter}); - final TopFieldCollector collector = - TopFieldCollector.create(sort, numHits, after, totalHitsThreshold); - searcher.search(new MatchAllDocsQuery(), collector); - TopDocs topDocs = collector.topDocs(); + final TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, numHits, after, totalHitsThreshold); + TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); int expNumHits = (searchAfter >= (numDocs - numHits)) ? (numDocs - searchAfter - 1) : numHits; assertEquals(expNumHits, topDocs.scoreDocs.length); @@ -429,7 +423,9 @@ public void testDocSortOptimizationWithAfter() throws IOException { int expectedDocID = searchAfter + 1 + i; assertEquals(expectedDocID, topDocs.scoreDocs[i].doc); } - assertTrue(collector.isEarlyTerminated()); + assertTrue( + collectorManager.getCollectors().stream() + .anyMatch(TopFieldCollector::isEarlyTerminated)); // check that very few docs were collected assertTrue(topDocs.totalHits.value < numDocs); } @@ -438,10 +434,9 @@ public void testDocSortOptimizationWithAfter() throws IOException { { final Sort sort = new Sort(FIELD_DOC, FIELD_SCORE); FieldDoc after = new FieldDoc(searchAfter, Float.NaN, new Object[] {searchAfter, 1.0f}); - final TopFieldCollector collector = - TopFieldCollector.create(sort, numHits, after, totalHitsThreshold); - searcher.search(new MatchAllDocsQuery(), collector); - TopDocs topDocs = collector.topDocs(); + final TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, numHits, after, totalHitsThreshold); + TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); int expNumHits = (searchAfter >= (numDocs - numHits)) ? (numDocs - searchAfter - 1) : numHits; assertEquals(expNumHits, topDocs.scoreDocs.length); @@ -449,7 +444,9 @@ public void testDocSortOptimizationWithAfter() throws IOException { int expectedDocID = searchAfter + 1 + i; assertEquals(expectedDocID, topDocs.scoreDocs[i].doc); } - assertTrue(collector.isEarlyTerminated()); + assertTrue( + collectorManager.getCollectors().stream() + .anyMatch(TopFieldCollector::isEarlyTerminated)); // assert that very few docs were collected assertTrue(topDocs.totalHits.value < numDocs); } @@ -458,10 +455,9 @@ public void testDocSortOptimizationWithAfter() throws IOException { { final Sort sort = new Sort(new SortField(null, SortField.Type.DOC, true)); FieldDoc after = new FieldDoc(searchAfter, Float.NaN, new Integer[] {searchAfter}); - final TopFieldCollector collector = - TopFieldCollector.create(sort, numHits, after, totalHitsThreshold); - searcher.search(new MatchAllDocsQuery(), collector); - TopDocs topDocs = collector.topDocs(); + final TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, numHits, after, totalHitsThreshold); + TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); int expNumHits = (searchAfter < numHits) ? searchAfter : numHits; assertEquals(expNumHits, topDocs.scoreDocs.length); for (int i = 0; i < topDocs.scoreDocs.length; i++) { @@ -502,38 +498,38 @@ public void testDocSortOptimization() throws IOException { // sort by _doc should skip all non-competitive documents { - final TopFieldCollector collector = - TopFieldCollector.create(sort, numHits, null, totalHitsThreshold); + final TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); IndexSearcher searcher = newSearcher(reader); - searcher.search(new MatchAllDocsQuery(), collector); - TopDocs topDocs = collector.topDocs(); + TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); assertEquals(numHits, topDocs.scoreDocs.length); for (int i = 0; i < numHits; i++) { assertEquals(i, topDocs.scoreDocs[i].doc); } - assertTrue(collector.isEarlyTerminated()); + assertTrue( + collectorManager.getCollectors().stream().anyMatch(TopFieldCollector::isEarlyTerminated)); assertTrue(topDocs.totalHits.value < 10); // assert that very few docs were collected } // sort by _doc with a bool query should skip all non-competitive documents { - final TopFieldCollector collector = - TopFieldCollector.create(sort, numHits, null, totalHitsThreshold); + final TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); int lowerRange = 40; BooleanQuery.Builder bq = new BooleanQuery.Builder(); bq.add(LongPoint.newRangeQuery("lf", lowerRange, Long.MAX_VALUE), BooleanClause.Occur.MUST); bq.add(new TermQuery(new Term("tf", "seg1")), BooleanClause.Occur.MUST); IndexSearcher searcher = newSearcher(reader); - searcher.search(bq.build(), collector); - TopDocs topDocs = collector.topDocs(); + TopDocs topDocs = searcher.search(bq.build(), collectorManager); assertEquals(numHits, topDocs.scoreDocs.length); for (int i = 0; i < numHits; i++) { Document d = searcher.doc(topDocs.scoreDocs[i].doc); assertEquals(Integer.toString(i + lowerRange), d.get("slf")); assertEquals("seg1", d.get("tf")); } - assertTrue(collector.isEarlyTerminated()); + assertTrue( + collectorManager.getCollectors().stream().anyMatch(TopFieldCollector::isEarlyTerminated)); assertTrue(topDocs.totalHits.value < 10); // assert that very few docs were collected } @@ -570,13 +566,12 @@ public void testDocSort() throws IOException { final Sort sort = new Sort(FIELD_DOC); { - final TopFieldCollector collector = - TopFieldCollector.create(sort, numHits, null, totalHitsThreshold); + final TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); BooleanQuery.Builder bq = new BooleanQuery.Builder(); bq.add(LongPoint.newExactQuery("lf", 1), BooleanClause.Occur.MUST); bq.add(new TermQuery(new Term("id", "id3")), BooleanClause.Occur.MUST_NOT); - searcher.search(bq.build(), collector); - TopDocs topDocs = collector.topDocs(); + TopDocs topDocs = searcher.search(bq.build(), collectorManager); assertEquals(2, topDocs.scoreDocs.length); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSynonymQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestSynonymQuery.java index 1772e248cdaa..f3f6471ecf0d 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSynonymQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSynonymQuery.java @@ -165,11 +165,10 @@ private void doTestScores(int totalHitsThreshold) throws IOException { .addTerm(new Term("f", "b"), boost == 0 ? 1f : boost) .build(); - TopScoreDocCollector collector = - TopScoreDocCollector.create( + TopScoreDocCollectorManager collectorManager = + new TopScoreDocCollectorManager( Math.min(reader.numDocs(), totalHitsThreshold), null, totalHitsThreshold); - searcher.search(query, collector); - TopDocs topDocs = collector.topDocs(); + TopDocs topDocs = searcher.search(query, collectorManager); if (topDocs.totalHits.value < totalHitsThreshold) { assertEquals(new TotalHits(11, TotalHits.Relation.EQUAL_TO), topDocs.totalHits); } else { @@ -225,11 +224,10 @@ public void doTestBoosts(int totalHitsThreshold) throws IOException { .addTerm(new Term("f", "c")) .build(); - TopScoreDocCollector collector = - TopScoreDocCollector.create( + TopScoreDocCollectorManager collectorManager = + new TopScoreDocCollectorManager( Math.min(reader.numDocs(), totalHitsThreshold), null, totalHitsThreshold); - searcher.search(query, collector); - TopDocs topDocs = collector.topDocs(); + TopDocs topDocs = searcher.search(query, collectorManager); if (topDocs.totalHits.value < totalHitsThreshold) { assertEquals(TotalHits.Relation.EQUAL_TO, topDocs.totalHits.relation); assertEquals(22, topDocs.totalHits.value); @@ -443,13 +441,14 @@ public void testRandomTopDocs() throws IOException { .addTerm(new Term("foo", Integer.toString(term2)), boost2) .build(); - TopScoreDocCollector collector1 = - TopScoreDocCollector.create(10, null, Integer.MAX_VALUE); // COMPLETE - TopScoreDocCollector collector2 = TopScoreDocCollector.create(10, null, 1); // TOP_SCORES + TopScoreDocCollectorManager collectorManager1 = + new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE); // COMPLETE + TopScoreDocCollectorManager collectorManager2 = + new TopScoreDocCollectorManager(10, null, 1); // TOP_SCORES - searcher.search(query, collector1); - searcher.search(query, collector2); - CheckHits.checkEqual(query, collector1.topDocs().scoreDocs, collector2.topDocs().scoreDocs); + TopDocs topDocs1 = searcher.search(query, collectorManager1); + TopDocs topDocs2 = searcher.search(query, collectorManager2); + CheckHits.checkEqual(query, topDocs1.scoreDocs, topDocs2.scoreDocs); int filterTerm = random().nextInt(15); Query filteredQuery = @@ -458,11 +457,12 @@ public void testRandomTopDocs() throws IOException { .add(new TermQuery(new Term("foo", Integer.toString(filterTerm))), Occur.FILTER) .build(); - collector1 = TopScoreDocCollector.create(10, null, Integer.MAX_VALUE); // COMPLETE - collector2 = TopScoreDocCollector.create(10, null, 1); // TOP_SCORES - searcher.search(filteredQuery, collector1); - searcher.search(filteredQuery, collector2); - CheckHits.checkEqual(query, collector1.topDocs().scoreDocs, collector2.topDocs().scoreDocs); + collectorManager1 = new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE); // COMPLETE + collectorManager2 = new TopScoreDocCollectorManager(10, null, 1); // TOP_SCORES + + topDocs1 = searcher.search(filteredQuery, collectorManager1); + topDocs2 = searcher.search(filteredQuery, collectorManager2); + CheckHits.checkEqual(query, topDocs1.scoreDocs, topDocs2.scoreDocs); } reader.close(); dir.close(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java index 32437a17a484..14a3daa5cc44 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java @@ -89,16 +89,18 @@ public void testCreateWeightDoesNotSeekIfScoresAreNotNeeded() throws IOException IndexSearcher searcher = new IndexSearcher(reader); // use a collector rather than searcher.count() which would just read the // doc freq instead of creating a scorer - TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(query, collector); - assertEquals(1, collector.getTotalHits()); + TotalHitCountCollectorManager collectorManager = new TotalHitCountCollectorManager(); + int totalHits = searcher.search(query, collectorManager); + assertEquals(1, totalHits); + TermQuery queryWithContext = new TermQuery( new Term("foo", "bar"), TermStates.build(reader.getContext(), new Term("foo", "bar"), true)); - collector = new TotalHitCountCollector(); - searcher.search(queryWithContext, collector); - assertEquals(1, collector.getTotalHits()); + + collectorManager = new TotalHitCountCollectorManager(); + totalHits = searcher.search(queryWithContext, collectorManager); + assertEquals(1, totalHits); IOUtils.close(reader, w, dir); } @@ -207,5 +209,4 @@ public CacheHelper getReaderCacheHelper() { return in.getReaderCacheHelper(); } } - ; } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java index 841613f0156a..11864ae4403c 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java @@ -241,13 +241,14 @@ public void testRandomTopDocs() throws IOException { for (int iter = 0; iter < 15; ++iter) { Query query = new TermQuery(new Term("foo", Integer.toString(iter))); - TopScoreDocCollector collector1 = - TopScoreDocCollector.create(10, null, Integer.MAX_VALUE); // COMPLETE - TopScoreDocCollector collector2 = TopScoreDocCollector.create(10, null, 1); // TOP_SCORES + TopScoreDocCollectorManager collectorManager1 = + new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE); // COMPLETE + TopScoreDocCollectorManager collectorManager2 = + new TopScoreDocCollectorManager(10, null, 1); // TOP_SCORES - searcher.search(query, collector1); - searcher.search(query, collector2); - CheckHits.checkEqual(query, collector1.topDocs().scoreDocs, collector2.topDocs().scoreDocs); + TopDocs topDocs1 = searcher.search(query, collectorManager1); + TopDocs topDocs2 = searcher.search(query, collectorManager2); + CheckHits.checkEqual(query, topDocs1.scoreDocs, topDocs2.scoreDocs); int filterTerm = random().nextInt(15); Query filteredQuery = @@ -256,11 +257,11 @@ public void testRandomTopDocs() throws IOException { .add(new TermQuery(new Term("foo", Integer.toString(filterTerm))), Occur.FILTER) .build(); - collector1 = TopScoreDocCollector.create(10, null, Integer.MAX_VALUE); // COMPLETE - collector2 = TopScoreDocCollector.create(10, null, 1); // TOP_SCORES - searcher.search(filteredQuery, collector1); - searcher.search(filteredQuery, collector2); - CheckHits.checkEqual(query, collector1.topDocs().scoreDocs, collector2.topDocs().scoreDocs); + collectorManager1 = new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE); // COMPLETE + collectorManager2 = new TopScoreDocCollectorManager(10, null, 1); // TOP_SCORES + topDocs1 = searcher.search(filteredQuery, collectorManager1); + topDocs2 = searcher.search(filteredQuery, collectorManager2); + CheckHits.checkEqual(query, topDocs1.scoreDocs, topDocs2.scoreDocs); } reader.close(); dir.close(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java index 35b5dbd6fa28..8ad9d702d625 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Collection; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -134,17 +135,32 @@ private TopDocsCollector doSearch(int numResults) throws IOException { private TopDocsCollector doSearch(int numResults, Query q) throws IOException { IndexSearcher searcher = newSearcher(reader); - TopDocsCollector tdc = new MyTopsDocCollector(numResults); - searcher.search(q, tdc); + TopDocsCollector tdc = + searcher.search( + q, + new CollectorManager, TopDocsCollector>() { + @Override + public TopDocsCollector newCollector() throws IOException { + return new MyTopsDocCollector(numResults); + } + + @Override + public TopDocsCollector reduce( + Collection> collectors) { + assert collectors.size() == 1 + : "there should be only one instance of collector created for test"; + return collectors.iterator().next(); + } + }); return tdc; } - private TopDocsCollector doSearchWithThreshold( + private TopDocs doSearchWithThreshold( int numResults, int thresHold, Query q, IndexReader indexReader) throws IOException { IndexSearcher searcher = new IndexSearcher(indexReader); - TopDocsCollector tdc = TopScoreDocCollector.create(numResults, thresHold); - searcher.search(q, tdc); - return tdc; + TopScoreDocCollectorManager collectorManager = + TopScoreDocCollectorManager.create(numResults, thresHold); + return searcher.search(q, collectorManager); } private TopDocs doConcurrentSearchWithThreshold( @@ -160,8 +176,8 @@ private TopDocs doConcurrentSearchWithThreshold( try { IndexSearcher searcher = new IndexSearcher(indexReader, service); - CollectorManager collectorManager = - TopScoreDocCollector.createSharedManager(numResults, null, threshold); + TopScoreDocCollectorManager collectorManager = + new TopScoreDocCollectorManager(numResults, null, threshold); return searcher.search(q, collectorManager); } finally { @@ -331,8 +347,9 @@ public void testSetMinCompetitiveScore() throws Exception { assertEquals(2, reader.leaves().size()); w.close(); - TopScoreDocCollector collector = TopScoreDocCollector.create(2, null, 2); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, null, 2); ScoreAndDoc scorer = new ScoreAndDoc(); + TopScoreDocCollector collector = collectorManager.newCollector(); LeafCollector leafCollector = collector.getLeafCollector(reader.leaves().get(0)); leafCollector.setScorer(scorer); @@ -399,9 +416,8 @@ public void testSharedCountCollectorManager() throws Exception { assertEquals(2, reader.leaves().size()); w.close(); - TopDocsCollector collector = doSearchWithThreshold(5, 10, q, reader); TopDocs tdc = doConcurrentSearchWithThreshold(5, 10, q, reader); - TopDocs tdc2 = collector.topDocs(); + TopDocs tdc2 = doSearchWithThreshold(5, 10, q, reader); CheckHits.checkEqual(q, tdc.scoreDocs, tdc2.scoreDocs); @@ -423,7 +439,9 @@ public void testTotalHits() throws Exception { w.close(); for (int totalHitsThreshold = 0; totalHitsThreshold < 20; ++totalHitsThreshold) { - TopScoreDocCollector collector = TopScoreDocCollector.create(2, null, totalHitsThreshold); + TopScoreDocCollectorManager collectorManager = + new TopScoreDocCollectorManager(2, null, totalHitsThreshold); + TopScoreDocCollector collector = collectorManager.newCollector(); ScoreAndDoc scorer = new ScoreAndDoc(); LeafCollector leafCollector = collector.getLeafCollector(reader.leaves().get(0)); @@ -477,20 +495,20 @@ public void testRelationVsTopDocsCount() throws Exception { try (IndexReader reader = DirectoryReader.open(w)) { IndexSearcher searcher = new IndexSearcher(reader); - TopScoreDocCollector collector = TopScoreDocCollector.create(2, null, 10); - searcher.search(new TermQuery(new Term("f", "foo")), collector); - assertEquals(10, collector.totalHits); - assertEquals(TotalHits.Relation.EQUAL_TO, collector.totalHitsRelation); - - collector = TopScoreDocCollector.create(2, null, 2); - searcher.search(new TermQuery(new Term("f", "foo")), collector); - assertTrue(10 >= collector.totalHits); - assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, collector.totalHitsRelation); - - collector = TopScoreDocCollector.create(10, null, 2); - searcher.search(new TermQuery(new Term("f", "foo")), collector); - assertEquals(10, collector.totalHits); - assertEquals(TotalHits.Relation.EQUAL_TO, collector.totalHitsRelation); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, null, 10); + TopDocs topDocs = searcher.search(new TermQuery(new Term("f", "foo")), collectorManager); + assertEquals(10, topDocs.totalHits.value); + assertEquals(TotalHits.Relation.EQUAL_TO, topDocs.totalHits.relation); + + collectorManager = new TopScoreDocCollectorManager(2, null, 2); + topDocs = searcher.search(new TermQuery(new Term("f", "foo")), collectorManager); + assertTrue(10 >= topDocs.totalHits.value); + assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, topDocs.totalHits.relation); + + collectorManager = new TopScoreDocCollectorManager(10, null, 2); + topDocs = searcher.search(new TermQuery(new Term("f", "foo")), collectorManager); + assertEquals(10, topDocs.totalHits.value); + assertEquals(TotalHits.Relation.EQUAL_TO, topDocs.totalHits.relation); } } } @@ -511,7 +529,7 @@ public void testConcurrentMinScore() throws Exception { w.close(); CollectorManager manager = - TopScoreDocCollector.createSharedManager(2, null, 0); + new TopScoreDocCollectorManager(2, null, 0); TopScoreDocCollector collector = manager.newCollector(); TopScoreDocCollector collector2 = manager.newCollector(); assertTrue(collector.minScoreAcc == collector2.minScoreAcc); @@ -650,9 +668,8 @@ public void testRandomMinCompetitiveScore() throws Exception { .build() }; for (Query query : queries) { - TopDocsCollector collector = doSearchWithThreshold(5, 0, query, indexReader); TopDocs tdc = doConcurrentSearchWithThreshold(5, 0, query, indexReader); - TopDocs tdc2 = collector.topDocs(); + TopDocs tdc2 = doSearchWithThreshold(5, 0, query, indexReader); assertTrue(tdc.totalHits.value > 0); assertTrue(tdc2.totalHits.value > 0); @@ -692,9 +709,8 @@ public void testRealisticConcurrentMinimumScore() throws Exception { BytesRef term = BytesRef.deepCopyOf(termsEnum.term()); Query query = new TermQuery(new Term("body", term)); - TopDocsCollector collector = doSearchWithThreshold(5, 0, query, reader); TopDocs tdc = doConcurrentSearchWithThreshold(5, 0, query, reader); - TopDocs tdc2 = collector.topDocs(); + TopDocs tdc2 = doSearchWithThreshold(5, 0, query, reader); CheckHits.checkEqual(query, tdc.scoreDocs, tdc2.scoreDocs); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java index 8419caa7a4c7..524ef989ed43 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Collection; import java.util.Comparator; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; @@ -72,13 +73,12 @@ public void tearDown() throws Exception { super.tearDown(); } - private TopFieldCollector doSearchWithThreshold( + private TopFieldDocs doSearchWithThreshold( int numResults, int thresHold, Query q, Sort sort, IndexReader indexReader) throws IOException { IndexSearcher searcher = new IndexSearcher(indexReader); - TopFieldCollector tdc = TopFieldCollector.create(sort, numResults, thresHold); - searcher.search(q, tdc); - return tdc; + TopFieldCollectorManager tdc = TopFieldCollectorManager.create(sort, numResults, thresHold); + return searcher.search(q, tdc); } private TopDocs doConcurrentSearchWithThreshold( @@ -95,12 +95,12 @@ private TopDocs doConcurrentSearchWithThreshold( try { IndexSearcher searcher = new IndexSearcher(indexReader, service); - CollectorManager collectorManager = - TopFieldCollector.createSharedManager(sort, numResults, null, threshold); + TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, numResults, null, threshold); - TopDocs tdc = searcher.search(q, collectorManager); + TopDocs topDoc = searcher.search(q, collectorManager); - return tdc; + return topDoc; } finally { service.shutdown(); } @@ -116,11 +116,10 @@ public void testSortWithoutFillFields() throws Exception { Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort()}; for (int i = 0; i < sort.length; i++) { Query q = new MatchAllDocsQuery(); - TopDocsCollector tdc = TopFieldCollector.create(sort[i], 10, Integer.MAX_VALUE); + TopFieldCollectorManager collectorManager = + TopFieldCollectorManager.create(sort[i], 10, Integer.MAX_VALUE); - is.search(q, tdc); - - ScoreDoc[] sd = tdc.topDocs().scoreDocs; + ScoreDoc[] sd = is.search(q, collectorManager).scoreDocs; for (int j = 1; j < sd.length; j++) { assertTrue(sd[j].doc != sd[j - 1].doc); } @@ -133,11 +132,9 @@ public void testSort() throws Exception { Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort()}; for (int i = 0; i < sort.length; i++) { Query q = new MatchAllDocsQuery(); - TopDocsCollector tdc = TopFieldCollector.create(sort[i], 10, Integer.MAX_VALUE); - - is.search(q, tdc); - - TopDocs td = tdc.topDocs(); + TopFieldCollectorManager tdc = + TopFieldCollectorManager.create(sort[i], 10, Integer.MAX_VALUE); + TopDocs td = is.search(q, tdc); ScoreDoc[] sd = td.scoreDocs; for (int j = 0; j < sd.length; j++) { assertTrue(Float.isNaN(sd[j].score)); @@ -162,15 +159,14 @@ public void testSharedHitcountCollector() throws Exception { Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort()}; for (int i = 0; i < sort.length; i++) { Query q = new MatchAllDocsQuery(); - TopDocsCollector tdc = TopFieldCollector.create(sort[i], 10, Integer.MAX_VALUE); + TopFieldCollectorManager tdc = + TopFieldCollectorManager.create(sort[i], 10, Integer.MAX_VALUE); + TopDocs td = is.search(q, tdc); - is.search(q, tdc); - - CollectorManager tsdc = - TopFieldCollector.createSharedManager(sort[i], 10, null, Integer.MAX_VALUE); - - TopDocs td = tdc.topDocs(); + TopFieldCollectorManager tsdc = + new TopFieldCollectorManager(sort[i], 10, null, Integer.MAX_VALUE); TopDocs td2 = concurrentSearcher.search(q, tsdc); + ScoreDoc[] sd = td.scoreDocs; for (int j = 0; j < sd.length; j++) { assertTrue(Float.isNaN(sd[j].score)); @@ -188,17 +184,15 @@ public void testSortWithoutTotalHitTracking() throws Exception { Query q = new MatchAllDocsQuery(); // check that setting trackTotalHits to false does not throw an NPE because // the index is not sorted - TopDocsCollector tdc; + TopFieldCollectorManager tdc; if (i % 2 == 0) { - tdc = TopFieldCollector.create(sort, 10, 1); + tdc = TopFieldCollectorManager.create(sort, 10, 1); } else { FieldDoc fieldDoc = new FieldDoc(1, Float.NaN, new Object[] {1}); - tdc = TopFieldCollector.create(sort, 10, fieldDoc, 1); + tdc = new TopFieldCollectorManager(sort, 10, fieldDoc, 1); } - is.search(q, tdc); - - TopDocs td = tdc.topDocs(); + TopDocs td = is.search(q, tdc); ScoreDoc[] sd = td.scoreDocs; for (int j = 0; j < sd.length; j++) { assertTrue(Float.isNaN(sd[j].score)); @@ -224,8 +218,10 @@ public void testTotalHits() throws Exception { for (int totalHitsThreshold = 0; totalHitsThreshold < 20; ++totalHitsThreshold) { for (FieldDoc after : new FieldDoc[] {null, new FieldDoc(4, Float.NaN, new Object[] {2L})}) { - TopFieldCollector collector = TopFieldCollector.create(sort, 2, after, totalHitsThreshold); + TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, 2, after, totalHitsThreshold); ScoreAndDoc scorer = new ScoreAndDoc(); + TopFieldCollector collector = collectorManager.newCollector(); LeafCollector leafCollector1 = collector.getLeafCollector(reader.leaves().get(0)); leafCollector1.setScorer(scorer); @@ -309,7 +305,7 @@ public void testSetMinCompetitiveScore() throws Exception { w.close(); Sort sort = new Sort(FIELD_SCORE, new SortField("foo", SortField.Type.LONG)); - TopFieldCollector collector = TopFieldCollector.create(sort, 2, null, 2); + TopFieldCollector collector = new TopFieldCollectorManager(sort, 2, null, 2).newCollector(); ScoreAndDoc scorer = new ScoreAndDoc(); LeafCollector leafCollector = collector.getLeafCollector(reader.leaves().get(0)); @@ -378,7 +374,8 @@ public void testTotalHitsWithScore() throws Exception { for (int totalHitsThreshold = 0; totalHitsThreshold < 20; ++totalHitsThreshold) { Sort sort = new Sort(FIELD_SCORE, new SortField("foo", SortField.Type.LONG)); - TopFieldCollector collector = TopFieldCollector.create(sort, 2, null, totalHitsThreshold); + TopFieldCollector collector = + new TopFieldCollectorManager(sort, 2, null, totalHitsThreshold).newCollector(); ScoreAndDoc scorer = new ScoreAndDoc(); LeafCollector leafCollector = collector.getLeafCollector(reader.leaves().get(0)); @@ -423,7 +420,8 @@ public void testSortNoResults() throws Exception { // Two Sort criteria to instantiate the multi/single comparators. Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort()}; for (int i = 0; i < sort.length; i++) { - TopDocsCollector tdc = TopFieldCollector.create(sort[i], 10, Integer.MAX_VALUE); + TopDocsCollector tdc = + TopFieldCollectorManager.create(sort[i], 10, Integer.MAX_VALUE).newCollector(); TopDocs td = tdc.topDocs(); assertEquals(0, td.totalHits.value); } @@ -457,41 +455,55 @@ public void testComputeScoresOnlyOnce() throws Exception { final IndexSearcher searcher = new IndexSearcher(reader); for (Sort sort : new Sort[] {new Sort(FIELD_SCORE), new Sort(new SortField("f", SortField.Type.SCORE))}) { - final TopFieldCollector topCollector = - TopFieldCollector.create(sort, TestUtil.nextInt(random(), 1, 2), Integer.MAX_VALUE); - final Collector assertingCollector = - new Collector() { + searcher.search( + query, + new CollectorManager<>() { + TopFieldCollectorManager topFieldCollectorManager = + TopFieldCollectorManager.create( + sort, TestUtil.nextInt(random(), 1, 2), Integer.MAX_VALUE); + @Override - public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { - final LeafCollector in = topCollector.getLeafCollector(context); - return new FilterLeafCollector(in) { + public Collector newCollector() throws IOException { + TopFieldCollector topCollector = topFieldCollectorManager.newCollector(); + return new Collector() { + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) + throws IOException { + final LeafCollector in = topCollector.getLeafCollector(context); + return new FilterLeafCollector(in) { + @Override + public void setScorer(final Scorable scorer) throws IOException { + Scorable s = + new FilterScorable(scorer) { + + int lastComputedDoc = -1; + + @Override + public float score() throws IOException { + if (lastComputedDoc == docID()) { + throw new AssertionError("Score computed twice on " + docID()); + } + lastComputedDoc = docID(); + return scorer.score(); + } + }; + super.setScorer(s); + } + }; + } + @Override - public void setScorer(final Scorable scorer) throws IOException { - Scorable s = - new FilterScorable(scorer) { - - int lastComputedDoc = -1; - - @Override - public float score() throws IOException { - if (lastComputedDoc == docID()) { - throw new AssertionError("Score computed twice on " + docID()); - } - lastComputedDoc = docID(); - return scorer.score(); - } - }; - super.setScorer(s); + public ScoreMode scoreMode() { + return topCollector.scoreMode(); } }; } @Override - public ScoreMode scoreMode() { - return topCollector.scoreMode(); + public Object reduce(Collection collectors) { + return null; } - }; - searcher.search(query, assertingCollector); + }); } reader.close(); w.close(); @@ -576,7 +588,7 @@ public void testConcurrentMinScore() throws Exception { Sort sort = new Sort(SortField.FIELD_SCORE, SortField.FIELD_DOC); CollectorManager manager = - TopFieldCollector.createSharedManager(sort, 2, null, 0); + new TopFieldCollectorManager(sort, 2, null, 0); TopFieldCollector collector = manager.newCollector(); TopFieldCollector collector2 = manager.newCollector(); assertTrue(collector.minScoreAcc == collector2.minScoreAcc); @@ -716,9 +728,8 @@ public void testRandomMinCompetitiveScore() throws Exception { }; for (Query query : queries) { Sort sort = new Sort(new SortField[] {SortField.FIELD_SCORE, SortField.FIELD_DOC}); - TopFieldCollector fieldCollector = doSearchWithThreshold(5, 0, query, sort, indexReader); TopDocs tdc = doConcurrentSearchWithThreshold(5, 0, query, sort, indexReader); - TopDocs tdc2 = fieldCollector.topDocs(); + TopDocs tdc2 = doSearchWithThreshold(5, 0, query, sort, indexReader); assertTrue(tdc.totalHits.value > 0); assertTrue(tdc2.totalHits.value > 0); @@ -743,20 +754,20 @@ public void testRelationVsTopDocsCount() throws Exception { try (IndexReader reader = DirectoryReader.open(w)) { IndexSearcher searcher = new IndexSearcher(reader); - TopFieldCollector collector = TopFieldCollector.create(sort, 2, 10); - searcher.search(new TermQuery(new Term("f", "foo")), collector); - assertEquals(10, collector.totalHits); - assertEquals(TotalHits.Relation.EQUAL_TO, collector.totalHitsRelation); - - collector = TopFieldCollector.create(sort, 2, 2); - searcher.search(new TermQuery(new Term("f", "foo")), collector); - assertTrue(10 >= collector.totalHits); - assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, collector.totalHitsRelation); - - collector = TopFieldCollector.create(sort, 10, 2); - searcher.search(new TermQuery(new Term("f", "foo")), collector); - assertEquals(10, collector.totalHits); - assertEquals(TotalHits.Relation.EQUAL_TO, collector.totalHitsRelation); + TopFieldCollectorManager collectorManager = TopFieldCollectorManager.create(sort, 2, 10); + TopDocs topDocs = searcher.search(new TermQuery(new Term("f", "foo")), collectorManager); + assertEquals(10, topDocs.totalHits.value); + assertEquals(TotalHits.Relation.EQUAL_TO, topDocs.totalHits.relation); + + collectorManager = TopFieldCollectorManager.create(sort, 2, 2); + topDocs = searcher.search(new TermQuery(new Term("f", "foo")), collectorManager); + assertTrue(10 >= topDocs.totalHits.value); + assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, topDocs.totalHits.relation); + + collectorManager = TopFieldCollectorManager.create(sort, 10, 2); + topDocs = searcher.search(new TermQuery(new Term("f", "foo")), collectorManager); + assertEquals(10, topDocs.totalHits.value); + assertEquals(TotalHits.Relation.EQUAL_TO, topDocs.totalHits.relation); } } } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollectorEarlyTermination.java b/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollectorEarlyTermination.java index 5cdfb8b17452..c90a46fddad7 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollectorEarlyTermination.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollectorEarlyTermination.java @@ -133,9 +133,10 @@ private void doTestEarlyTermination(boolean paging) throws IOException { } else { after = null; } - final TopFieldCollector collector1 = - TopFieldCollector.create(sort, numHits, after, Integer.MAX_VALUE); - final TopFieldCollector collector2 = TopFieldCollector.create(sort, numHits, after, 1); + final TopFieldCollectorManager collectorManager1 = + new TopFieldCollectorManager(sort, numHits, after, Integer.MAX_VALUE); + final TopFieldCollectorManager collectorManager2 = + new TopFieldCollectorManager(sort, numHits, after, 1); final Query query; if (random().nextBoolean()) { @@ -143,17 +144,21 @@ private void doTestEarlyTermination(boolean paging) throws IOException { } else { query = new MatchAllDocsQuery(); } - searcher.search(query, collector1); - searcher.search(query, collector2); - TopDocs td1 = collector1.topDocs(); - TopDocs td2 = collector2.topDocs(); - assertFalse(collector1.isEarlyTerminated()); + TopDocs td1 = searcher.search(query, collectorManager1); + TopDocs td2 = searcher.search(query, collectorManager2); + + assertFalse( + collectorManager1.getCollectors().stream() + .anyMatch(TopFieldCollector::isEarlyTerminated)); if (paging == false && maxSegmentSize > numHits && query instanceof MatchAllDocsQuery) { // Make sure that we sometimes early terminate - assertTrue(collector2.isEarlyTerminated()); + assertTrue( + collectorManager2.getCollectors().stream() + .anyMatch(TopFieldCollector::isEarlyTerminated)); } - if (collector2.isEarlyTerminated()) { + if (collectorManager2.getCollectors().stream() + .anyMatch(TopFieldCollector::isEarlyTerminated)) { assertTrue(td2.totalHits.value >= td1.scoreDocs.length); assertTrue(td2.totalHits.value <= reader.maxDoc()); } else { diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java index b02779ca0c3e..57aa69dd0a9c 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java @@ -39,9 +39,8 @@ public void testBasics() throws Exception { writer.close(); IndexSearcher searcher = newSearcher(reader); - TotalHitCountCollector c = new TotalHitCountCollector(); - searcher.search(new MatchAllDocsQuery(), c); - assertEquals(5, c.getTotalHits()); + TotalHitCountCollectorManager c = new TotalHitCountCollectorManager(); + assertEquals(5, searcher.search(new MatchAllDocsQuery(), c).intValue()); reader.close(); indexStore.close(); } diff --git a/lucene/luke/src/java/org/apache/lucene/luke/models/search/SearchImpl.java b/lucene/luke/src/java/org/apache/lucene/luke/models/search/SearchImpl.java index a1122a3118d0..a8c515354d5f 100644 --- a/lucene/luke/src/java/org/apache/lucene/luke/models/search/SearchImpl.java +++ b/lucene/luke/src/java/org/apache/lucene/luke/models/search/SearchImpl.java @@ -57,7 +57,7 @@ import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.search.SortedSetSortField; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TopScoreDocCollector; +import org.apache.lucene.search.TopScoreDocCollectorManager; import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.similarities.BM25Similarity; import org.apache.lucene.search.similarities.ClassicSimilarity; @@ -314,9 +314,9 @@ private SearchResults search() throws IOException { topDocs = searcher.searchAfter(after, query, pageSize, sort); } else { int hitsThreshold = exactHitsCount ? Integer.MAX_VALUE : DEFAULT_TOTAL_HITS_THRESHOLD; - TopScoreDocCollector collector = TopScoreDocCollector.create(pageSize, after, hitsThreshold); - searcher.search(query, collector); - topDocs = collector.topDocs(); + TopScoreDocCollectorManager collectorManager = + new TopScoreDocCollectorManager(pageSize, after, hitsThreshold); + topDocs = searcher.search(query, collectorManager); } // reset total hits for the current query diff --git a/lucene/queries/src/test/org/apache/lucene/queries/spans/TestBasics.java b/lucene/queries/src/test/org/apache/lucene/queries/spans/TestBasics.java index 9e5bf95d255c..90c3168c275f 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/spans/TestBasics.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/spans/TestBasics.java @@ -39,7 +39,8 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TopScoreDocCollector; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopScoreDocCollectorManager; import org.apache.lucene.store.Directory; import org.apache.lucene.util.English; import org.apache.lucene.util.LuceneTestCase; @@ -508,10 +509,11 @@ public void testBooleanSpanQuery() throws Exception { SpanQuery sq2 = new SpanTermQuery(new Term(FIELD, "clckwork")); query.add(sq1, BooleanClause.Occur.SHOULD); query.add(sq2, BooleanClause.Occur.SHOULD); - TopScoreDocCollector collector = TopScoreDocCollector.create(1000, Integer.MAX_VALUE); - searcher.search(query.build(), collector); - hits = collector.topDocs().scoreDocs.length; - for (ScoreDoc scoreDoc : collector.topDocs().scoreDocs) { + TopScoreDocCollectorManager collectorManager = + TopScoreDocCollectorManager.create(1000, Integer.MAX_VALUE); + TopDocs topDocs = searcher.search(query.build(), collectorManager); + hits = topDocs.scoreDocs.length; + for (ScoreDoc scoreDoc : topDocs.scoreDocs) { System.out.println(scoreDoc.doc); } indexReader.close(); @@ -543,10 +545,11 @@ public void testDismaxSpanQuery() throws Exception { new SpanTermQuery(new Term(FIELD, "clockwork")), new SpanTermQuery(new Term(FIELD, "clckwork"))), 1.0f); - TopScoreDocCollector collector = TopScoreDocCollector.create(1000, Integer.MAX_VALUE); - searcher.search(query, collector); - hits = collector.topDocs().scoreDocs.length; - for (ScoreDoc scoreDoc : collector.topDocs().scoreDocs) { + TopScoreDocCollectorManager collectorManager = + TopScoreDocCollectorManager.create(1000, Integer.MAX_VALUE); + TopDocs topDocs = searcher.search(query, collectorManager); + hits = topDocs.scoreDocs.length; + for (ScoreDoc scoreDoc : topDocs.scoreDocs) { System.out.println(scoreDoc.doc); } indexReader.close(); diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestCombinedFieldQuery.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestCombinedFieldQuery.java index 7798a2a7017f..27f88292aef3 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestCombinedFieldQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestCombinedFieldQuery.java @@ -37,7 +37,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermStatistics; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TopScoreDocCollector; +import org.apache.lucene.search.TopScoreDocCollectorManager; import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.similarities.BM25Similarity; import org.apache.lucene.search.similarities.BooleanSimilarity; @@ -149,11 +149,10 @@ public void testSameScore() throws IOException { .addField("g", 1f) .addTerm(new BytesRef("a")) .build(); - TopScoreDocCollector collector = - TopScoreDocCollector.create( + TopScoreDocCollectorManager collectorManager = + new TopScoreDocCollectorManager( Math.min(reader.numDocs(), Integer.MAX_VALUE), null, Integer.MAX_VALUE); - searcher.search(query, collector); - TopDocs topDocs = collector.topDocs(); + TopDocs topDocs = searcher.search(query, collectorManager); assertEquals(new TotalHits(11, TotalHits.Relation.EQUAL_TO), topDocs.totalHits); // All docs must have the same score for (int i = 0; i < topDocs.scoreDocs.length; ++i) { @@ -190,7 +189,7 @@ public void testNormsDisabled() throws IOException { Similarity searchSimilarity = randomCompatibleSimilarity(); searcher.setSimilarity(searchSimilarity); - TopScoreDocCollector collector = TopScoreDocCollector.create(10, null, 10); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(10, null, 10); CombinedFieldQuery query = new CombinedFieldQuery.Builder() @@ -198,8 +197,7 @@ public void testNormsDisabled() throws IOException { .addField("b", 1.0f) .addTerm(new BytesRef("value")) .build(); - searcher.search(query, collector); - TopDocs topDocs = collector.topDocs(); + TopDocs topDocs = searcher.search(query, collectorManager); assertEquals(new TotalHits(2, TotalHits.Relation.EQUAL_TO), topDocs.totalHits); CombinedFieldQuery invalidQuery = @@ -210,7 +208,7 @@ public void testNormsDisabled() throws IOException { .build(); IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, () -> searcher.search(invalidQuery, collector)); + IllegalArgumentException.class, () -> searcher.search(invalidQuery, collectorManager)); assertTrue(e.getMessage().contains("requires norms to be consistent across fields")); reader.close(); @@ -422,16 +420,14 @@ private static Similarity randomCompatibleSimilarity() { private void checkExpectedHits( IndexSearcher searcher, int numHits, Query firstQuery, Query secondQuery) throws IOException { - TopScoreDocCollector firstCollector = - TopScoreDocCollector.create(numHits, null, Integer.MAX_VALUE); - searcher.search(firstQuery, firstCollector); - TopDocs firstTopDocs = firstCollector.topDocs(); + TopScoreDocCollectorManager collectorManager = + new TopScoreDocCollectorManager(numHits, null, Integer.MAX_VALUE); + + TopDocs firstTopDocs = searcher.search(firstQuery, collectorManager); assertEquals(numHits, firstTopDocs.totalHits.value); - TopScoreDocCollector secondCollector = - TopScoreDocCollector.create(numHits, null, Integer.MAX_VALUE); - searcher.search(secondQuery, secondCollector); - TopDocs secondTopDocs = secondCollector.topDocs(); + collectorManager = new TopScoreDocCollectorManager(numHits, null, Integer.MAX_VALUE); + TopDocs secondTopDocs = searcher.search(secondQuery, collectorManager); CheckHits.checkEqual(firstQuery, secondTopDocs.scoreDocs, firstTopDocs.scoreDocs); } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java index 7b312062f1d1..3121a3b27d86 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java @@ -61,7 +61,7 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TopFieldCollector; +import org.apache.lucene.search.TopFieldCollectorManager; import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.search.suggest.InputIterator; import org.apache.lucene.search.suggest.Lookup; @@ -708,7 +708,7 @@ public List lookup( // System.out.println("finalQuery=" + finalQuery); // Sort by weight, descending: - TopFieldCollector c = TopFieldCollector.create(SORT, num, 1); + TopFieldCollectorManager c = TopFieldCollectorManager.create(SORT, num, 1); List results = null; SearcherManager mgr; IndexSearcher searcher; @@ -718,9 +718,7 @@ public List lookup( } try { // System.out.println("got searcher=" + searcher); - searcher.search(finalQuery, c); - - TopFieldDocs hits = c.topDocs(); + TopFieldDocs hits = searcher.search(finalQuery, c); // Slower way if postings are not pre-sorted by weight: // hits = searcher.search(query, null, num, SORT); diff --git a/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java index 5b18e0c74d90..37885dcddd03 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java @@ -42,7 +42,6 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiBits; import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.NumericDocValues; @@ -55,13 +54,12 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.SimpleCollector; import org.apache.lucene.search.Sort; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.FixedBitSetCollector; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.SloppyMath; @@ -1275,29 +1273,7 @@ private void indexPoints(double[] lats, double[] lons, Set deleted, Ind } private FixedBitSet searchIndex(IndexSearcher s, Query query, int maxDoc) throws IOException { - final FixedBitSet hits = new FixedBitSet(maxDoc); - s.search( - query, - new SimpleCollector() { - - private int docBase; - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE_NO_SCORES; - } - - @Override - protected void doSetNextReader(LeafReaderContext context) { - docBase = context.docBase; - } - - @Override - public void collect(int doc) { - hits.set(docBase + doc); - } - }); - return hits; + return s.search(query, FixedBitSetCollector.create(maxDoc)); } private void buildError( diff --git a/lucene/test-framework/src/java/org/apache/lucene/geo/BaseXYPointTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/geo/BaseXYPointTestCase.java index bcdecf794f1a..3c2f524e739e 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/geo/BaseXYPointTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/geo/BaseXYPointTestCase.java @@ -42,7 +42,6 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiBits; import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.NumericDocValues; @@ -54,13 +53,12 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.SimpleCollector; import org.apache.lucene.search.Sort; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.FixedBitSetCollector; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; @@ -1162,29 +1160,7 @@ private void indexPoints(float[] xs, float[] ys, Set deleted, IndexWrit } private FixedBitSet searchIndex(IndexSearcher s, Query query, int maxDoc) throws IOException { - final FixedBitSet hits = new FixedBitSet(maxDoc); - s.search( - query, - new SimpleCollector() { - - private int docBase; - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE_NO_SCORES; - } - - @Override - protected void doSetNextReader(LeafReaderContext context) { - docBase = context.docBase; - } - - @Override - public void collect(int doc) { - hits.set(docBase + doc); - } - }); - return hits; + return s.search(query, FixedBitSetCollector.create(maxDoc)); } private void buildError( diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java index 6e1f705a356d..6dc9a3989849 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java @@ -16,7 +16,6 @@ */ package org.apache.lucene.search; -import java.io.IOException; import java.util.Arrays; import java.util.HashSet; import java.util.Set; @@ -27,7 +26,6 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiBits; import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.NumericDocValues; @@ -36,6 +34,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.FixedBitSetCollector; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; @@ -293,27 +292,7 @@ private void verify(Range[][] ranges) throws Exception { System.out.println(" query=" + query); } - final FixedBitSet hits = new FixedBitSet(maxDoc); - s.search( - query, - new SimpleCollector() { - private int docBase; - - @Override - public void collect(int doc) { - hits.set(docBase + doc); - } - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - docBase = context.docBase; - } - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE_NO_SCORES; - } - }); + FixedBitSet hits = s.search(query, FixedBitSetCollector.create(maxDoc)); NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id"); for (int docID = 0; docID < maxDoc; ++docID) { diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java b/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java index 96dc555a6d2e..743488e4459b 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java @@ -617,12 +617,13 @@ public static void checkTopScores(Random random, Query query, IndexSearcher sear private static void doCheckTopScores(Query query, IndexSearcher searcher, int numHits) throws IOException { - TopScoreDocCollector collector1 = - TopScoreDocCollector.create(numHits, null, Integer.MAX_VALUE); // COMPLETE - TopScoreDocCollector collector2 = TopScoreDocCollector.create(numHits, null, 1); // TOP_SCORES - searcher.search(query, collector1); - searcher.search(query, collector2); - checkEqual(query, collector1.topDocs().scoreDocs, collector2.topDocs().scoreDocs); + TopScoreDocCollectorManager collectorManager1 = + new TopScoreDocCollectorManager(numHits, null, Integer.MAX_VALUE); // COMPLETE + TopScoreDocCollectorManager collectorManager2 = + new TopScoreDocCollectorManager(numHits, null, 1); // TOP_SCORES + TopDocs topDocs1 = searcher.search(query, collectorManager1); + TopDocs topDocs2 = searcher.search(query, collectorManager2); + checkEqual(query, topDocs1.scoreDocs, topDocs2.scoreDocs); } private static void doCheckMaxScores(Random random, Query query, IndexSearcher searcher) diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/FixedBitSetCollector.java b/lucene/test-framework/src/java/org/apache/lucene/util/FixedBitSetCollector.java new file mode 100644 index 000000000000..629a389d295e --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/util/FixedBitSetCollector.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.util; + +import java.util.Collection; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.CollectorManager; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.SimpleCollector; + +/** Test utility collector that uses FixedBitSet to record hits. */ +public class FixedBitSetCollector extends SimpleCollector { + private FixedBitSet hits; + private int docBase; + + public FixedBitSetCollector(int maxDoc) { + hits = new FixedBitSet(maxDoc); + } + + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; + } + + @Override + protected void doSetNextReader(LeafReaderContext context) { + docBase = context.docBase; + } + + @Override + public void collect(int doc) { + hits.set(docBase + doc); + } + + public FixedBitSet getHits() { + return hits; + } + + public static CollectorManager create(int maxDoc) { + return new CollectorManager<>() { + @Override + public FixedBitSetCollector newCollector() { + return new FixedBitSetCollector(maxDoc); + } + + @Override + public FixedBitSet reduce(Collection collectors) { + FixedBitSet result = new FixedBitSet(maxDoc); + collectors.stream().forEach(c -> result.or(c.getHits())); + return result; + } + }; + } +} From 776759dc8eb5d94786709d197690fbb17664463f Mon Sep 17 00:00:00 2001 From: Zach Chen Date: Wed, 11 Aug 2021 21:22:38 -0700 Subject: [PATCH 02/14] LUCENE-10002: Refactor DrillSideways to use extracted out collector managers --- .../apache/lucene/facet/DrillSideways.java | 111 ++++-------------- 1 file changed, 23 insertions(+), 88 deletions(-) diff --git a/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java b/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java index 758a31e09f25..89448bb096a8 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java @@ -18,7 +18,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -45,8 +44,9 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopFieldCollector; +import org.apache.lucene.search.TopFieldCollectorManager; import org.apache.lucene.search.TopFieldDocs; -import org.apache.lucene.search.TopScoreDocCollector; +import org.apache.lucene.search.TopScoreDocCollectorManager; import org.apache.lucene.util.ThreadInterruptedException; /** @@ -296,54 +296,20 @@ public DrillSidewaysResult search( limit = 1; // the collector does not alow numHits = 0 } final int fTopN = Math.min(topN, limit); + final TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, fTopN, after, Integer.MAX_VALUE); - if (executor != null) { // We have an executor, let use the multi-threaded version - - final CollectorManager collectorManager = - new CollectorManager<>() { - - @Override - public TopFieldCollector newCollector() { - return TopFieldCollector.create(sort, fTopN, after, Integer.MAX_VALUE); - } - - @Override - public TopFieldDocs reduce(Collection collectors) { - final TopFieldDocs[] topFieldDocs = new TopFieldDocs[collectors.size()]; - int pos = 0; - for (TopFieldCollector collector : collectors) - topFieldDocs[pos++] = collector.topDocs(); - return TopDocs.merge(sort, topN, topFieldDocs); - } - }; - ConcurrentDrillSidewaysResult r = searchConcurrently(query, collectorManager); - TopFieldDocs topDocs = r.collectorResult; - if (doDocScores) { - TopFieldCollector.populateScores(topDocs.scoreDocs, searcher, query); - } - return new DrillSidewaysResult( - r.facets, - topDocs, - r.drillDownFacetsCollector, - r.drillSidewaysFacetsCollector, - r.drillSidewaysDims); - - } else { - - final TopFieldCollector hitCollector = - TopFieldCollector.create(sort, fTopN, after, Integer.MAX_VALUE); - DrillSidewaysResult r = search(query, hitCollector); - TopFieldDocs topDocs = hitCollector.topDocs(); - if (doDocScores) { - TopFieldCollector.populateScores(topDocs.scoreDocs, searcher, query); - } - return new DrillSidewaysResult( - r.facets, - topDocs, - r.drillDownFacetsCollector, - r.drillSidewaysFacetsCollector, - r.drillSidewaysDims); + ConcurrentDrillSidewaysResult r = search(query, collectorManager); + TopDocs topDocs = r.collectorResult; + if (doDocScores) { + TopFieldCollector.populateScores(topDocs.scoreDocs, searcher, query); } + return new DrillSidewaysResult( + r.facets, + r.collectorResult, + r.drillDownFacetsCollector, + r.drillSidewaysFacetsCollector, + r.drillSidewaysDims); } else { return search(after, query, topN); } @@ -362,46 +328,15 @@ public DrillSidewaysResult search(ScoreDoc after, DrillDownQuery query, int topN limit = 1; // the collector does not alow numHits = 0 } final int fTopN = Math.min(topN, limit); - - if (executor != null) { // We have an executor, let use the multi-threaded version - - final CollectorManager collectorManager = - new CollectorManager<>() { - - @Override - public TopScoreDocCollector newCollector() { - return TopScoreDocCollector.create(fTopN, after, Integer.MAX_VALUE); - } - - @Override - public TopDocs reduce(Collection collectors) { - final TopDocs[] topDocs = new TopDocs[collectors.size()]; - int pos = 0; - for (TopScoreDocCollector collector : collectors) - topDocs[pos++] = collector.topDocs(); - return TopDocs.merge(topN, topDocs); - } - }; - ConcurrentDrillSidewaysResult r = searchConcurrently(query, collectorManager); - return new DrillSidewaysResult( - r.facets, - r.collectorResult, - r.drillDownFacetsCollector, - r.drillSidewaysFacetsCollector, - r.drillSidewaysDims); - - } else { - - TopScoreDocCollector hitCollector = - TopScoreDocCollector.create(topN, after, Integer.MAX_VALUE); - DrillSidewaysResult r = search(query, hitCollector); - return new DrillSidewaysResult( - r.facets, - hitCollector.topDocs(), - r.drillDownFacetsCollector, - r.drillSidewaysFacetsCollector, - r.drillSidewaysDims); - } + final TopScoreDocCollectorManager collectorManager = + new TopScoreDocCollectorManager(fTopN, after, Integer.MAX_VALUE); + ConcurrentDrillSidewaysResult r = search(query, collectorManager); + return new DrillSidewaysResult( + r.facets, + r.collectorResult, + r.drillDownFacetsCollector, + r.drillSidewaysFacetsCollector, + r.drillSidewaysDims); } /** From 0c2c12bb7464ed6f64ef498b7f094823b87b806c Mon Sep 17 00:00:00 2001 From: Zach Chen Date: Thu, 12 Aug 2021 21:24:29 -0700 Subject: [PATCH 03/14] LUCENE-10002: Refactor some more tests to use collector manager --- .../CachingNaiveBayesClassifier.java | 6 +- .../SimpleNaiveBayesClassifier.java | 10 +- .../SimpleNaiveBayesDocumentClassifier.java | 6 +- .../org/apache/lucene/index/TestOmitTf.java | 182 +++++++++++------- .../lucene/search/TestBooleanQuery.java | 91 ++++----- .../lucene/search/TestConjunctions.java | 58 +++--- .../lucene/search/TestDoubleValuesSource.java | 119 +++++++----- .../lucene/search/TestEarlyTermination.java | 59 +++--- .../lucene/search/TestLRUQueryCache.java | 10 +- .../apache/lucene/search/TestScorerPerf.java | 2 +- .../apache/lucene/search/TestSimilarity.java | 155 +++++++++------ .../lucene/search/TestSloppyPhraseQuery.java | 36 ++-- .../lucene/search/TestTopDocsMerge.java | 7 +- .../demo/facet/DistanceFacetsExample.java | 6 +- .../demo/facet/SimpleFacetsExample.java | 5 +- .../apache/lucene/facet/TestFacetsConfig.java | 3 +- .../facet/TestLongValueFacetCounts.java | 25 ++- .../facet/TestStringValueFacetCounts.java | 6 +- .../facet/range/TestRangeFacetCounts.java | 68 +++---- .../TestSortedSetDocValuesFacets.java | 8 +- .../taxonomy/TestSearcherTaxonomyManager.java | 9 +- .../TestTaxonomyFacetAssociations.java | 23 +-- .../taxonomy/TestTaxonomyFacetCounts.java | 19 +- .../taxonomy/TestTaxonomyFacetCounts2.java | 13 +- .../taxonomy/TestTaxonomyFacetLabels.java | 4 +- .../TestTaxonomyFacetSumValueSource.java | 20 +- .../highlight/TestHighlighterPhrase.java | 33 +--- .../lucene/search/join/TestJoinUtil.java | 5 +- ...TestIndexAndTaxonomyReplicationClient.java | 4 +- .../TestLargeNumHitsTopDocsCollector.java | 49 +++-- .../prefix/TestHeatmapFacetCounter.java | 9 +- .../spatial/prefix/TestNumberRangeFacets.java | 29 +-- .../lucene/spatial3d/TestGeo3DPoint.java | 28 +-- .../org/apache/lucene/search/CheckHits.java | 23 ++- 34 files changed, 577 insertions(+), 553 deletions(-) diff --git a/lucene/classification/src/java/org/apache/lucene/classification/CachingNaiveBayesClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/CachingNaiveBayesClassifier.java index db8c6dbba3d6..98576e618efe 100644 --- a/lucene/classification/src/java/org/apache/lucene/classification/CachingNaiveBayesClassifier.java +++ b/lucene/classification/src/java/org/apache/lucene/classification/CachingNaiveBayesClassifier.java @@ -32,7 +32,7 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TotalHitCountCollector; +import org.apache.lucene.search.TotalHitCountCollectorManager; import org.apache.lucene.util.BytesRef; /** @@ -179,10 +179,8 @@ private Map getWordFreqForClassess(String word) throws IOExce if (query != null) { booleanQuery.add(query, BooleanClause.Occur.MUST); } - TotalHitCountCollector totalHitCountCollector = new TotalHitCountCollector(); - indexSearcher.search(booleanQuery.build(), totalHitCountCollector); - int ret = totalHitCountCollector.getTotalHits(); + int ret = indexSearcher.search(booleanQuery.build(), new TotalHitCountCollectorManager()); if (ret != 0) { searched.put(cclass, ret); } diff --git a/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java index 4083cf82f0c6..fc2b909e27d9 100644 --- a/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java +++ b/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java @@ -35,7 +35,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TotalHitCountCollector; +import org.apache.lucene.search.TotalHitCountCollectorManager; import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.util.BytesRef; @@ -169,7 +169,6 @@ protected int countDocsWithClass() throws IOException { Terms terms = MultiTerms.getTerms(this.indexReader, this.classFieldName); int docCount; if (terms == null || terms.getDocCount() == -1) { // in case codec doesn't support getDocCount - TotalHitCountCollector classQueryCountCollector = new TotalHitCountCollector(); BooleanQuery.Builder q = new BooleanQuery.Builder(); q.add( new BooleanClause( @@ -179,8 +178,7 @@ protected int countDocsWithClass() throws IOException { if (query != null) { q.add(query, BooleanClause.Occur.MUST); } - indexSearcher.search(q.build(), classQueryCountCollector); - docCount = classQueryCountCollector.getTotalHits(); + docCount = indexSearcher.search(q.build(), new TotalHitCountCollectorManager()); } else { docCount = terms.getDocCount(); } @@ -276,9 +274,7 @@ private int getWordFreqForClass(String word, Term term) throws IOException { if (query != null) { booleanQuery.add(query, BooleanClause.Occur.MUST); } - TotalHitCountCollector totalHitCountCollector = new TotalHitCountCollector(); - indexSearcher.search(booleanQuery.build(), totalHitCountCollector); - return totalHitCountCollector.getTotalHits(); + return indexSearcher.search(booleanQuery.build(), new TotalHitCountCollectorManager()); } private double calculateLogPrior(Term term, int docsWithClassSize) throws IOException { diff --git a/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java index eddf02d59848..184c41b9fcb8 100644 --- a/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java +++ b/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java @@ -40,7 +40,7 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TotalHitCountCollector; +import org.apache.lucene.search.TotalHitCountCollectorManager; import org.apache.lucene.util.BytesRef; /** @@ -263,9 +263,7 @@ private int getWordFreqForClass(String word, String fieldName, Term term) throws if (query != null) { booleanQuery.add(query, BooleanClause.Occur.MUST); } - TotalHitCountCollector totalHitCountCollector = new TotalHitCountCollector(); - indexSearcher.search(booleanQuery.build(), totalHitCountCollector); - return totalHitCountCollector.getTotalHits(); + return indexSearcher.search(booleanQuery.build(), new TotalHitCountCollectorManager()); } private double calculateLogPrior(Term term, int docsWithClassSize) throws IOException { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java b/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java index 2549396fe5bd..bf3cfa7ae42c 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java @@ -17,6 +17,7 @@ package org.apache.lucene.index; import java.io.IOException; +import java.util.Collection; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; @@ -26,6 +27,7 @@ import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.CollectionStatistics; +import org.apache.lucene.search.CollectorManager; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PhraseQuery; @@ -34,6 +36,7 @@ import org.apache.lucene.search.SimpleCollector; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermStatistics; +import org.apache.lucene.search.TotalHitCountCollectorManager; import org.apache.lucene.search.similarities.TFIDFSimilarity; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; @@ -218,120 +221,155 @@ public void testBasic() throws Exception { searcher.search( q1, - new CountingHitCollector() { - private Scorable scorer; - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE; - } - + new CollectorManager() { @Override - public final void setScorer(Scorable scorer) { - this.scorer = scorer; + public CountingHitCollector newCollector() throws IOException { + return new CountingHitCollector() { + private Scorable scorer; + + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE; + } + + @Override + public final void setScorer(Scorable scorer) { + this.scorer = scorer; + } + + @Override + public final void collect(int doc) throws IOException { + // System.out.println("Q1: Doc=" + doc + " score=" + score); + float score = scorer.score(); + assertTrue("got score=" + score, score == 1.0f); + super.collect(doc); + } + }; } @Override - public final void collect(int doc) throws IOException { - // System.out.println("Q1: Doc=" + doc + " score=" + score); - float score = scorer.score(); - assertTrue("got score=" + score, score == 1.0f); - super.collect(doc); + public Object reduce(Collection collectors) throws IOException { + return null; } }); // System.out.println(CountingHitCollector.getCount()); searcher.search( q2, - new CountingHitCollector() { - private Scorable scorer; - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE; - } - + new CollectorManager() { @Override - public final void setScorer(Scorable scorer) { - this.scorer = scorer; + public CountingHitCollector newCollector() throws IOException { + return new CountingHitCollector() { + private Scorable scorer; + + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE; + } + + @Override + public final void setScorer(Scorable scorer) { + this.scorer = scorer; + } + + @Override + public final void collect(int doc) throws IOException { + // System.out.println("Q2: Doc=" + doc + " score=" + score); + float score = scorer.score(); + assertEquals(1.0f + doc, score, 0.00001f); + super.collect(doc); + } + }; } @Override - public final void collect(int doc) throws IOException { - // System.out.println("Q2: Doc=" + doc + " score=" + score); - float score = scorer.score(); - assertEquals(1.0f + doc, score, 0.00001f); - super.collect(doc); + public Object reduce(Collection collectors) throws IOException { + return null; } }); + // System.out.println(CountingHitCollector.getCount()); searcher.search( q3, - new CountingHitCollector() { - private Scorable scorer; - + new CollectorManager() { @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE; + public CountingHitCollector newCollector() throws IOException { + return new CountingHitCollector() { + private Scorable scorer; + + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE; + } + + @Override + public final void setScorer(Scorable scorer) { + this.scorer = scorer; + } + + @Override + public final void collect(int doc) throws IOException { + // System.out.println("Q1: Doc=" + doc + " score=" + score); + float score = scorer.score(); + assertTrue(score == 1.0f); + assertFalse(doc % 2 == 0); + super.collect(doc); + } + }; } @Override - public final void setScorer(Scorable scorer) { - this.scorer = scorer; - } - - @Override - public final void collect(int doc) throws IOException { - // System.out.println("Q1: Doc=" + doc + " score=" + score); - float score = scorer.score(); - assertTrue(score == 1.0f); - assertFalse(doc % 2 == 0); - super.collect(doc); + public Object reduce(Collection collectors) throws IOException { + return null; } }); + // System.out.println(CountingHitCollector.getCount()); searcher.search( q4, - new CountingHitCollector() { - private Scorable scorer; - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE; - } - + new CollectorManager() { @Override - public final void setScorer(Scorable scorer) { - this.scorer = scorer; + public CountingHitCollector newCollector() throws IOException { + return new CountingHitCollector() { + private Scorable scorer; + + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE; + } + + @Override + public final void setScorer(Scorable scorer) { + this.scorer = scorer; + } + + @Override + public final void collect(int doc) throws IOException { + float score = scorer.score(); + // System.out.println("Q1: Doc=" + doc + " score=" + score); + assertTrue(score == 1.0f); + assertTrue(doc % 2 == 0); + super.collect(doc); + } + }; } @Override - public final void collect(int doc) throws IOException { - float score = scorer.score(); - // System.out.println("Q1: Doc=" + doc + " score=" + score); - assertTrue(score == 1.0f); - assertTrue(doc % 2 == 0); - super.collect(doc); + public Object reduce(Collection collectors) throws IOException { + return null; } }); + // System.out.println(CountingHitCollector.getCount()); BooleanQuery.Builder bq = new BooleanQuery.Builder(); bq.add(q1, Occur.MUST); bq.add(q4, Occur.MUST); - searcher.search( - bq.build(), - new CountingHitCollector() { - @Override - public final void collect(int doc) throws IOException { - // System.out.println("BQ: Doc=" + doc + " score=" + score); - super.collect(doc); - } - }); - assertEquals(15, CountingHitCollector.getCount()); + int count = searcher.search(bq.build(), new TotalHitCountCollectorManager()); + assertEquals(15, count); reader.close(); dir.close(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java index 282f68f150c7..15c289cba50b 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.BitSet; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutorService; @@ -41,6 +41,8 @@ import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.similarities.ClassicSimilarity; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.FixedBitSetCollector; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.NamedThreadFactory; import org.apache.lucene.util.TestUtil; @@ -409,30 +411,8 @@ public void testMinShouldMatchLeniency() throws Exception { dir.close(); } - private static BitSet getMatches(IndexSearcher searcher, Query query) throws IOException { - BitSet set = new BitSet(); - searcher.search( - query, - new SimpleCollector() { - int docBase = 0; - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE_NO_SCORES; - } - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - super.doSetNextReader(context); - docBase = context.docBase; - } - - @Override - public void collect(int doc) throws IOException { - set.set(docBase + doc); - } - }); - return set; + private static FixedBitSet getMatches(IndexSearcher searcher, Query query) throws IOException { + return searcher.search(query, FixedBitSetCollector.create(searcher.reader.maxDoc())); } public void testFILTERClauseBehavesLikeMUST() throws IOException { @@ -466,8 +446,8 @@ public void testFILTERClauseBehavesLikeMUST() throws IOException { bq2.add(q, Occur.FILTER); } - final BitSet matches1 = getMatches(searcher, bq1.build()); - final BitSet matches2 = getMatches(searcher, bq2.build()); + final FixedBitSet matches1 = getMatches(searcher, bq1.build()); + final FixedBitSet matches2 = getMatches(searcher, bq2.build()); assertEquals(matches1, matches2); } @@ -490,35 +470,46 @@ private void assertSameScoresWithoutFilters(IndexSearcher searcher, BooleanQuery final AtomicBoolean matched = new AtomicBoolean(); searcher.search( bq, - new SimpleCollector() { - int docBase; - Scorable scorer; - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - super.doSetNextReader(context); - docBase = context.docBase; - } - + new CollectorManager() { @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE; + public SimpleCollector newCollector() throws IOException { + return new SimpleCollector() { + int docBase; + Scorable scorer; + + @Override + protected void doSetNextReader(LeafReaderContext context) throws IOException { + super.doSetNextReader(context); + docBase = context.docBase; + } + + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE; + } + + @Override + public void setScorer(Scorable scorer) throws IOException { + this.scorer = scorer; + } + + @Override + public void collect(int doc) throws IOException { + final float actualScore = scorer.score(); + final float expectedScore = + searcher.explain(bq2, docBase + doc).getValue().floatValue(); + assertEquals(expectedScore, actualScore, 10e-5); + matched.set(true); + } + }; } @Override - public void setScorer(Scorable scorer) throws IOException { - this.scorer = scorer; - } - - @Override - public void collect(int doc) throws IOException { - final float actualScore = scorer.score(); - final float expectedScore = - searcher.explain(bq2, docBase + doc).getValue().floatValue(); - assertEquals(expectedScore, actualScore, 10e-5); - matched.set(true); + public Object reduce(Collection collectors) throws IOException { + return null; } }); + assertTrue(matched.get()); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java b/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java index 24968aeae356..743b9ab0b7cd 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java @@ -20,6 +20,7 @@ import java.util.Collection; import java.util.HashSet; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; @@ -123,37 +124,48 @@ public void testScorerGetChildren() throws Exception { b.add(new TermQuery(new Term("field", "b")), BooleanClause.Occur.FILTER); Query q = b.build(); IndexSearcher s = new IndexSearcher(r); - final boolean[] setScorerCalled = new boolean[1]; + final AtomicBoolean setScorerCalled = new AtomicBoolean(false); s.search( q, - new SimpleCollector() { + new CollectorManager() { @Override - public void setScorer(Scorable s) throws IOException { - Collection childScorers = s.getChildren(); - setScorerCalled[0] = true; - assertEquals(2, childScorers.size()); - Set terms = new HashSet<>(); - for (Scorer.ChildScorable childScorer : childScorers) { - Query query = ((Scorer) childScorer.child).getWeight().getQuery(); - assertTrue(query instanceof TermQuery); - Term term = ((TermQuery) query).getTerm(); - assertEquals("field", term.field()); - terms.add(term.text()); - } - assertEquals(2, terms.size()); - assertTrue(terms.contains("a")); - assertTrue(terms.contains("b")); - } + public SimpleCollector newCollector() throws IOException { + return new SimpleCollector() { + @Override + public void setScorer(Scorable s) throws IOException { + Collection childScorers = s.getChildren(); + setScorerCalled.set(true); + assertEquals(2, childScorers.size()); + Set terms = new HashSet<>(); + for (Scorer.ChildScorable childScorer : childScorers) { + Query query = ((Scorer) childScorer.child).getWeight().getQuery(); + assertTrue(query instanceof TermQuery); + Term term = ((TermQuery) query).getTerm(); + assertEquals("field", term.field()); + terms.add(term.text()); + } + assertEquals(2, terms.size()); + assertTrue(terms.contains("a")); + assertTrue(terms.contains("b")); + } - @Override - public void collect(int doc) {} + @Override + public void collect(int doc) {} + + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE; + } + }; + } @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE; + public Object reduce(Collection collectors) throws IOException { + return null; } }); - assertTrue(setScorerCalled[0]); + + assertTrue(setScorerCalled.get()); IOUtils.close(r, w, dir); } } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.java b/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.java index 73535777f566..479d9a8b9b9f 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import org.apache.lucene.document.Document; import org.apache.lucene.document.DoubleDocValuesField; @@ -256,35 +257,49 @@ private void testExplanations(Query q, DoubleValuesSource vs) throws IOException DoubleValuesSource rewritten = vs.rewrite(searcher); searcher.search( q, - new SimpleCollector() { - - DoubleValues v; - LeafReaderContext ctx; - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - this.ctx = context; - } - - @Override - public void setScorer(Scorable scorer) throws IOException { - this.v = rewritten.getValues(this.ctx, DoubleValuesSource.fromScorer(scorer)); - } - + new CollectorManager() { @Override - public void collect(int doc) throws IOException { - Explanation scoreExpl = searcher.explain(q, ctx.docBase + doc); - if (this.v.advanceExact(doc)) { - CheckHits.verifyExplanation( - "", doc, (float) v.doubleValue(), true, rewritten.explain(ctx, doc, scoreExpl)); - } else { - assertFalse(rewritten.explain(ctx, doc, scoreExpl).isMatch()); - } + public SimpleCollector newCollector() throws IOException { + return new SimpleCollector() { + + DoubleValues v; + LeafReaderContext ctx; + + @Override + protected void doSetNextReader(LeafReaderContext context) throws IOException { + this.ctx = context; + } + + @Override + public void setScorer(Scorable scorer) throws IOException { + this.v = rewritten.getValues(this.ctx, DoubleValuesSource.fromScorer(scorer)); + } + + @Override + public void collect(int doc) throws IOException { + Explanation scoreExpl = searcher.explain(q, ctx.docBase + doc); + if (this.v.advanceExact(doc)) { + CheckHits.verifyExplanation( + "", + doc, + (float) v.doubleValue(), + true, + rewritten.explain(ctx, doc, scoreExpl)); + } else { + assertFalse(rewritten.explain(ctx, doc, scoreExpl).isMatch()); + } + } + + @Override + public ScoreMode scoreMode() { + return vs.needsScores() ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES; + } + }; } @Override - public ScoreMode scoreMode() { - return vs.needsScores() ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES; + public Object reduce(Collection collectors) throws IOException { + return null; } }); } @@ -310,32 +325,42 @@ private void doTestQueryDoubleValuesSources(Query q) throws Exception { DoubleValuesSource vs = DoubleValuesSource.fromQuery(q).rewrite(searcher); searcher.search( q, - new SimpleCollector() { - - DoubleValues v; - Scorable scorer; - LeafReaderContext ctx; - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - this.ctx = context; - } - - @Override - public void setScorer(Scorable scorer) throws IOException { - this.scorer = scorer; - this.v = vs.getValues(this.ctx, DoubleValuesSource.fromScorer(scorer)); - } - + new CollectorManager() { @Override - public void collect(int doc) throws IOException { - assertTrue(v.advanceExact(doc)); - assertEquals(scorer.score(), v.doubleValue(), 0.00001); + public SimpleCollector newCollector() throws IOException { + return new SimpleCollector() { + + DoubleValues v; + Scorable scorer; + LeafReaderContext ctx; + + @Override + protected void doSetNextReader(LeafReaderContext context) throws IOException { + this.ctx = context; + } + + @Override + public void setScorer(Scorable scorer) throws IOException { + this.scorer = scorer; + this.v = vs.getValues(this.ctx, DoubleValuesSource.fromScorer(scorer)); + } + + @Override + public void collect(int doc) throws IOException { + assertTrue(v.advanceExact(doc)); + assertEquals(scorer.score(), v.doubleValue(), 0.00001); + } + + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE; + } + }; } @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE; + public Object reduce(Collection collectors) throws IOException { + return null; } }); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestEarlyTermination.java b/lucene/core/src/test/org/apache/lucene/search/TestEarlyTermination.java index 3895812019e7..8f20b292fa27 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestEarlyTermination.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestEarlyTermination.java @@ -17,6 +17,7 @@ package org.apache.lucene.search; import java.io.IOException; +import java.util.Collection; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; @@ -56,37 +57,47 @@ public void testEarlyTermination() throws IOException { for (int i = 0; i < iters; ++i) { final IndexSearcher searcher = newSearcher(reader); - final Collector collector = - new SimpleCollector() { - - boolean collectionTerminated = true; + searcher.search( + new MatchAllDocsQuery(), + new CollectorManager() { @Override - public void collect(int doc) throws IOException { - assertFalse(collectionTerminated); - if (rarely()) { - collectionTerminated = true; - throw new CollectionTerminatedException(); - } - } + public SimpleCollector newCollector() throws IOException { + return new SimpleCollector() { - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - if (random().nextBoolean()) { - collectionTerminated = true; - throw new CollectionTerminatedException(); - } else { - collectionTerminated = false; - } + boolean collectionTerminated = true; + + @Override + public void collect(int doc) throws IOException { + assertFalse(collectionTerminated); + if (rarely()) { + collectionTerminated = true; + throw new CollectionTerminatedException(); + } + } + + @Override + protected void doSetNextReader(LeafReaderContext context) throws IOException { + if (random().nextBoolean()) { + collectionTerminated = true; + throw new CollectionTerminatedException(); + } else { + collectionTerminated = false; + } + } + + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; + } + }; } @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE_NO_SCORES; + public Object reduce(Collection collectors) throws IOException { + return null; } - }; - - searcher.search(new MatchAllDocsQuery(), collector); + }); } reader.close(); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java index 526990a54860..a7d5580ba5eb 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java @@ -163,9 +163,9 @@ public void run() { RandomPicks.randomFrom( random(), new String[] {"blue", "red", "yellow", "green"}); final Query q = new TermQuery(new Term("color", value)); - TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(q, collector); // will use the cache - final int totalHits1 = collector.getTotalHits(); + final int totalHits1 = + searcher.search( + q, new TotalHitCountCollectorManager()); // will use the cache TotalHitCountCollector collector2 = new TotalHitCountCollector(); searcher.search( q, @@ -902,7 +902,7 @@ public void onUse(Query query) { searcher.setQueryCache(queryCache); searcher.setQueryCachingPolicy(policy); - searcher.search(query.build(), new TotalHitCountCollector()); + searcher.search(query.build(), new TotalHitCountCollectorManager()); reader.close(); dir.close(); @@ -1120,7 +1120,7 @@ public void testDetectMutatedQueries() throws IOException { try { // trigger an eviction - searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollector()); + searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager()); fail(); } catch ( @SuppressWarnings("unused") diff --git a/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java b/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java index 9b4a6c880aa1..f6fcf4ce30c4 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java @@ -260,7 +260,7 @@ public int doTermConjunctions( } CountingHitCollector hc = new CountingHitCollector(); - s.search(bq.build(), hc); + nMatches += hc.getCount(); ret += hc.getSum(); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java b/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java index ac7b683ed106..8641a276b83f 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java @@ -17,6 +17,7 @@ package org.apache.lucene.search; import java.io.IOException; +import java.util.Collection; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -81,22 +82,32 @@ public void testSimilarity() throws Exception { searcher.search( new TermQuery(b), - new SimpleCollector() { - private Scorable scorer; - + new CollectorManager() { @Override - public void setScorer(Scorable scorer) { - this.scorer = scorer; + public SimpleCollector newCollector() throws IOException { + return new SimpleCollector() { + private Scorable scorer; + + @Override + public void setScorer(Scorable scorer) { + this.scorer = scorer; + } + + @Override + public final void collect(int doc) throws IOException { + assertEquals(1.0f, scorer.score(), 0); + } + + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE; + } + }; } @Override - public final void collect(int doc) throws IOException { - assertEquals(1.0f, scorer.score(), 0); - } - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE; + public Object reduce(Collection collectors) throws IOException { + return null; } }); @@ -106,29 +117,39 @@ public ScoreMode scoreMode() { // System.out.println(bq.toString("field")); searcher.search( bq.build(), - new SimpleCollector() { - private int base = 0; - private Scorable scorer; - + new CollectorManager() { @Override - public void setScorer(Scorable scorer) { - this.scorer = scorer; + public SimpleCollector newCollector() throws IOException { + return new SimpleCollector() { + private int base = 0; + private Scorable scorer; + + @Override + public void setScorer(Scorable scorer) { + this.scorer = scorer; + } + + @Override + public final void collect(int doc) throws IOException { + // System.out.println("Doc=" + doc + " score=" + score); + assertEquals((float) doc + base + 1, scorer.score(), 0); + } + + @Override + protected void doSetNextReader(LeafReaderContext context) throws IOException { + base = context.docBase; + } + + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE; + } + }; } @Override - public final void collect(int doc) throws IOException { - // System.out.println("Doc=" + doc + " score=" + score); - assertEquals((float) doc + base + 1, scorer.score(), 0); - } - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - base = context.docBase; - } - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE; + public Object reduce(Collection collectors) throws IOException { + return null; } }); @@ -136,23 +157,33 @@ public ScoreMode scoreMode() { // System.out.println(pq.toString("field")); searcher.search( pq, - new SimpleCollector() { - private Scorable scorer; - - @Override - public void setScorer(Scorable scorer) { - this.scorer = scorer; - } - + new CollectorManager() { @Override - public final void collect(int doc) throws IOException { - // System.out.println("Doc=" + doc + " score=" + score); - assertEquals(1.0f, scorer.score(), 0); + public SimpleCollector newCollector() throws IOException { + return new SimpleCollector() { + private Scorable scorer; + + @Override + public void setScorer(Scorable scorer) { + this.scorer = scorer; + } + + @Override + public final void collect(int doc) throws IOException { + // System.out.println("Doc=" + doc + " score=" + score); + assertEquals(1.0f, scorer.score(), 0); + } + + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE; + } + }; } @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE; + public Object reduce(Collection collectors) throws IOException { + return null; } }); @@ -160,23 +191,33 @@ public ScoreMode scoreMode() { // System.out.println(pq.toString("field")); searcher.search( pq, - new SimpleCollector() { - private Scorable scorer; - - @Override - public void setScorer(Scorable scorer) { - this.scorer = scorer; - } - + new CollectorManager() { @Override - public final void collect(int doc) throws IOException { - // System.out.println("Doc=" + doc + " score=" + score); - assertEquals(0.5f, scorer.score(), 0); + public SimpleCollector newCollector() throws IOException { + return new SimpleCollector() { + private Scorable scorer; + + @Override + public void setScorer(Scorable scorer) { + this.scorer = scorer; + } + + @Override + public final void collect(int doc) throws IOException { + // System.out.println("Doc=" + doc + " score=" + score); + assertEquals(0.5f, scorer.score(), 0); + } + + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE; + } + }; } @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE; + public Object reduce(Collection collectors) throws IOException { + return null; } }); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSloppyPhraseQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestSloppyPhraseQuery.java index f8fe426b9f8f..e7876ebafd94 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSloppyPhraseQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSloppyPhraseQuery.java @@ -17,6 +17,7 @@ package org.apache.lucene.search; import java.io.IOException; +import java.util.Collection; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; @@ -218,24 +219,35 @@ public ScoreMode scoreMode() { private void assertSaneScoring(PhraseQuery pq, IndexSearcher searcher) throws Exception { searcher.search( pq, - new SimpleCollector() { - Scorer scorer; - + new CollectorManager() { @Override - public void setScorer(Scorable scorer) { - this.scorer = (Scorer) AssertingScorable.unwrap(scorer); + public SimpleCollector newCollector() throws IOException { + return new SimpleCollector() { + Scorer scorer; + + @Override + public void setScorer(Scorable scorer) { + this.scorer = (Scorer) AssertingScorable.unwrap(scorer); + } + + @Override + public void collect(int doc) throws IOException { + assertFalse(Float.isInfinite(scorer.score())); + } + + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE; + } + }; } @Override - public void collect(int doc) throws IOException { - assertFalse(Float.isInfinite(scorer.score())); - } - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE; + public Object reduce(Collection collectors) throws IOException { + return null; } }); + QueryUtils.check(random(), pq, searcher); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java index d678cb96983a..f0bdce28f324 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java @@ -271,11 +271,12 @@ void testSort(boolean useFrom) throws Exception { final TopDocs topHits; if (sort == null) { if (useFrom) { - TopScoreDocCollector c = TopScoreDocCollector.create(numHits, Integer.MAX_VALUE); - searcher.search(query, c); + from = TestUtil.nextInt(random(), 0, numHits - 1); size = numHits - from; - TopDocs tempTopHits = c.topDocs(); + TopDocs tempTopHits = + searcher.search( + query, TopScoreDocCollectorManager.create(numHits, Integer.MAX_VALUE)); if (from < tempTopHits.scoreDocs.length) { // Can't use TopDocs#topDocs(start, howMany), since it has different behaviour when // start >= hitCount diff --git a/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java b/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java index fefec656971e..6d61ace9f6f1 100644 --- a/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java +++ b/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java @@ -31,6 +31,7 @@ import org.apache.lucene.facet.FacetResult; import org.apache.lucene.facet.Facets; import org.apache.lucene.facet.FacetsCollector; +import org.apache.lucene.facet.FacetsCollectorManager; import org.apache.lucene.facet.FacetsConfig; import org.apache.lucene.facet.range.DoubleRange; import org.apache.lucene.facet.range.DoubleRangeFacetCounts; @@ -210,10 +211,7 @@ public static Query getBoundingBoxQuery( /** User runs a query and counts facets. */ public FacetResult search() throws IOException { - - FacetsCollector fc = new FacetsCollector(); - - searcher.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = new DoubleRangeFacetCounts( diff --git a/lucene/demo/src/java/org/apache/lucene/demo/facet/SimpleFacetsExample.java b/lucene/demo/src/java/org/apache/lucene/demo/facet/SimpleFacetsExample.java index dc916fc2c102..b36689817310 100644 --- a/lucene/demo/src/java/org/apache/lucene/demo/facet/SimpleFacetsExample.java +++ b/lucene/demo/src/java/org/apache/lucene/demo/facet/SimpleFacetsExample.java @@ -28,6 +28,7 @@ import org.apache.lucene.facet.FacetResult; import org.apache.lucene.facet.Facets; import org.apache.lucene.facet.FacetsCollector; +import org.apache.lucene.facet.FacetsCollectorManager; import org.apache.lucene.facet.FacetsConfig; import org.apache.lucene.facet.taxonomy.FastTaxonomyFacetCounts; import org.apache.lucene.facet.taxonomy.TaxonomyReader; @@ -125,12 +126,10 @@ private List facetsOnly() throws IOException { IndexSearcher searcher = new IndexSearcher(indexReader); TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir); - FacetsCollector fc = new FacetsCollector(); - // MatchAllDocsQuery is for "browsing" (counts facets // for all non-deleted docs in the index); normally // you'd use a "normal" query: - searcher.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); // Retrieve results List results = new ArrayList<>(); diff --git a/lucene/facet/src/test/org/apache/lucene/facet/TestFacetsConfig.java b/lucene/facet/src/test/org/apache/lucene/facet/TestFacetsConfig.java index fc15cf240e0b..ff81f46c08e6 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/TestFacetsConfig.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/TestFacetsConfig.java @@ -73,8 +73,7 @@ public void testAddSameDocTwice() throws Exception { DirectoryReader indexReader = DirectoryReader.open(indexDir); DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir); IndexSearcher searcher = newSearcher(indexReader); - FacetsCollector fc = new FacetsCollector(); - searcher.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = getTaxonomyFacetCounts(taxoReader, facetsConfig, fc); FacetResult res = facets.getTopChildren(10, "a"); diff --git a/lucene/facet/src/test/org/apache/lucene/facet/TestLongValueFacetCounts.java b/lucene/facet/src/test/org/apache/lucene/facet/TestLongValueFacetCounts.java index aab1c15e6b0d..9e34ab3c7b4b 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/TestLongValueFacetCounts.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/TestLongValueFacetCounts.java @@ -56,9 +56,8 @@ public void testBasic() throws Exception { IndexReader r = w.getReader(); w.close(); - FacetsCollector fc = new FacetsCollector(); IndexSearcher s = newSearcher(r); - s.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); LongValueFacetCounts facets = new LongValueFacetCounts("field", fc); @@ -83,9 +82,8 @@ public void testOnlyBigLongs() throws Exception { IndexReader r = w.getReader(); w.close(); - FacetsCollector fc = new FacetsCollector(); IndexSearcher s = newSearcher(r); - s.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); LongValueFacetCounts facets = new LongValueFacetCounts("field", fc); @@ -115,9 +113,8 @@ public void testGetAllDims() throws Exception { IndexReader r = w.getReader(); w.close(); - FacetsCollector fc = new FacetsCollector(); IndexSearcher s = newSearcher(r); - s.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = new LongValueFacetCounts("field", fc); @@ -176,7 +173,7 @@ public void testRandomSingleValued() throws Exception { int iters = atLeast(100); for (int iter = 0; iter < iters; iter++) { - FacetsCollector fc = new FacetsCollector(); + FacetsCollector fc; if (VERBOSE) { System.out.println("\nTEST: iter=" + iter); System.out.println(" test all docs"); @@ -203,7 +200,7 @@ public void testRandomSingleValued() throws Exception { LongValueFacetCounts facetCounts; if (random().nextBoolean()) { - s.search(new MatchAllDocsQuery(), fc); + fc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); if (random().nextBoolean()) { if (VERBOSE) { System.out.println(" use value source"); @@ -281,8 +278,8 @@ public void testRandomSingleValued() throws Exception { System.out.println(" test id range " + minId + "-" + maxId); } - fc = new FacetsCollector(); - s.search(IntPoint.newRangeQuery("id", minId, maxId), fc); + fc = s.search(IntPoint.newRangeQuery("id", minId, maxId), new FacetsCollectorManager()); + if (random().nextBoolean()) { if (VERBOSE) { System.out.println(" use doc values"); @@ -420,7 +417,7 @@ public void testRandomMultiValued() throws Exception { int iters = atLeast(100); for (int iter = 0; iter < iters; iter++) { - FacetsCollector fc = new FacetsCollector(); + FacetsCollector fc; if (VERBOSE) { System.out.println("\nTEST: iter=" + iter); System.out.println(" test all docs"); @@ -444,7 +441,7 @@ public void testRandomMultiValued() throws Exception { LongValueFacetCounts facetCounts; if (random().nextBoolean()) { - s.search(new MatchAllDocsQuery(), fc); + fc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); if (VERBOSE) { System.out.println(" use doc values"); } @@ -506,8 +503,8 @@ public void testRandomMultiValued() throws Exception { System.out.println(" test id range " + minId + "-" + maxId); } - fc = new FacetsCollector(); - s.search(IntPoint.newRangeQuery("id", minId, maxId), fc); + fc = s.search(IntPoint.newRangeQuery("id", minId, maxId), new FacetsCollectorManager()); + // cannot use value source here because we are multi valued facetCounts = new LongValueFacetCounts("field", fc); diff --git a/lucene/facet/src/test/org/apache/lucene/facet/TestStringValueFacetCounts.java b/lucene/facet/src/test/org/apache/lucene/facet/TestStringValueFacetCounts.java index 7d7d88889c85..d1138be1f144 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/TestStringValueFacetCounts.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/TestStringValueFacetCounts.java @@ -191,8 +191,7 @@ public void testStaleState() throws Exception { IndexSearcher searcher = newSearcher(writer.getReader()); writer.close(); - FacetsCollector c = new FacetsCollector(); - searcher.search(new MatchAllDocsQuery(), c); + FacetsCollector c = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); // using a stale state expectThrows(IllegalStateException.class, () -> new StringValueFacetCounts(state, c)); @@ -259,8 +258,7 @@ private void checkFacetResult( StringDocValuesReaderState state = new StringDocValuesReaderState(searcher.getIndexReader(), "field"); - FacetsCollector c = new FacetsCollector(); - searcher.search(new MatchAllDocsQuery(), c); + FacetsCollector c = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); for (int topN : topNs) { diff --git a/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java b/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java index e7959101ce7f..47ff14bbdcda 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java @@ -36,6 +36,7 @@ import org.apache.lucene.facet.FacetTestCase; import org.apache.lucene.facet.Facets; import org.apache.lucene.facet.FacetsCollector; +import org.apache.lucene.facet.FacetsCollectorManager; import org.apache.lucene.facet.FacetsConfig; import org.apache.lucene.facet.LabelAndValue; import org.apache.lucene.facet.MultiFacets; @@ -83,9 +84,8 @@ public void testBasicLong() throws Exception { IndexReader r = w.getReader(); w.close(); - FacetsCollector fc = new FacetsCollector(); IndexSearcher s = newSearcher(r); - s.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = new LongRangeFacetCounts( @@ -129,9 +129,8 @@ public void testBasicLongMultiValued() throws Exception { IndexReader r = w.getReader(); w.close(); - FacetsCollector fc = new FacetsCollector(); IndexSearcher s = newSearcher(r); - s.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = new LongRangeFacetCounts( @@ -180,9 +179,8 @@ public void testBasicLongMultiValuedMixedSegmentTypes() throws Exception { IndexReader r = w.getReader(); w.close(); - FacetsCollector fc = new FacetsCollector(); IndexSearcher s = newSearcher(r); - s.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = new LongRangeFacetCounts( @@ -221,9 +219,8 @@ public void testLongGetAllDims() throws Exception { IndexReader r = w.getReader(); w.close(); - FacetsCollector fc = new FacetsCollector(); IndexSearcher s = newSearcher(r); - s.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = new LongRangeFacetCounts( @@ -285,9 +282,8 @@ public void testLongMinMax() throws Exception { IndexReader r = w.getReader(); w.close(); - FacetsCollector fc = new FacetsCollector(); IndexSearcher s = newSearcher(r); - s.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = new LongRangeFacetCounts( @@ -325,9 +321,8 @@ public void testOverlappedEndStart() throws Exception { IndexReader r = w.getReader(); w.close(); - FacetsCollector fc = new FacetsCollector(); IndexSearcher s = newSearcher(r); - s.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = new LongRangeFacetCounts( @@ -361,9 +356,8 @@ public void testEmptyRangesSingleValued() throws Exception { IndexReader r = w.getReader(); w.close(); - FacetsCollector fc = new FacetsCollector(); IndexSearcher s = newSearcher(r); - s.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = new LongRangeFacetCounts("field", fc); @@ -391,9 +385,8 @@ public void testEmptyRangesMultiValued() throws Exception { IndexReader r = w.getReader(); w.close(); - FacetsCollector fc = new FacetsCollector(); IndexSearcher s = newSearcher(r); - s.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = new LongRangeFacetCounts("field", fc); @@ -537,10 +530,9 @@ public void testBasicDouble() throws Exception { IndexReader r = w.getReader(); - FacetsCollector fc = new FacetsCollector(); - IndexSearcher s = newSearcher(r); - s.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); + Facets facets = new DoubleRangeFacetCounts( "field", @@ -575,10 +567,9 @@ public void testBasicDoubleMultiValued() throws Exception { IndexReader r = w.getReader(); - FacetsCollector fc = new FacetsCollector(); - IndexSearcher s = newSearcher(r); - s.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); + Facets facets = new DoubleRangeFacetCounts( "field", @@ -624,9 +615,8 @@ public void testBasicDoubleMultiValuedMixedSegmentTypes() throws Exception { IndexReader r = w.getReader(); w.close(); - FacetsCollector fc = new FacetsCollector(); IndexSearcher s = newSearcher(r); - s.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = new DoubleRangeFacetCounts( @@ -754,8 +744,8 @@ public void testRandomLongsSingleValued() throws Exception { } } - FacetsCollector sfc = new FacetsCollector(); - s.search(new MatchAllDocsQuery(), sfc); + FacetsCollector sfc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); + Query fastMatchQuery; if (random().nextBoolean()) { if (random().nextBoolean()) { @@ -912,8 +902,8 @@ public void testRandomLongsMultiValued() throws Exception { } } - FacetsCollector sfc = new FacetsCollector(); - s.search(new MatchAllDocsQuery(), sfc); + FacetsCollector sfc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); + Query fastMatchQuery; if (random().nextBoolean()) { if (random().nextBoolean()) { @@ -1056,8 +1046,8 @@ public void testRandomDoublesSingleValued() throws Exception { } } - FacetsCollector sfc = new FacetsCollector(); - s.search(new MatchAllDocsQuery(), sfc); + FacetsCollector sfc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); + Query fastMatchFilter; if (random().nextBoolean()) { if (random().nextBoolean()) { @@ -1210,8 +1200,8 @@ public void testRandomDoublesMultiValued() throws Exception { } } - FacetsCollector sfc = new FacetsCollector(); - s.search(new MatchAllDocsQuery(), sfc); + FacetsCollector sfc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); + Query fastMatchFilter; if (random().nextBoolean()) { if (random().nextBoolean()) { @@ -1273,10 +1263,9 @@ public void testMissingValues() throws Exception { IndexReader r = w.getReader(); - FacetsCollector fc = new FacetsCollector(); - IndexSearcher s = newSearcher(r); - s.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); + Facets facets = new LongRangeFacetCounts( "field", @@ -1317,10 +1306,9 @@ public void testMissingValuesMultiValued() throws Exception { IndexReader r = w.getReader(); - FacetsCollector fc = new FacetsCollector(); - IndexSearcher s = newSearcher(r); - s.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); + Facets facets = new LongRangeFacetCounts( "field", @@ -1465,12 +1453,10 @@ public void testCustomDoubleValuesSource() throws Exception { FacetsConfig config = new FacetsConfig(); - FacetsCollector fc = new FacetsCollector(); - IndexReader r = writer.getReader(); IndexSearcher s = newSearcher(r, false, false); - s.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = s.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); final DoubleRange[] ranges = new DoubleRange[] { diff --git a/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java b/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java index 83caaacbbfe7..f19396adef16 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java @@ -33,6 +33,7 @@ import org.apache.lucene.facet.FacetTestCase; import org.apache.lucene.facet.Facets; import org.apache.lucene.facet.FacetsCollector; +import org.apache.lucene.facet.FacetsCollectorManager; import org.apache.lucene.facet.FacetsConfig; import org.apache.lucene.facet.LabelAndValue; import org.apache.lucene.index.IndexReader; @@ -220,9 +221,7 @@ public void testStaleState() throws Exception { IndexSearcher searcher = newSearcher(writer.getReader()); - FacetsCollector c = new FacetsCollector(); - - searcher.search(new MatchAllDocsQuery(), c); + FacetsCollector c = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); expectThrows( IllegalStateException.class, @@ -487,8 +486,7 @@ private static Facets getAllFacets( IndexSearcher searcher, SortedSetDocValuesReaderState state, ExecutorService exec) throws IOException, InterruptedException { if (random().nextBoolean()) { - FacetsCollector c = new FacetsCollector(); - searcher.search(new MatchAllDocsQuery(), c); + FacetsCollector c = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); if (exec != null) { return new ConcurrentSortedSetDocValuesFacetCounts(state, c, exec); } else { diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestSearcherTaxonomyManager.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestSearcherTaxonomyManager.java index 2bb98f684bb2..7e410f183747 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestSearcherTaxonomyManager.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestSearcherTaxonomyManager.java @@ -29,6 +29,7 @@ import org.apache.lucene.facet.FacetTestCase; import org.apache.lucene.facet.Facets; import org.apache.lucene.facet.FacetsCollector; +import org.apache.lucene.facet.FacetsCollectorManager; import org.apache.lucene.facet.FacetsConfig; import org.apache.lucene.facet.taxonomy.SearcherTaxonomyManager.SearcherAndTaxonomy; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; @@ -184,8 +185,8 @@ public void run() { SearcherAndTaxonomy pair = mgr.acquire(); try { // System.out.println("search maxOrd=" + pair.taxonomyReader.getSize()); - FacetsCollector sfc = new FacetsCollector(); - pair.searcher.search(new MatchAllDocsQuery(), sfc); + FacetsCollector sfc = + pair.searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = getTaxonomyFacetCounts(pair.taxonomyReader, config, sfc); FacetResult result = facets.getTopChildren(10, "field"); if (pair.searcher.getIndexReader().numDocs() > 0) { @@ -239,8 +240,8 @@ public void testDirectory() throws Exception { SearcherAndTaxonomy pair = mgr.acquire(); try { // System.out.println("search maxOrd=" + pair.taxonomyReader.getSize()); - FacetsCollector sfc = new FacetsCollector(); - pair.searcher.search(new MatchAllDocsQuery(), sfc); + FacetsCollector sfc = + pair.searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = getTaxonomyFacetCounts(pair.taxonomyReader, config, sfc); FacetResult result = facets.getTopChildren(10, "field"); if (pair.searcher.getIndexReader().numDocs() > 0) { diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetAssociations.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetAssociations.java index a07b4e11eae3..e6376b3c649e 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetAssociations.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetAssociations.java @@ -21,6 +21,7 @@ import org.apache.lucene.facet.FacetTestCase; import org.apache.lucene.facet.Facets; import org.apache.lucene.facet.FacetsCollector; +import org.apache.lucene.facet.FacetsCollectorManager; import org.apache.lucene.facet.FacetsConfig; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; @@ -96,10 +97,8 @@ public static void afterClass() throws Exception { public void testIntSumAssociation() throws Exception { - FacetsCollector fc = new FacetsCollector(); - IndexSearcher searcher = newSearcher(reader); - searcher.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = new TaxonomyFacetSumIntAssociations("$facets.int", taxoReader, config, fc); assertEquals( @@ -112,10 +111,8 @@ public void testIntSumAssociation() throws Exception { } public void testFloatSumAssociation() throws Exception { - FacetsCollector fc = new FacetsCollector(); - IndexSearcher searcher = newSearcher(reader); - searcher.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = new TaxonomyFacetSumFloatAssociations("$facets.float", taxoReader, config, fc); assertEquals( @@ -138,10 +135,8 @@ public void testFloatSumAssociation() throws Exception { * different field. */ public void testIntAndFloatAssocation() throws Exception { - FacetsCollector fc = new FacetsCollector(); - IndexSearcher searcher = newSearcher(reader); - searcher.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = new TaxonomyFacetSumFloatAssociations("$facets.float", taxoReader, config, fc); assertEquals( @@ -163,10 +158,9 @@ public void testIntAndFloatAssocation() throws Exception { } public void testWrongIndexFieldName() throws Exception { - FacetsCollector fc = new FacetsCollector(); - IndexSearcher searcher = newSearcher(reader); - searcher.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); + Facets facets = new TaxonomyFacetSumFloatAssociations(taxoReader, config, fc); expectThrows( IllegalArgumentException.class, @@ -244,12 +238,11 @@ public void testRequireDimCount() throws Exception { } public void testIntSumAssociationDrillDown() throws Exception { - FacetsCollector fc = new FacetsCollector(); - IndexSearcher searcher = newSearcher(reader); DrillDownQuery q = new DrillDownQuery(config); q.add("int", "b"); - searcher.search(q, fc); + IndexSearcher searcher = newSearcher(reader); + FacetsCollector fc = searcher.search(q, new FacetsCollectorManager()); Facets facets = new TaxonomyFacetSumIntAssociations("$facets.int", taxoReader, config, fc); assertEquals( diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java index 944860dfab80..a73d2b243f0f 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java @@ -36,6 +36,7 @@ import org.apache.lucene.facet.FacetTestCase; import org.apache.lucene.facet.Facets; import org.apache.lucene.facet.FacetsCollector; +import org.apache.lucene.facet.FacetsCollectorManager; import org.apache.lucene.facet.FacetsConfig; import org.apache.lucene.facet.LabelAndValue; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; @@ -122,8 +123,7 @@ public void testBasic() throws Exception { // Now user drills down on Publish Date/2010: DrillDownQuery q2 = new DrillDownQuery(config); q2.add("Publish Date", "2010"); - FacetsCollector c = new FacetsCollector(); - searcher.search(q2, c); + FacetsCollector c = searcher.search(q2, new FacetsCollectorManager()); facets = new FastTaxonomyFacetCounts(taxoReader, config, c); assertEquals( "dim=Author path=[] value=2 childCount=2\n Bob (1)\n Lisa (1)\n", @@ -231,8 +231,7 @@ public void testWrongIndexFieldName() throws Exception { // NRT open TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter); - FacetsCollector c = new FacetsCollector(); - searcher.search(new MatchAllDocsQuery(), c); + FacetsCollector c = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); // Uses default $facets field: Facets facets; @@ -568,8 +567,8 @@ public void testSeparateIndexedFields() throws Exception { DirectoryReader r = DirectoryReader.open(iw); DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter); - FacetsCollector sfc = new FacetsCollector(); - newSearcher(r).search(new MatchAllDocsQuery(), sfc); + FacetsCollector sfc = + newSearcher(r).search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets1 = getTaxonomyFacetCounts(taxoReader, config, sfc); Facets facets2 = getTaxonomyFacetCounts(taxoReader, config, sfc, "$b"); assertEquals(r.maxDoc(), facets1.getTopChildren(10, "a").value.intValue()); @@ -710,8 +709,7 @@ public void testSegmentsWithoutCategoriesOrResults() throws Exception { // search for "f:a", only segments 1 and 3 should match results Query q = new TermQuery(new Term("f", "a")); - FacetsCollector sfc = new FacetsCollector(); - indexSearcher.search(q, sfc); + FacetsCollector sfc = indexSearcher.search(q, new FacetsCollectorManager()); Facets facets = getTaxonomyFacetCounts(taxoReader, config, sfc); FacetResult result = facets.getTopChildren(10, "A"); assertEquals("wrong number of children", 2, result.labelValues.length); @@ -869,14 +867,11 @@ private static Facets getAllFacets( String indexFieldName, IndexSearcher searcher, TaxonomyReader taxoReader, FacetsConfig config) throws IOException { if (random().nextBoolean()) { - // Aggregate the facet counts: - FacetsCollector c = new FacetsCollector(); - // MatchAllDocsQuery is for "browsing" (counts facets // for all non-deleted docs in the index); normally // you'd use a "normal" query, and use MultiCollector to // wrap collecting the "normal" hits and also facets: - searcher.search(new MatchAllDocsQuery(), c); + FacetsCollector c = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); return new FastTaxonomyFacetCounts(taxoReader, config, c); } else { diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts2.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts2.java index 8642a347bcbb..ab6e279b6525 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts2.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts2.java @@ -33,6 +33,7 @@ import org.apache.lucene.facet.FacetTestCase; import org.apache.lucene.facet.Facets; import org.apache.lucene.facet.FacetsCollector; +import org.apache.lucene.facet.FacetsCollectorManager; import org.apache.lucene.facet.FacetsConfig; import org.apache.lucene.facet.LabelAndValue; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; @@ -274,9 +275,8 @@ public void testDifferentNumResults() throws Exception { TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir); IndexSearcher searcher = newSearcher(indexReader); - FacetsCollector sfc = new FacetsCollector(); TermQuery q = new TermQuery(A); - searcher.search(q, sfc); + FacetsCollector sfc = searcher.search(q, new FacetsCollectorManager()); Facets facets = getTaxonomyFacetCounts(taxoReader, getConfig(), sfc); FacetResult result = facets.getTopChildren(NUM_CHILDREN_CP_A, CP_A); assertEquals(-1, result.value.intValue()); @@ -298,8 +298,7 @@ public void testAllCounts() throws Exception { TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir); IndexSearcher searcher = newSearcher(indexReader); - FacetsCollector sfc = new FacetsCollector(); - searcher.search(new MatchAllDocsQuery(), sfc); + FacetsCollector sfc = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = getTaxonomyFacetCounts(taxoReader, getConfig(), sfc); @@ -340,8 +339,7 @@ public void testBigNumResults() throws Exception { TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir); IndexSearcher searcher = newSearcher(indexReader); - FacetsCollector sfc = new FacetsCollector(); - searcher.search(new MatchAllDocsQuery(), sfc); + FacetsCollector sfc = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = getTaxonomyFacetCounts(taxoReader, getConfig(), sfc); @@ -365,8 +363,7 @@ public void testNoParents() throws Exception { TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir); IndexSearcher searcher = newSearcher(indexReader); - FacetsCollector sfc = new FacetsCollector(); - searcher.search(new MatchAllDocsQuery(), sfc); + FacetsCollector sfc = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = getTaxonomyFacetCounts(taxoReader, getConfig(), sfc); diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetLabels.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetLabels.java index bdca7d826608..8464406379b3 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetLabels.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetLabels.java @@ -27,6 +27,7 @@ import org.apache.lucene.facet.FacetTestCase; import org.apache.lucene.facet.FacetsCollector; import org.apache.lucene.facet.FacetsCollector.MatchingDocs; +import org.apache.lucene.facet.FacetsCollectorManager; import org.apache.lucene.facet.FacetsConfig; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter; @@ -148,8 +149,7 @@ public void testBasic() throws Exception { // NRT open TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter); - FacetsCollector fc = new FacetsCollector(); - searcher.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); TaxonomyFacetLabels taxoLabels = new TaxonomyFacetLabels(taxoReader, FacetsConfig.DEFAULT_INDEX_FIELD_NAME); diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetSumValueSource.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetSumValueSource.java index 8c3cdaf8e011..015b72c68946 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetSumValueSource.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetSumValueSource.java @@ -31,6 +31,7 @@ import org.apache.lucene.facet.FacetTestCase; import org.apache.lucene.facet.Facets; import org.apache.lucene.facet.FacetsCollector; +import org.apache.lucene.facet.FacetsCollectorManager; import org.apache.lucene.facet.FacetsConfig; import org.apache.lucene.facet.LabelAndValue; import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader; @@ -104,14 +105,11 @@ public void testBasic() throws Exception { TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter); taxoWriter.close(); - // Aggregate the facet counts: - FacetsCollector c = new FacetsCollector(); - // MatchAllDocsQuery is for "browsing" (counts facets // for all non-deleted docs in the index); normally // you'd use a "normal" query and one of the // Facets.search utility methods: - searcher.search(new MatchAllDocsQuery(), c); + FacetsCollector c = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); TaxonomyFacetSumValueSource facets = new TaxonomyFacetSumValueSource( @@ -175,8 +173,7 @@ public void testSparseFacets() throws Exception { TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter); taxoWriter.close(); - FacetsCollector c = new FacetsCollector(); - searcher.search(new MatchAllDocsQuery(), c); + FacetsCollector c = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); TaxonomyFacetSumValueSource facets = new TaxonomyFacetSumValueSource( @@ -226,8 +223,7 @@ public void testWrongIndexFieldName() throws Exception { TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter); taxoWriter.close(); - FacetsCollector c = new FacetsCollector(); - searcher.search(new MatchAllDocsQuery(), c); + FacetsCollector c = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); TaxonomyFacetSumValueSource facets = new TaxonomyFacetSumValueSource( @@ -305,8 +301,8 @@ public void testNoScore() throws Exception { DirectoryReader r = DirectoryReader.open(iw); DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter); - FacetsCollector sfc = new FacetsCollector(); - newSearcher(r).search(new MatchAllDocsQuery(), sfc); + FacetsCollector sfc = + newSearcher(r).search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = new TaxonomyFacetSumValueSource( taxoReader, config, sfc, DoubleValuesSource.fromLongField("price")); @@ -372,8 +368,8 @@ public void testRollupValues() throws Exception { DirectoryReader r = DirectoryReader.open(iw); DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter); - FacetsCollector sfc = new FacetsCollector(); - newSearcher(r).search(new MatchAllDocsQuery(), sfc); + FacetsCollector sfc = + newSearcher(r).search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = new TaxonomyFacetSumValueSource( taxoReader, config, sfc, DoubleValuesSource.fromLongField("price")); diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighterPhrase.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighterPhrase.java index 33d3ee31ba54..c19ee0d90913 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighterPhrase.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighterPhrase.java @@ -33,7 +33,6 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; import org.apache.lucene.queries.spans.SpanNearQuery; import org.apache.lucene.queries.spans.SpanQuery; @@ -41,12 +40,10 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.Scorable; -import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.SimpleCollector; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.FixedBitSetCollector; import org.apache.lucene.util.LuceneTestCase; public class TestHighlighterPhrase extends LuceneTestCase { @@ -123,32 +120,8 @@ public void testConcurrentSpan() throws IOException, InvalidTokenOffsetsExceptio }, 0, true); - final FixedBitSet bitset = new FixedBitSet(indexReader.maxDoc()); - indexSearcher.search( - phraseQuery, - new SimpleCollector() { - private int baseDoc; - - @Override - public void collect(int i) { - bitset.set(this.baseDoc + i); - } - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - this.baseDoc = context.docBase; - } - - @Override - public void setScorer(Scorable scorer) { - // Do Nothing - } - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE_NO_SCORES; - } - }); + final FixedBitSet bitset = + indexSearcher.search(phraseQuery, FixedBitSetCollector.create(indexReader.maxDoc())); assertEquals(1, bitset.cardinality()); final int maxDoc = indexReader.maxDoc(); final Highlighter highlighter = diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java index f5431af816bb..542131dc7f5e 100644 --- a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java +++ b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java @@ -681,15 +681,14 @@ public void testMinMaxDocs() throws Exception { Query joinQuery = JoinUtil.createJoinQuery( "join_field", fromQuery, toQuery, searcher, scoreMode, ordinalMap, min, max); - TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(joinQuery, collector); + int totalHits = searcher.search(joinQuery, new TotalHitCountCollectorManager()); int expectedCount = 0; for (int numChildDocs : childDocsPerParent) { if (numChildDocs >= min && numChildDocs <= max) { expectedCount++; } } - assertEquals(expectedCount, collector.getTotalHits()); + assertEquals(expectedCount, totalHits); } searcher.getIndexReader().close(); dir.close(); diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/TestIndexAndTaxonomyReplicationClient.java b/lucene/replicator/src/test/org/apache/lucene/replicator/TestIndexAndTaxonomyReplicationClient.java index 5da165ca5c62..bc10b98ecf40 100644 --- a/lucene/replicator/src/test/org/apache/lucene/replicator/TestIndexAndTaxonomyReplicationClient.java +++ b/lucene/replicator/src/test/org/apache/lucene/replicator/TestIndexAndTaxonomyReplicationClient.java @@ -30,6 +30,7 @@ import org.apache.lucene.facet.FacetField; import org.apache.lucene.facet.Facets; import org.apache.lucene.facet.FacetsCollector; +import org.apache.lucene.facet.FacetsCollectorManager; import org.apache.lucene.facet.FacetsConfig; import org.apache.lucene.facet.taxonomy.FastTaxonomyFacetCounts; import org.apache.lucene.facet.taxonomy.TaxonomyReader; @@ -109,8 +110,7 @@ public Boolean call() throws Exception { // verify faceted search int id = Integer.parseInt(indexReader.getIndexCommit().getUserData().get(VERSION_ID), 16); IndexSearcher searcher = new IndexSearcher(indexReader); - FacetsCollector fc = new FacetsCollector(); - searcher.search(new MatchAllDocsQuery(), fc); + FacetsCollector fc = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = new FastTaxonomyFacetCounts(taxoReader, config, fc); assertEquals(1, facets.getSpecificValue("A", Integer.toString(id, 16)).intValue()); diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.java index 3dcd17057c4a..1b3c138b377f 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.java +++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.java @@ -32,7 +32,7 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TopScoreDocCollector; +import org.apache.lucene.search.TopScoreDocCollectorManager; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; @@ -83,13 +83,13 @@ public void testRequestLessHitsThanCollected() throws Exception { public void testIllegalArguments() throws IOException { IndexSearcher searcher = newSearcher(reader); LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(15); - TopScoreDocCollector regularCollector = - TopScoreDocCollector.create(15, null, Integer.MAX_VALUE); + TopScoreDocCollectorManager regularCollectorManager = + new TopScoreDocCollectorManager(15, null, Integer.MAX_VALUE); searcher.search(testQuery, largeCollector); - searcher.search(testQuery, regularCollector); + TopDocs topDocs = searcher.search(testQuery, regularCollectorManager); - assertEquals(largeCollector.totalHits, regularCollector.getTotalHits()); + assertEquals(largeCollector.totalHits, topDocs.totalHits.value); IllegalArgumentException expected = expectThrows( @@ -104,13 +104,13 @@ public void testIllegalArguments() throws IOException { public void testNoPQBuild() throws IOException { IndexSearcher searcher = newSearcher(reader); LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(250_000); - TopScoreDocCollector regularCollector = - TopScoreDocCollector.create(250_000, null, Integer.MAX_VALUE); + TopScoreDocCollectorManager regularCollectorManager = + new TopScoreDocCollectorManager(250_000, null, Integer.MAX_VALUE); searcher.search(testQuery, largeCollector); - searcher.search(testQuery, regularCollector); + TopDocs topDocs = searcher.search(testQuery, regularCollectorManager); - assertEquals(largeCollector.totalHits, regularCollector.getTotalHits()); + assertEquals(largeCollector.totalHits, topDocs.totalHits.value); assertEquals(largeCollector.pq, null); assertEquals(largeCollector.pqTop, null); @@ -119,13 +119,13 @@ public void testNoPQBuild() throws IOException { public void testPQBuild() throws IOException { IndexSearcher searcher = newSearcher(reader); LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(50); - TopScoreDocCollector regularCollector = - TopScoreDocCollector.create(50, null, Integer.MAX_VALUE); + TopScoreDocCollectorManager regularCollectorManager = + new TopScoreDocCollectorManager(50, null, Integer.MAX_VALUE); searcher.search(testQuery, largeCollector); - searcher.search(testQuery, regularCollector); + TopDocs topDocs = searcher.search(testQuery, regularCollectorManager); - assertEquals(largeCollector.totalHits, regularCollector.getTotalHits()); + assertEquals(largeCollector.totalHits, topDocs.totalHits.value); assertNotEquals(largeCollector.pq, null); assertNotEquals(largeCollector.pqTop, null); @@ -134,18 +134,18 @@ public void testPQBuild() throws IOException { public void testNoPQHitsOrder() throws IOException { IndexSearcher searcher = newSearcher(reader); LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(250_000); - TopScoreDocCollector regularCollector = - TopScoreDocCollector.create(250_000, null, Integer.MAX_VALUE); + TopScoreDocCollectorManager regularCollectorManager = + new TopScoreDocCollectorManager(250_000, null, Integer.MAX_VALUE); searcher.search(testQuery, largeCollector); - searcher.search(testQuery, regularCollector); + TopDocs topDocs = searcher.search(testQuery, regularCollectorManager); - assertEquals(largeCollector.totalHits, regularCollector.getTotalHits()); + assertEquals(largeCollector.totalHits, topDocs.totalHits.value); assertEquals(largeCollector.pq, null); assertEquals(largeCollector.pqTop, null); - TopDocs topDocs = largeCollector.topDocs(); + topDocs = largeCollector.topDocs(); if (topDocs.scoreDocs.length > 0) { float preScore = topDocs.scoreDocs[0].score; @@ -159,19 +159,18 @@ public void testNoPQHitsOrder() throws IOException { private void runNumHits(int numHits) throws IOException { IndexSearcher searcher = newSearcher(reader); LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(numHits); - TopScoreDocCollector regularCollector = - TopScoreDocCollector.create(numHits, null, Integer.MAX_VALUE); + TopScoreDocCollectorManager regularCollector = + new TopScoreDocCollectorManager(numHits, null, Integer.MAX_VALUE); searcher.search(testQuery, largeCollector); - searcher.search(testQuery, regularCollector); + TopDocs topDocs = searcher.search(testQuery, regularCollector); - assertEquals(largeCollector.totalHits, regularCollector.getTotalHits()); + assertEquals(largeCollector.totalHits, topDocs.totalHits.value); TopDocs firstTopDocs = largeCollector.topDocs(); - TopDocs secondTopDocs = regularCollector.topDocs(); - assertEquals(firstTopDocs.scoreDocs.length, secondTopDocs.scoreDocs.length); + assertEquals(firstTopDocs.scoreDocs.length, topDocs.scoreDocs.length); - CheckHits.checkEqual(testQuery, firstTopDocs.scoreDocs, secondTopDocs.scoreDocs); + CheckHits.checkEqual(testQuery, firstTopDocs.scoreDocs, topDocs.scoreDocs); } } diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestHeatmapFacetCounter.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestHeatmapFacetCounter.java index 8a2f72d0f637..30294f7b23c2 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestHeatmapFacetCounter.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestHeatmapFacetCounter.java @@ -24,7 +24,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.lucene.search.Query; -import org.apache.lucene.search.TotalHitCountCollector; +import org.apache.lucene.search.TotalHitCountCollectorManager; import org.apache.lucene.spatial.StrategyTestCase; import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; @@ -282,13 +282,12 @@ private int countMatchingDocsAtLevel(Point pt, int facetLevel) throws IOExceptio Query filter = new IntersectsPrefixTreeQuery( pt, strategy.getFieldName(), grid, facetLevel, grid.getMaxLevels()); - final TotalHitCountCollector collector = new TotalHitCountCollector(); - indexSearcher.search(filter, collector); + int totalHits = indexSearcher.search(filter, new TotalHitCountCollectorManager()); cellsValidated++; - if (collector.getTotalHits() > 0) { + if (totalHits > 0) { cellValidatedNonZero++; } - return collector.getTotalHits(); + return totalHits; } private Shape randomIndexedShape() { diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestNumberRangeFacets.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestNumberRangeFacets.java index 69418d2ec595..a6133da78121 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestNumberRangeFacets.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestNumberRangeFacets.java @@ -24,10 +24,7 @@ import java.util.Calendar; import java.util.Collections; import java.util.List; -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.SimpleCollector; import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.spatial.StrategyTestCase; import org.apache.lucene.spatial.prefix.NumberRangePrefixTreeStrategy.Facets; @@ -38,7 +35,7 @@ import org.apache.lucene.spatial.prefix.tree.NumberRangePrefixTree.UnitNRShape; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.FixedBitSetCollector; import org.junit.Before; import org.junit.Test; import org.locationtech.spatial4j.shape.Shape; @@ -219,28 +216,8 @@ public void test() throws IOException { } private Bits searchForDocBits(Query query) throws IOException { - FixedBitSet bitSet = new FixedBitSet(indexSearcher.getIndexReader().maxDoc()); - indexSearcher.search( - query, - new SimpleCollector() { - int leafDocBase; - - @Override - public void collect(int doc) throws IOException { - bitSet.set(leafDocBase + doc); - } - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - leafDocBase = context.docBase; - } - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE_NO_SCORES; - } - }); - return bitSet; + return indexSearcher.search( + query, FixedBitSetCollector.create(indexSearcher.getIndexReader().maxDoc())); } private void preQueryHavoc() { diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index a2e5a9fa912d..cef9ab041ba6 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -45,7 +45,6 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.PointValues.IntersectVisitor; @@ -57,8 +56,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.SimpleCollector; import org.apache.lucene.spatial3d.geom.GeoArea; import org.apache.lucene.spatial3d.geom.GeoAreaFactory; import org.apache.lucene.spatial3d.geom.GeoBBoxFactory; @@ -77,6 +74,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.DocIdSetBuilder; import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.FixedBitSetCollector; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.NumericUtils; @@ -1039,29 +1037,7 @@ private static void verify(double[] lats, double[] lons, final PlanetModel plane System.err.println(" using query: " + query); } - final FixedBitSet hits = new FixedBitSet(r.maxDoc()); - - s.search( - query, - new SimpleCollector() { - - private int docBase; - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE_NO_SCORES; - } - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - docBase = context.docBase; - } - - @Override - public void collect(int doc) { - hits.set(docBase + doc); - } - }); + final FixedBitSet hits = s.search(query, FixedBitSetCollector.create(r.maxDoc())); if (VERBOSE) { System.err.println(" hitCount: " + hits.cardinality()); diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java b/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java index 743488e4459b..7e2403851ab6 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java @@ -19,6 +19,7 @@ import static org.junit.Assert.*; import java.io.IOException; +import java.util.Collection; import java.util.Locale; import java.util.Random; import java.util.Set; @@ -467,7 +468,20 @@ public ExplanationAssertingSearcher(IndexReader r) { } protected void checkExplanations(Query q) throws IOException { - super.search(q, new ExplanationAsserter(q, null, this)); + super.search( + q, + new CollectorManager() { + @Override + public ExplanationAsserter newCollector() throws IOException { + return new ExplanationAsserter(q, null, ExplanationAssertingSearcher.this); + } + + @Override + public Object reduce(Collection collectors) throws IOException { + // no need to reduce, as the collection is mostly for explanation verification + return null; + } + }); } @Override @@ -489,6 +503,13 @@ public TopDocs search(Query query, int n) throws IOException { checkExplanations(query); return super.search(query, n); } + + @Override + public T search(Query query, CollectorManager collectorManager) + throws IOException { + checkExplanations(query); + return super.search(query, collectorManager); + } } /** From 4af77405fcb914d497d82eb60f0011f93079ec7b Mon Sep 17 00:00:00 2001 From: Zach Chen Date: Wed, 18 Aug 2021 20:08:55 -0700 Subject: [PATCH 04/14] Add flag to collector managers' constructors to allow specifying if thread-safe internal states are needed, and remove static methods in favor of constructors --- .../benchmark/byTask/tasks/ReadTask.java | 10 ++- .../apache/lucene/search/IndexSearcher.java | 12 ++- .../apache/lucene/search/SortRescorer.java | 4 +- .../org/apache/lucene/search/TopDocs.java | 4 +- .../lucene/search/TopFieldCollector.java | 11 +-- .../search/TopFieldCollectorManager.java | 66 ++++++++++----- .../lucene/search/TopScoreDocCollector.java | 10 +-- .../search/TopScoreDocCollectorManager.java | 81 ++++++++++++++----- .../TestLatLonPointDistanceFeatureQuery.java | 10 +-- .../TestLongDistanceFeatureQuery.java | 14 ++-- .../apache/lucene/index/TestIndexSorting.java | 5 +- .../lucene/index/TestIndexWriterMaxDocs.java | 2 +- .../apache/lucene/search/TestBoolean2.java | 18 ++--- .../search/TestBooleanMinShouldMatch.java | 2 +- .../search/TestConstantScoreScorer.java | 4 +- .../lucene/search/TestMatchAllDocsQuery.java | 4 +- .../apache/lucene/search/TestPhraseQuery.java | 14 ++-- .../lucene/search/TestReqOptSumScorer.java | 12 +-- .../lucene/search/TestSortOptimization.java | 24 +++--- .../lucene/search/TestSynonymQuery.java | 12 +-- .../apache/lucene/search/TestTermScorer.java | 8 +- .../lucene/search/TestTopDocsCollector.java | 17 ++-- .../lucene/search/TestTopDocsMerge.java | 3 +- .../lucene/search/TestTopFieldCollector.java | 34 ++++---- .../apache/lucene/facet/DrillSideways.java | 6 +- .../lucene/luke/models/search/SearchImpl.java | 3 +- .../lucene/queries/spans/TestBasics.java | 4 +- .../search/TestCombinedFieldQuery.java | 8 +- .../TestLargeNumHitsTopDocsCollector.java | 10 +-- .../analyzing/AnalyzingInfixSuggester.java | 3 +- .../org/apache/lucene/search/CheckHits.java | 6 +- 31 files changed, 251 insertions(+), 170 deletions(-) diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java index 0a4adeb9614f..2ddd33a66dc9 100644 --- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java +++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java @@ -109,17 +109,19 @@ public int doLogic() throws Exception { // the IndexSearcher search methods that take // Weight public again, we can go back to // pulling the Weight ourselves: + int totalHitsThreshold = withTotalHits() ? Integer.MAX_VALUE : 1; TopFieldCollectorManager collectorManager = - TopFieldCollectorManager.create( - sort, numHits, withTotalHits() ? Integer.MAX_VALUE : 1); + new TopFieldCollectorManager( + sort, numHits, null, totalHitsThreshold, searcher.getExecutor() != null); hits = searcher.search(q, collectorManager); } else { hits = searcher.search(q, numHits); } } else { + int totalHitsThreshold = withTotalHits() ? Integer.MAX_VALUE : 1; TopScoreDocCollectorManager collectorManager = - TopScoreDocCollectorManager.create( - numHits(), withTotalHits() ? Integer.MAX_VALUE : 1); + new TopScoreDocCollectorManager( + numHits(), null, totalHitsThreshold, searcher.getExecutor() != null); searcher.search(q, collectorManager); // hits = collectorManager.topDocs(); } diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java index 268897c665f3..d082d46e1f75 100644 --- a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java +++ b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java @@ -67,8 +67,8 @@ * hits so this trade-off allows to get some minimal information about the hit count without slowing * down search too much. The {@link TopDocs#scoreDocs} array is always accurate however. If this * behavior doesn't suit your needs, you should create collectorManagers manually with either {@link - * TopScoreDocCollectorManager#create} or {@link TopFieldCollectorManager#create} and call {@link - * #search(Query, CollectorManager)}. + * TopScoreDocCollectorManager} or {@link TopFieldCollectorManager} and call {@link #search(Query, + * CollectorManager)}. * *

* @@ -478,8 +478,10 @@ public TopDocs searchAfter(ScoreDoc after, Query query, int numHits) throws IOEx } final int cappedNumHits = Math.min(numHits, limit); + final boolean supportsConcurrency = executor != null && leafSlices.length > 1; CollectorManager manager = - new TopScoreDocCollectorManager(cappedNumHits, after, TOTAL_HITS_THRESHOLD); + new TopScoreDocCollectorManager( + cappedNumHits, after, TOTAL_HITS_THRESHOLD, supportsConcurrency); return search(query, manager); } @@ -590,8 +592,10 @@ private TopFieldDocs searchAfter( final int cappedNumHits = Math.min(numHits, limit); final Sort rewrittenSort = sort.rewrite(this); + final boolean supportsConcurrency = executor != null && leafSlices.length > 1; final CollectorManager manager = - new TopFieldCollectorManager(rewrittenSort, cappedNumHits, after, TOTAL_HITS_THRESHOLD); + new TopFieldCollectorManager( + rewrittenSort, cappedNumHits, after, TOTAL_HITS_THRESHOLD, supportsConcurrency); TopFieldDocs topDocs = search(query, manager); if (doDocScores) { diff --git a/lucene/core/src/java/org/apache/lucene/search/SortRescorer.java b/lucene/core/src/java/org/apache/lucene/search/SortRescorer.java index 095f32695eac..51bf92c4606d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/SortRescorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/SortRescorer.java @@ -46,7 +46,9 @@ public TopDocs rescore(IndexSearcher searcher, TopDocs firstPassTopDocs, int top List leaves = searcher.getIndexReader().leaves(); TopFieldCollector collector = - TopFieldCollectorManager.create(sort, topN, Integer.MAX_VALUE).newCollector(); + new TopFieldCollectorManager( + sort, topN, null, Integer.MAX_VALUE, searcher.getExecutor() != null) + .newCollector(); // Now merge sort docIDs from hits, with reader's leaves: int hitUpto = 0; diff --git a/lucene/core/src/java/org/apache/lucene/search/TopDocs.java b/lucene/core/src/java/org/apache/lucene/search/TopDocs.java index 66c6a266e8f8..dffeedc72d74 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopDocs.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopDocs.java @@ -232,8 +232,8 @@ public static TopDocs merge( /** * Returns a new TopFieldDocs, containing topN results across the provided TopFieldDocs, sorting * by the specified {@link Sort}. Each of the TopDocs must have been sorted by the same Sort, and - * sort field values must have been filled (ie, fillFields=true must be passed to - * {@link TopFieldCollectorManager#create}). + * sort field values must have been filled (ie, fillFields=true must be passed to the + * constructor of {@link TopFieldCollectorManager}). * * @see #merge(Sort, int, int, TopFieldDocs[]) * @lucene.experimental diff --git a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java index 32601d935d1b..57d896a27c97 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java @@ -382,13 +382,13 @@ protected void updateMinCompetitiveScore(Scorable scorer) throws IOException { * count of the result will be accurate. {@link Integer#MAX_VALUE} may be used to make the hit * count accurate, but this will also make query processing slower. * @return a {@link TopFieldCollector} instance which will sort the results by the sort criteria. - * @deprecated This method is being deprecated in favor of {@link - * TopFieldCollectorManager#create(Sort, int, int)} due to its support for concurrency in - * IndexSearcher + * @deprecated This method is being deprecated in favor of using the constructor of {@link + * TopFieldCollectorManager} due to its support for concurrency in IndexSearcher */ @Deprecated public static TopFieldCollector create(Sort sort, int numHits, int totalHitsThreshold) { - return TopFieldCollectorManager.create(sort, numHits, totalHitsThreshold).newCollector(); + return new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold, false) + .newCollector(); } /** @@ -416,7 +416,8 @@ public static TopFieldCollector create(Sort sort, int numHits, int totalHitsThre @Deprecated public static TopFieldCollector create( Sort sort, int numHits, FieldDoc after, int totalHitsThreshold) { - return new TopFieldCollectorManager(sort, numHits, after, totalHitsThreshold).newCollector(); + return new TopFieldCollectorManager(sort, numHits, after, totalHitsThreshold, false) + .newCollector(); } /** diff --git a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java index 2f648a4d6b32..ed97ca775951 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java @@ -36,7 +36,24 @@ public class TopFieldCollectorManager implements CollectorManager collectors; - public TopFieldCollectorManager(Sort sort, int numHits, FieldDoc after, int totalHitsThreshold) { + /** + * Creates a new {@link TopFieldCollectorManager} from the given arguments. + * + *

NOTE: The instances returned by this method pre-allocate a full array of length + * numHits. + * + * @param sort the sort criteria (SortFields). + * @param numHits the number of results to collect. + * @param after the previous doc after which matching docs will be collected. + * @param totalHitsThreshold the number of docs to count accurately. If the query matches more + * than {@code totalHitsThreshold} hits then its hit count will be a lower bound. On the other + * hand if the query matches less than or exactly {@code totalHitsThreshold} hits then the hit + * count of the result will be accurate. {@link Integer#MAX_VALUE} may be used to make the hit + * count accurate, but this will also make query processing slower. + * @param supportsConcurrency to use thread-safe and slower internal states for count tracking. + */ + public TopFieldCollectorManager( + Sort sort, int numHits, FieldDoc after, int totalHitsThreshold, boolean supportsConcurrency) { if (totalHitsThreshold < 0) { throw new IllegalArgumentException( "totalHitsThreshold must be >= 0, got " + totalHitsThreshold); @@ -45,26 +62,37 @@ public TopFieldCollectorManager(Sort sort, int numHits, FieldDoc after, int tota this.sort = sort; this.numHits = numHits; this.after = after; - /* - nocommit - Should the following two be passed in instead? Possible custom initialization based on executor status and slices? - On the other hand, in a single-threaded environment, shared HitsThresholdChecker and MaxScoreAccumulator should be fast without lock contention anyway? - - final HitsThresholdChecker hitsThresholdChecker = - (executor == null || leafSlices.length <= 1) - ? HitsThresholdChecker.create(Math.max(TOTAL_HITS_THRESHOLD, numHits)) - : HitsThresholdChecker.createShared(Math.max(TOTAL_HITS_THRESHOLD, numHits)); - - final MaxScoreAccumulator minScoreAcc = (executor == null || leafSlices.length <= 1) ? null : new MaxScoreAccumulator(); - */ this.hitsThresholdChecker = - HitsThresholdChecker.createShared(Math.max(totalHitsThreshold, numHits)); - this.minScoreAcc = new MaxScoreAccumulator(); + supportsConcurrency + ? HitsThresholdChecker.createShared(Math.max(totalHitsThreshold, numHits)) + : HitsThresholdChecker.create(Math.max(totalHitsThreshold, numHits)); + this.minScoreAcc = supportsConcurrency ? new MaxScoreAccumulator() : null; this.collectors = new ArrayList<>(); } /** - * Creates a new {@link TopFieldCollectorManager} from the given arguments. + * Creates a new {@link TopFieldCollectorManager} from the given arguments, with thread-safe + * internal states. + * + *

NOTE: The instances returned by this method pre-allocate a full array of length + * numHits. + * + * @param sort the sort criteria (SortFields). + * @param numHits the number of results to collect. + * @param after the previous doc after which matching docs will be collected. + * @param totalHitsThreshold the number of docs to count accurately. If the query matches more + * than {@code totalHitsThreshold} hits then its hit count will be a lower bound. On the other + * hand if the query matches less than or exactly {@code totalHitsThreshold} hits then the hit + * count of the result will be accurate. {@link Integer#MAX_VALUE} may be used to make the hit + * count accurate, but this will also make query processing slower. + */ + public TopFieldCollectorManager(Sort sort, int numHits, FieldDoc after, int totalHitsThreshold) { + this(sort, numHits, after, totalHitsThreshold, true); + } + + /** + * Creates a new {@link TopFieldCollectorManager} from the given arguments, with thread-safe + * internal states. * *

NOTE: The instances returned by this method pre-allocate a full array of length * numHits. @@ -76,11 +104,9 @@ public TopFieldCollectorManager(Sort sort, int numHits, FieldDoc after, int tota * hand if the query matches less than or exactly {@code totalHitsThreshold} hits then the hit * count of the result will be accurate. {@link Integer#MAX_VALUE} may be used to make the hit * count accurate, but this will also make query processing slower. - * @return a {@link TopFieldCollectorManager} instance which will be used to create {@link - * TopFieldCollector} to sort the results by the sort criteria. */ - public static TopFieldCollectorManager create(Sort sort, int numHits, int totalHitsThreshold) { - return new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); + public TopFieldCollectorManager(Sort sort, int numHits, int totalHitsThreshold) { + this(sort, numHits, null, totalHitsThreshold, true); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java index f684fab7557e..4aca27e95126 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java @@ -192,13 +192,12 @@ public void collect(int doc) throws IOException { *

NOTE: The instances returned by this method pre-allocate a full array of length * numHits, and fill the array with sentinel objects. * - * @deprecated This method is being deprecated in favor of {@link - * TopScoreDocCollectorManager#create(int, int)} due to its support for concurrency in - * IndexSearcher + * @deprecated This method is being deprecated in favor of using the constructor of {@link + * TopScoreDocCollectorManager} due to its support for concurrency in IndexSearcher */ @Deprecated public static TopScoreDocCollector create(int numHits, int totalHitsThreshold) { - return TopScoreDocCollectorManager.create(numHits, totalHitsThreshold).newCollector(); + return new TopScoreDocCollectorManager(numHits, null, totalHitsThreshold, false).newCollector(); } /** @@ -219,7 +218,8 @@ public static TopScoreDocCollector create(int numHits, int totalHitsThreshold) { */ @Deprecated public static TopScoreDocCollector create(int numHits, ScoreDoc after, int totalHitsThreshold) { - return new TopScoreDocCollectorManager(numHits, after, totalHitsThreshold).newCollector(); + return new TopScoreDocCollectorManager(numHits, after, totalHitsThreshold, false) + .newCollector(); } int docBase; diff --git a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java index f089371ae2c2..37a934e0a94f 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java @@ -32,7 +32,30 @@ public class TopScoreDocCollectorManager private final HitsThresholdChecker hitsThresholdChecker; private final MaxScoreAccumulator minScoreAcc; - public TopScoreDocCollectorManager(int numHits, ScoreDoc after, int totalHitsThreshold) { + /** + * Creates a new {@link TopScoreDocCollectorManager} given the number of hits to collect and the + * number of hits to count accurately. + * + *

NOTE: If the total hit count of the top docs is less than or exactly {@code + * totalHitsThreshold} then this value is accurate. On the other hand, if the {@link + * TopDocs#totalHits} value is greater than {@code totalHitsThreshold} then its value is a lower + * bound of the hit count. A value of {@link Integer#MAX_VALUE} will make the hit count accurate + * but will also likely make query processing slower. + * + *

NOTE: The instances returned by this method pre-allocate a full array of length + * numHits, and fill the array with sentinel objects. + * + * @param numHits the number of results to collect. + * @param after the previous doc after which matching docs will be collected. + * @param totalHitsThreshold the number of docs to count accurately. If the query matches more + * than {@code totalHitsThreshold} hits then its hit count will be a lower bound. On the other + * hand if the query matches less than or exactly {@code totalHitsThreshold} hits then the hit + * count of the result will be accurate. {@link Integer#MAX_VALUE} may be used to make the hit + * count accurate, but this will also make query processing slower. + * @param supportsConcurrency to use thread-safe and slower internal states for count tracking. + */ + public TopScoreDocCollectorManager( + int numHits, ScoreDoc after, int totalHitsThreshold, boolean supportsConcurrency) { if (numHits <= 0) { throw new IllegalArgumentException( "numHits must be > 0; please use TotalHitCountCollectorManager if you just need the total hit count"); @@ -40,26 +63,16 @@ public TopScoreDocCollectorManager(int numHits, ScoreDoc after, int totalHitsThr this.numHits = numHits; this.after = after; - /* - nocommit - Should the following two be passed in instead? Possible custom initialization based on executor status and slices? - On the other hand, in a single-threaded environment, shared HitsThresholdChecker and MaxScoreAccumulator should be fast without lock contention anyway? - - final HitsThresholdChecker hitsThresholdChecker = - (executor == null || leafSlices.length <= 1) - ? HitsThresholdChecker.create(Math.max(TOTAL_HITS_THRESHOLD, numHits)) - : HitsThresholdChecker.createShared(Math.max(TOTAL_HITS_THRESHOLD, numHits)); - - final MaxScoreAccumulator minScoreAcc = (executor == null || leafSlices.length <= 1) ? null : new MaxScoreAccumulator(); - */ this.hitsThresholdChecker = - HitsThresholdChecker.createShared(Math.max(totalHitsThreshold, numHits)); - this.minScoreAcc = new MaxScoreAccumulator(); + supportsConcurrency + ? HitsThresholdChecker.createShared(Math.max(totalHitsThreshold, numHits)) + : HitsThresholdChecker.create(Math.max(totalHitsThreshold, numHits)); + this.minScoreAcc = supportsConcurrency ? new MaxScoreAccumulator() : null; } /** * Creates a new {@link TopScoreDocCollectorManager} given the number of hits to collect and the - * number of hits to count accurately. + * number of hits to count accurately, with thread-safe internal states. * *

NOTE: If the total hit count of the top docs is less than or exactly {@code * totalHitsThreshold} then this value is accurate. On the other hand, if the {@link @@ -69,9 +82,41 @@ public TopScoreDocCollectorManager(int numHits, ScoreDoc after, int totalHitsThr * *

NOTE: The instances returned by this method pre-allocate a full array of length * numHits, and fill the array with sentinel objects. + * + * @param numHits the number of results to collect. + * @param after the previous doc after which matching docs will be collected. + * @param totalHitsThreshold the number of docs to count accurately. If the query matches more + * than {@code totalHitsThreshold} hits then its hit count will be a lower bound. On the other + * hand if the query matches less than or exactly {@code totalHitsThreshold} hits then the hit + * count of the result will be accurate. {@link Integer#MAX_VALUE} may be used to make the hit + * count accurate, but this will also make query processing slower. + */ + public TopScoreDocCollectorManager(int numHits, ScoreDoc after, int totalHitsThreshold) { + this(numHits, after, totalHitsThreshold, true); + } + + /** + * Creates a new {@link TopScoreDocCollectorManager} given the number of hits to collect and the + * number of hits to count accurately, with thread-safe internal states. + * + *

NOTE: If the total hit count of the top docs is less than or exactly {@code + * totalHitsThreshold} then this value is accurate. On the other hand, if the {@link + * TopDocs#totalHits} value is greater than {@code totalHitsThreshold} then its value is a lower + * bound of the hit count. A value of {@link Integer#MAX_VALUE} will make the hit count accurate + * but will also likely make query processing slower. + * + *

NOTE: The instances returned by this method pre-allocate a full array of length + * numHits, and fill the array with sentinel objects. + * + * @param numHits the number of results to collect. + * @param totalHitsThreshold the number of docs to count accurately. If the query matches more + * than {@code totalHitsThreshold} hits then its hit count will be a lower bound. On the other + * hand if the query matches less than or exactly {@code totalHitsThreshold} hits then the hit + * count of the result will be accurate. {@link Integer#MAX_VALUE} may be used to make the hit + * count accurate, but this will also make query processing slower. */ - public static TopScoreDocCollectorManager create(int numHits, int totalHitsThreshold) { - return new TopScoreDocCollectorManager(numHits, null, totalHitsThreshold); + public TopScoreDocCollectorManager(int numHits, int totalHitsThreshold) { + this(numHits, null, totalHitsThreshold, true); } @Override diff --git a/lucene/core/src/test/org/apache/lucene/document/TestLatLonPointDistanceFeatureQuery.java b/lucene/core/src/test/org/apache/lucene/document/TestLatLonPointDistanceFeatureQuery.java index 0a2957bc596c..10698c081f44 100644 --- a/lucene/core/src/test/org/apache/lucene/document/TestLatLonPointDistanceFeatureQuery.java +++ b/lucene/core/src/test/org/apache/lucene/document/TestLatLonPointDistanceFeatureQuery.java @@ -139,7 +139,7 @@ public void testBasics() throws IOException { 9); q = LatLonPoint.newDistanceFeatureQuery("foo", 3, 9, 9, pivotDistance); - collectorManager = new TopScoreDocCollectorManager(2, null, 1); + collectorManager = new TopScoreDocCollectorManager(2, 1); topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); CheckHits.checkExplanations(q, "", searcher); @@ -196,7 +196,7 @@ public void testCrossesDateLine() throws IOException { IndexSearcher searcher = newSearcher(reader); Query q = LatLonPoint.newDistanceFeatureQuery("foo", 3, 0, 179, pivotDistance); - TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, null, 1); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, 1); TopDocs topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); @@ -262,7 +262,7 @@ public void testMissingValue() throws IOException { IndexSearcher searcher = newSearcher(reader); Query q = LatLonPoint.newDistanceFeatureQuery("foo", 3, 10, 10, 5); - TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(3, null, 1); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(3, 1); TopDocs topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); @@ -341,7 +341,7 @@ public void testMultiValued() throws IOException { IndexSearcher searcher = newSearcher(reader); Query q = LatLonPoint.newDistanceFeatureQuery("foo", 3, 0, 0, 200); - TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, null, 1); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, 1); TopDocs topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); @@ -367,7 +367,7 @@ public void testMultiValued() throws IOException { topHits.scoreDocs); q = LatLonPoint.newDistanceFeatureQuery("foo", 3, -90, 0, 10000.); - collectorManager = new TopScoreDocCollectorManager(2, null, 1); + collectorManager = new TopScoreDocCollectorManager(2, 1); topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); CheckHits.checkExplanations(q, "", searcher); diff --git a/lucene/core/src/test/org/apache/lucene/document/TestLongDistanceFeatureQuery.java b/lucene/core/src/test/org/apache/lucene/document/TestLongDistanceFeatureQuery.java index 702422fa2a83..247d86c58a9e 100644 --- a/lucene/core/src/test/org/apache/lucene/document/TestLongDistanceFeatureQuery.java +++ b/lucene/core/src/test/org/apache/lucene/document/TestLongDistanceFeatureQuery.java @@ -89,7 +89,7 @@ public void testBasics() throws IOException { IndexSearcher searcher = newSearcher(reader); Query q = LongPoint.newDistanceFeatureQuery("foo", 3, 10, 5); - TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, null, 1); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, 1); TopDocs topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); @@ -102,7 +102,7 @@ public void testBasics() throws IOException { topHits.scoreDocs); q = LongPoint.newDistanceFeatureQuery("foo", 3, 7, 5); - collectorManager = new TopScoreDocCollectorManager(2, null, 1); + collectorManager = new TopScoreDocCollectorManager(2, 1); topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); CheckHits.checkExplanations(q, "", searcher); @@ -157,7 +157,7 @@ public void testOverUnderFlow() throws IOException { IndexSearcher searcher = newSearcher(reader); Query q = LongPoint.newDistanceFeatureQuery("foo", 3, Long.MAX_VALUE - 1, 100); - TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, null, 1); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, 1); TopDocs topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); @@ -178,7 +178,7 @@ public void testOverUnderFlow() throws IOException { topHits.scoreDocs); q = LongPoint.newDistanceFeatureQuery("foo", 3, Long.MIN_VALUE + 1, 100); - collectorManager = new TopScoreDocCollectorManager(2, null, 1); + collectorManager = new TopScoreDocCollectorManager(2, 1); topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); CheckHits.checkExplanations(q, "", searcher); @@ -240,7 +240,7 @@ public void testMissingValue() throws IOException { IndexSearcher searcher = newSearcher(reader); Query q = LongPoint.newDistanceFeatureQuery("foo", 3, 10, 5); - TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(3, null, 1); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(3, 1); TopDocs topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); @@ -306,7 +306,7 @@ public void testMultiValued() throws IOException { IndexSearcher searcher = newSearcher(reader); Query q = LongPoint.newDistanceFeatureQuery("foo", 3, 10, 5); - TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, null, 1); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, 1); TopDocs topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); @@ -319,7 +319,7 @@ public void testMultiValued() throws IOException { topHits.scoreDocs); q = LongPoint.newDistanceFeatureQuery("foo", 3, 7, 5); - collectorManager = new TopScoreDocCollectorManager(2, null, 1); + collectorManager = new TopScoreDocCollectorManager(2, 1); topHits = searcher.search(q, collectorManager); assertEquals(2, topHits.scoreDocs.length); CheckHits.checkExplanations(q, "", searcher); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java index 827e36474d90..c3344694b4f7 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java @@ -2643,11 +2643,10 @@ public void testRandom3() throws Exception { System.out.println("TEST: iter=" + iter + " numHits=" + numHits); } - TopFieldCollectorManager c1 = - TopFieldCollectorManager.create(sort, numHits, Integer.MAX_VALUE); + TopFieldCollectorManager c1 = new TopFieldCollectorManager(sort, numHits, Integer.MAX_VALUE); TopDocs hits1 = s1.search(new MatchAllDocsQuery(), c1); - TopFieldCollectorManager c2 = TopFieldCollectorManager.create(sort, numHits, 1); + TopFieldCollectorManager c2 = new TopFieldCollectorManager(sort, numHits, 1); TopDocs hits2 = s2.search(new MatchAllDocsQuery(), c2); if (VERBOSE) { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java index 35dde49b6eb3..18c6a1abd3da 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java @@ -67,7 +67,7 @@ public void testExactlyAtTrueLimit() throws Exception { assertEquals(IndexWriter.MAX_DOCS, ir.numDocs()); IndexSearcher searcher = new IndexSearcher(ir); TopScoreDocCollectorManager collectorManager = - TopScoreDocCollectorManager.create(10, Integer.MAX_VALUE); + new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE, false); TopDocs hits = searcher.search(new TermQuery(new Term("field", "text")), collectorManager); assertEquals(IndexWriter.MAX_DOCS, hits.totalHits.value); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java b/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java index 48fc95a067e8..c67429f086ee 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java @@ -235,16 +235,16 @@ public void queriesTest(Query query, int[] expDocNrs) throws Exception { // sometimes return a default impl around the scorer so that we can // compare BS1 and BS2 TopScoreDocCollectorManager collectorManager = - TopScoreDocCollectorManager.create(topDocsToCheck, Integer.MAX_VALUE); + new TopScoreDocCollectorManager(topDocsToCheck, Integer.MAX_VALUE); ScoreDoc[] hits1 = searcher.search(query, collectorManager).scoreDocs; - collectorManager = TopScoreDocCollectorManager.create(topDocsToCheck, Integer.MAX_VALUE); + collectorManager = new TopScoreDocCollectorManager(topDocsToCheck, Integer.MAX_VALUE); ScoreDoc[] hits2 = searcher.search(query, collectorManager).scoreDocs; CheckHits.checkHitsQuery(query, hits1, hits2, expDocNrs); // Since we have no deleted docs, we should also be able to verify identical matches & // scores against an single segment copy of our index - collectorManager = TopScoreDocCollectorManager.create(topDocsToCheck, Integer.MAX_VALUE); + collectorManager = new TopScoreDocCollectorManager(topDocsToCheck, Integer.MAX_VALUE); TopDocs topDocs2 = singleSegmentSearcher.search(query, collectorManager); hits2 = topDocs2.scoreDocs; CheckHits.checkHitsQuery(query, hits1, hits2, expDocNrs); @@ -253,9 +253,9 @@ public void queriesTest(Query query, int[] expDocNrs) throws Exception { assertEquals(mulFactor * topDocs2.totalHits.value, bigSearcher.count(query)); // now check 2 diff scorers from the bigSearcher as well - collectorManager = TopScoreDocCollectorManager.create(topDocsToCheck, Integer.MAX_VALUE); + collectorManager = new TopScoreDocCollectorManager(topDocsToCheck, Integer.MAX_VALUE); hits1 = bigSearcher.search(query, collectorManager).scoreDocs; - collectorManager = TopScoreDocCollectorManager.create(topDocsToCheck, Integer.MAX_VALUE); + collectorManager = new TopScoreDocCollectorManager(topDocsToCheck, Integer.MAX_VALUE); hits2 = bigSearcher.search(query, collectorManager).scoreDocs; // NOTE: just comparing results, not vetting against expDocNrs @@ -384,9 +384,9 @@ public void testRandomQueries() throws Exception { } // check diff (randomized) scorers (from AssertingSearcher) produce the same results - TopFieldCollectorManager collectorManager = TopFieldCollectorManager.create(sort, 1000, 1); + TopFieldCollectorManager collectorManager = new TopFieldCollectorManager(sort, 1000, 1); ScoreDoc[] hits1 = searcher.search(q1, collectorManager).scoreDocs; - collectorManager = TopFieldCollectorManager.create(sort, 1000, 1); + collectorManager = new TopFieldCollectorManager(sort, 1000, 1); TopDocs topDocs = searcher.search(q1, collectorManager); ScoreDoc[] hits2 = topDocs.scoreDocs; CheckHits.checkEqual(q1, hits1, hits2); @@ -399,9 +399,9 @@ public void testRandomQueries() throws Exception { bigSearcher.count(q3.build())); // test diff (randomized) scorers produce the same results on bigSearcher as well - collectorManager = TopFieldCollectorManager.create(sort, 1000 * mulFactor, 1); + collectorManager = new TopFieldCollectorManager(sort, 1000 * mulFactor, 1); hits1 = bigSearcher.search(q1, collectorManager).scoreDocs; - collectorManager = TopFieldCollectorManager.create(sort, 1000 * mulFactor, 1); + collectorManager = new TopFieldCollectorManager(sort, 1000 * mulFactor, 1); hits2 = bigSearcher.search(q1, collectorManager).scoreDocs; CheckHits.checkEqual(q1, hits1, hits2); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java index 131b10672b05..1a03e16dba7f 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java @@ -91,7 +91,7 @@ public void verifyNrHits(Query q, int expected) throws Exception { // System.out.println("TEST: now check"); // bs2 TopScoreDocCollectorManager collectorManager = - TopScoreDocCollectorManager.create(1000, Integer.MAX_VALUE); + new TopScoreDocCollectorManager(1000, Integer.MAX_VALUE); TopDocs topDocs = s.search(q, collectorManager); ScoreDoc[] h2 = topDocs.scoreDocs; if (expected != h2.length) { diff --git a/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreScorer.java index 952c4d6a17a8..2136b3aff87b 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreScorer.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreScorer.java @@ -240,12 +240,12 @@ public void testEarlyTermination() throws IOException { IndexSearcher is = newSearcher(ir); - TopScoreDocCollectorManager c = new TopScoreDocCollectorManager(10, null, 10); + TopScoreDocCollectorManager c = new TopScoreDocCollectorManager(10, 10); TopDocs topDocs = is.search(new ConstantScoreQuery(new TermQuery(new Term("key", "foo"))), c); assertEquals(11, topDocs.totalHits.value); assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, topDocs.totalHits.relation); - c = new TopScoreDocCollectorManager(10, null, 10); + c = new TopScoreDocCollectorManager(10, 10); Query query = new BooleanQuery.Builder() .add(new ConstantScoreQuery(new TermQuery(new Term("key", "foo"))), Occur.SHOULD) diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java index 95b08b09788b..071e5bbd9b22 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java @@ -119,13 +119,13 @@ public void testEarlyTermination() throws IOException { final int totalHitsThreshold = 200; TopScoreDocCollectorManager collectorManager = - new TopScoreDocCollectorManager(10, null, totalHitsThreshold); + new TopScoreDocCollectorManager(10, totalHitsThreshold); TopDocs topDocs = is.search(new MatchAllDocsQuery(), collectorManager); assertEquals(totalHitsThreshold + 1, topDocs.totalHits.value); assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, topDocs.totalHits.relation); - collectorManager = new TopScoreDocCollectorManager(10, null, numDocs); + collectorManager = new TopScoreDocCollectorManager(10, numDocs); topDocs = is.search(new MatchAllDocsQuery(), collectorManager); assertEquals(numDocs, topDocs.totalHits.value); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java index 66e4b25cc020..4cbceecc7f2d 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java @@ -771,12 +771,11 @@ public void testTopPhrases() throws IOException { )) { for (int topN = 1; topN <= 2; ++topN) { TopScoreDocCollectorManager collectorManager1 = - new TopScoreDocCollectorManager(topN, null, Integer.MAX_VALUE); + new TopScoreDocCollectorManager(topN, Integer.MAX_VALUE); TopDocs topDocs1 = searcher.search(query, collectorManager1); ScoreDoc[] hits1 = topDocs1.scoreDocs; - TopScoreDocCollectorManager collectorManager2 = - new TopScoreDocCollectorManager(topN, null, 1); + TopScoreDocCollectorManager collectorManager2 = new TopScoreDocCollectorManager(topN, 1); TopDocs topDocs2 = searcher.search(query, collectorManager2); ScoreDoc[] hits2 = topDocs2.scoreDocs; @@ -1031,11 +1030,11 @@ public void testRandomTopDocs() throws IOException { Query query = new PhraseQuery("foo", new BytesRef(firstTerm), new BytesRef(secondTerm)); TopScoreDocCollectorManager collectorManager1 = - new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE); // COMPLETE + new TopScoreDocCollectorManager(10, Integer.MAX_VALUE); // COMPLETE TopDocs topDocs1 = searcher.search(query, collectorManager1); TopScoreDocCollectorManager collectorManager2 = - new TopScoreDocCollectorManager(10, null, 10); // TOP_SCORES + new TopScoreDocCollectorManager(10, 10); // TOP_SCORES TopDocs topDocs2 = searcher.search(query, collectorManager2); CheckHits.checkEqual(query, topDocs1.scoreDocs, topDocs2.scoreDocs); @@ -1046,11 +1045,10 @@ public void testRandomTopDocs() throws IOException { .add(new TermQuery(new Term("foo", "b")), Occur.FILTER) .build(); - collectorManager1 = - new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE); // COMPLETE + collectorManager1 = new TopScoreDocCollectorManager(10, Integer.MAX_VALUE); // COMPLETE topDocs1 = searcher.search(filteredQuery, collectorManager1); - collectorManager2 = new TopScoreDocCollectorManager(10, null, 10); // TOP_SCORES + collectorManager2 = new TopScoreDocCollectorManager(10, 10); // TOP_SCORES topDocs2 = searcher.search(filteredQuery, collectorManager2); CheckHits.checkEqual(query, topDocs1.scoreDocs, topDocs2.scoreDocs); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestReqOptSumScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestReqOptSumScorer.java index 2e9ed9929b47..54ccb6f5bf86 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestReqOptSumScorer.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestReqOptSumScorer.java @@ -264,7 +264,7 @@ private void doTestRandom(double optFreq) throws IOException { new BooleanQuery.Builder().add(mustTerm, Occur.MUST).add(shouldTerm, Occur.SHOULD).build(); TopScoreDocCollectorManager collectorManager = - new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE); + new TopScoreDocCollectorManager(10, Integer.MAX_VALUE); TopDocs topDocs = searcher.search(query, collectorManager); ScoreDoc[] expected = topDocs.scoreDocs; @@ -276,7 +276,7 @@ private void doTestRandom(double optFreq) throws IOException { .add(new TermQuery(new Term("f", "C")), Occur.FILTER) .build(); - collectorManager = new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE); + collectorManager = new TopScoreDocCollectorManager(10, Integer.MAX_VALUE); topDocs = searcher.search(query, collectorManager); ScoreDoc[] expectedFiltered = topDocs.scoreDocs; @@ -289,7 +289,7 @@ private void doTestRandom(double optFreq) throws IOException { .add(shouldTerm, Occur.SHOULD) .build(); - collectorManager = new TopScoreDocCollectorManager(10, null, 1); + collectorManager = new TopScoreDocCollectorManager(10, 1); topDocs = searcher.search(q, collectorManager); ScoreDoc[] actual = topDocs.scoreDocs; CheckHits.checkEqual(query, expected, actual); @@ -299,7 +299,7 @@ private void doTestRandom(double optFreq) throws IOException { .add(mustTerm, Occur.MUST) .add(new RandomApproximationQuery(shouldTerm, random()), Occur.SHOULD) .build(); - collectorManager = new TopScoreDocCollectorManager(10, null, 1); + collectorManager = new TopScoreDocCollectorManager(10, 1); topDocs = searcher.search(q, collectorManager); actual = topDocs.scoreDocs; CheckHits.checkEqual(q, expected, actual); @@ -309,7 +309,7 @@ private void doTestRandom(double optFreq) throws IOException { .add(new RandomApproximationQuery(mustTerm, random()), Occur.MUST) .add(new RandomApproximationQuery(shouldTerm, random()), Occur.SHOULD) .build(); - collectorManager = new TopScoreDocCollectorManager(10, null, 1); + collectorManager = new TopScoreDocCollectorManager(10, 1); topDocs = searcher.search(q, collectorManager); actual = topDocs.scoreDocs; CheckHits.checkEqual(q, expected, actual); @@ -331,7 +331,7 @@ private void doTestRandom(double optFreq) throws IOException { Occur.FILTER) .build(); - collectorManager = new TopScoreDocCollectorManager(10, null, 1); + collectorManager = new TopScoreDocCollectorManager(10, 1); topDocs = searcher.search(nestedQ, collectorManager); ScoreDoc[] actualFiltered = topDocs.scoreDocs; CheckHits.checkEqual(nestedQ, expectedFiltered, actualFiltered); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java b/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java index 31d86865abc5..dd9a8cb49ad1 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java @@ -61,7 +61,7 @@ public void testLongSortOptimization() throws IOException { { // simple sort final TopFieldCollectorManager collectorManager = - new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); + new TopFieldCollectorManager(sort, numHits, totalHitsThreshold); TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); @@ -94,7 +94,7 @@ public void testLongSortOptimization() throws IOException { { // test that if there is the secondary sort on _score, scores are filled correctly final TopFieldCollectorManager collectorManager = new TopFieldCollectorManager( - new Sort(sortField, FIELD_SCORE), numHits, null, totalHitsThreshold); + new Sort(sortField, FIELD_SCORE), numHits, totalHitsThreshold); TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); assertEquals(topDocs.scoreDocs.length, numHits); for (int i = 0; i < numHits; i++) { @@ -111,7 +111,7 @@ public void testLongSortOptimization() throws IOException { { // test that if numeric field is a secondary sort, no optimization is run final TopFieldCollectorManager collectorManager = new TopFieldCollectorManager( - new Sort(FIELD_SCORE, sortField), numHits, null, totalHitsThreshold); + new Sort(FIELD_SCORE, sortField), numHits, totalHitsThreshold); TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); assertEquals(topDocs.scoreDocs.length, numHits); assertEquals( @@ -146,7 +146,7 @@ public void testLongSortOptimizationOnFieldNotIndexedWithPoints() throws IOExcep final int totalHitsThreshold = 3; final TopFieldCollectorManager collectorManager = - new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); + new TopFieldCollectorManager(sort, numHits, totalHitsThreshold); TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); assertEquals( topDocs.scoreDocs.length, numHits); // sort still works and returns expected number of docs @@ -186,7 +186,7 @@ public void testSortOptimizationWithMissingValues() throws IOException { sortField.setMissingValue(0L); // set a competitive missing value final Sort sort = new Sort(sortField); final TopFieldCollectorManager collectorManager = - new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); + new TopFieldCollectorManager(sort, numHits, totalHitsThreshold); TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); assertEquals(topDocs.scoreDocs.length, numHits); assertEquals( @@ -198,7 +198,7 @@ public void testSortOptimizationWithMissingValues() throws IOException { sortField.setMissingValue(100L); // set a NON competitive missing value final Sort sort = new Sort(sortField); final TopFieldCollectorManager collectorManager = - new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); + new TopFieldCollectorManager(sort, numHits, totalHitsThreshold); TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); assertEquals(topDocs.scoreDocs.length, numHits); assertTrue( @@ -235,7 +235,7 @@ public void testSortOptimizationEqualValues() throws IOException { final SortField sortField = new SortField("my_field1", SortField.Type.INT); final Sort sort = new Sort(sortField); final TopFieldCollectorManager collectorManager = - new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); + new TopFieldCollectorManager(sort, numHits, totalHitsThreshold); TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); assertEquals(topDocs.scoreDocs.length, numHits); for (int i = 0; i < numHits; i++) { @@ -272,7 +272,7 @@ public void testSortOptimizationEqualValues() throws IOException { final SortField sortField2 = new SortField("my_field2", SortField.Type.INT); final Sort sort = new Sort(sortField1, sortField2); final TopFieldCollectorManager collectorManager = - new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); + new TopFieldCollectorManager(sort, numHits, totalHitsThreshold); TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); assertEquals(topDocs.scoreDocs.length, numHits); for (int i = 0; i < numHits; i++) { @@ -311,7 +311,7 @@ public void testFloatSortOptimization() throws IOException { { // simple sort final TopFieldCollectorManager collectorManager = - new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); + new TopFieldCollectorManager(sort, numHits, totalHitsThreshold); TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); assertEquals(topDocs.scoreDocs.length, numHits); @@ -499,7 +499,7 @@ public void testDocSortOptimization() throws IOException { // sort by _doc should skip all non-competitive documents { final TopFieldCollectorManager collectorManager = - new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); + new TopFieldCollectorManager(sort, numHits, totalHitsThreshold); IndexSearcher searcher = newSearcher(reader); TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), collectorManager); assertEquals(numHits, topDocs.scoreDocs.length); @@ -514,7 +514,7 @@ public void testDocSortOptimization() throws IOException { // sort by _doc with a bool query should skip all non-competitive documents { final TopFieldCollectorManager collectorManager = - new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); + new TopFieldCollectorManager(sort, numHits, totalHitsThreshold); int lowerRange = 40; BooleanQuery.Builder bq = new BooleanQuery.Builder(); bq.add(LongPoint.newRangeQuery("lf", lowerRange, Long.MAX_VALUE), BooleanClause.Occur.MUST); @@ -567,7 +567,7 @@ public void testDocSort() throws IOException { { final TopFieldCollectorManager collectorManager = - new TopFieldCollectorManager(sort, numHits, null, totalHitsThreshold); + new TopFieldCollectorManager(sort, numHits, totalHitsThreshold); BooleanQuery.Builder bq = new BooleanQuery.Builder(); bq.add(LongPoint.newExactQuery("lf", 1), BooleanClause.Occur.MUST); bq.add(new TermQuery(new Term("id", "id3")), BooleanClause.Occur.MUST_NOT); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSynonymQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestSynonymQuery.java index f3f6471ecf0d..8783008ecbc6 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSynonymQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSynonymQuery.java @@ -167,7 +167,7 @@ private void doTestScores(int totalHitsThreshold) throws IOException { TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager( - Math.min(reader.numDocs(), totalHitsThreshold), null, totalHitsThreshold); + Math.min(reader.numDocs(), totalHitsThreshold), totalHitsThreshold); TopDocs topDocs = searcher.search(query, collectorManager); if (topDocs.totalHits.value < totalHitsThreshold) { assertEquals(new TotalHits(11, TotalHits.Relation.EQUAL_TO), topDocs.totalHits); @@ -226,7 +226,7 @@ public void doTestBoosts(int totalHitsThreshold) throws IOException { TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager( - Math.min(reader.numDocs(), totalHitsThreshold), null, totalHitsThreshold); + Math.min(reader.numDocs(), totalHitsThreshold), totalHitsThreshold); TopDocs topDocs = searcher.search(query, collectorManager); if (topDocs.totalHits.value < totalHitsThreshold) { assertEquals(TotalHits.Relation.EQUAL_TO, topDocs.totalHits.relation); @@ -442,9 +442,9 @@ public void testRandomTopDocs() throws IOException { .build(); TopScoreDocCollectorManager collectorManager1 = - new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE); // COMPLETE + new TopScoreDocCollectorManager(10, Integer.MAX_VALUE); // COMPLETE TopScoreDocCollectorManager collectorManager2 = - new TopScoreDocCollectorManager(10, null, 1); // TOP_SCORES + new TopScoreDocCollectorManager(10, 1); // TOP_SCORES TopDocs topDocs1 = searcher.search(query, collectorManager1); TopDocs topDocs2 = searcher.search(query, collectorManager2); @@ -457,8 +457,8 @@ public void testRandomTopDocs() throws IOException { .add(new TermQuery(new Term("foo", Integer.toString(filterTerm))), Occur.FILTER) .build(); - collectorManager1 = new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE); // COMPLETE - collectorManager2 = new TopScoreDocCollectorManager(10, null, 1); // TOP_SCORES + collectorManager1 = new TopScoreDocCollectorManager(10, Integer.MAX_VALUE); // COMPLETE + collectorManager2 = new TopScoreDocCollectorManager(10, 1); // TOP_SCORES topDocs1 = searcher.search(filteredQuery, collectorManager1); topDocs2 = searcher.search(filteredQuery, collectorManager2); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java index 11864ae4403c..0bed70ae4cc1 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java @@ -242,9 +242,9 @@ public void testRandomTopDocs() throws IOException { Query query = new TermQuery(new Term("foo", Integer.toString(iter))); TopScoreDocCollectorManager collectorManager1 = - new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE); // COMPLETE + new TopScoreDocCollectorManager(10, Integer.MAX_VALUE); // COMPLETE TopScoreDocCollectorManager collectorManager2 = - new TopScoreDocCollectorManager(10, null, 1); // TOP_SCORES + new TopScoreDocCollectorManager(10, 1); // TOP_SCORES TopDocs topDocs1 = searcher.search(query, collectorManager1); TopDocs topDocs2 = searcher.search(query, collectorManager2); @@ -257,8 +257,8 @@ public void testRandomTopDocs() throws IOException { .add(new TermQuery(new Term("foo", Integer.toString(filterTerm))), Occur.FILTER) .build(); - collectorManager1 = new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE); // COMPLETE - collectorManager2 = new TopScoreDocCollectorManager(10, null, 1); // TOP_SCORES + collectorManager1 = new TopScoreDocCollectorManager(10, Integer.MAX_VALUE); // COMPLETE + collectorManager2 = new TopScoreDocCollectorManager(10, 1); // TOP_SCORES topDocs1 = searcher.search(filteredQuery, collectorManager1); topDocs2 = searcher.search(filteredQuery, collectorManager2); CheckHits.checkEqual(query, topDocs1.scoreDocs, topDocs2.scoreDocs); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java index 8ad9d702d625..f4f6f41d71b8 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java @@ -159,7 +159,7 @@ private TopDocs doSearchWithThreshold( int numResults, int thresHold, Query q, IndexReader indexReader) throws IOException { IndexSearcher searcher = new IndexSearcher(indexReader); TopScoreDocCollectorManager collectorManager = - TopScoreDocCollectorManager.create(numResults, thresHold); + new TopScoreDocCollectorManager(numResults, null, thresHold, false); return searcher.search(q, collectorManager); } @@ -177,7 +177,7 @@ private TopDocs doConcurrentSearchWithThreshold( IndexSearcher searcher = new IndexSearcher(indexReader, service); TopScoreDocCollectorManager collectorManager = - new TopScoreDocCollectorManager(numResults, null, threshold); + new TopScoreDocCollectorManager(numResults, threshold); return searcher.search(q, collectorManager); } finally { @@ -347,7 +347,7 @@ public void testSetMinCompetitiveScore() throws Exception { assertEquals(2, reader.leaves().size()); w.close(); - TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, null, 2); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, 2); ScoreAndDoc scorer = new ScoreAndDoc(); TopScoreDocCollector collector = collectorManager.newCollector(); @@ -440,7 +440,7 @@ public void testTotalHits() throws Exception { for (int totalHitsThreshold = 0; totalHitsThreshold < 20; ++totalHitsThreshold) { TopScoreDocCollectorManager collectorManager = - new TopScoreDocCollectorManager(2, null, totalHitsThreshold); + new TopScoreDocCollectorManager(2, totalHitsThreshold); TopScoreDocCollector collector = collectorManager.newCollector(); ScoreAndDoc scorer = new ScoreAndDoc(); @@ -495,17 +495,17 @@ public void testRelationVsTopDocsCount() throws Exception { try (IndexReader reader = DirectoryReader.open(w)) { IndexSearcher searcher = new IndexSearcher(reader); - TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, null, 10); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(2, 10); TopDocs topDocs = searcher.search(new TermQuery(new Term("f", "foo")), collectorManager); assertEquals(10, topDocs.totalHits.value); assertEquals(TotalHits.Relation.EQUAL_TO, topDocs.totalHits.relation); - collectorManager = new TopScoreDocCollectorManager(2, null, 2); + collectorManager = new TopScoreDocCollectorManager(2, 2); topDocs = searcher.search(new TermQuery(new Term("f", "foo")), collectorManager); assertTrue(10 >= topDocs.totalHits.value); assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, topDocs.totalHits.relation); - collectorManager = new TopScoreDocCollectorManager(10, null, 2); + collectorManager = new TopScoreDocCollectorManager(10, 2); topDocs = searcher.search(new TermQuery(new Term("f", "foo")), collectorManager); assertEquals(10, topDocs.totalHits.value); assertEquals(TotalHits.Relation.EQUAL_TO, topDocs.totalHits.relation); @@ -528,8 +528,7 @@ public void testConcurrentMinScore() throws Exception { assertEquals(3, reader.leaves().size()); w.close(); - CollectorManager manager = - new TopScoreDocCollectorManager(2, null, 0); + CollectorManager manager = new TopScoreDocCollectorManager(2, 0); TopScoreDocCollector collector = manager.newCollector(); TopScoreDocCollector collector2 = manager.newCollector(); assertTrue(collector.minScoreAcc == collector2.minScoreAcc); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java index f0bdce28f324..daa0e5b8748e 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsMerge.java @@ -275,8 +275,7 @@ void testSort(boolean useFrom) throws Exception { from = TestUtil.nextInt(random(), 0, numHits - 1); size = numHits - from; TopDocs tempTopHits = - searcher.search( - query, TopScoreDocCollectorManager.create(numHits, Integer.MAX_VALUE)); + searcher.search(query, new TopScoreDocCollectorManager(numHits, Integer.MAX_VALUE)); if (from < tempTopHits.scoreDocs.length) { // Can't use TopDocs#topDocs(start, howMany), since it has different behaviour when // start >= hitCount diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java index 524ef989ed43..e1acb0417367 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java @@ -77,7 +77,8 @@ private TopFieldDocs doSearchWithThreshold( int numResults, int thresHold, Query q, Sort sort, IndexReader indexReader) throws IOException { IndexSearcher searcher = new IndexSearcher(indexReader); - TopFieldCollectorManager tdc = TopFieldCollectorManager.create(sort, numResults, thresHold); + TopFieldCollectorManager tdc = + new TopFieldCollectorManager(sort, numResults, null, thresHold, false); return searcher.search(q, tdc); } @@ -96,7 +97,7 @@ private TopDocs doConcurrentSearchWithThreshold( IndexSearcher searcher = new IndexSearcher(indexReader, service); TopFieldCollectorManager collectorManager = - new TopFieldCollectorManager(sort, numResults, null, threshold); + new TopFieldCollectorManager(sort, numResults, threshold); TopDocs topDoc = searcher.search(q, collectorManager); @@ -117,7 +118,7 @@ public void testSortWithoutFillFields() throws Exception { for (int i = 0; i < sort.length; i++) { Query q = new MatchAllDocsQuery(); TopFieldCollectorManager collectorManager = - TopFieldCollectorManager.create(sort[i], 10, Integer.MAX_VALUE); + new TopFieldCollectorManager(sort[i], 10, Integer.MAX_VALUE); ScoreDoc[] sd = is.search(q, collectorManager).scoreDocs; for (int j = 1; j < sd.length; j++) { @@ -133,7 +134,7 @@ public void testSort() throws Exception { for (int i = 0; i < sort.length; i++) { Query q = new MatchAllDocsQuery(); TopFieldCollectorManager tdc = - TopFieldCollectorManager.create(sort[i], 10, Integer.MAX_VALUE); + new TopFieldCollectorManager(sort[i], 10, null, Integer.MAX_VALUE, false); TopDocs td = is.search(q, tdc); ScoreDoc[] sd = td.scoreDocs; for (int j = 0; j < sd.length; j++) { @@ -159,12 +160,10 @@ public void testSharedHitcountCollector() throws Exception { Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort()}; for (int i = 0; i < sort.length; i++) { Query q = new MatchAllDocsQuery(); - TopFieldCollectorManager tdc = - TopFieldCollectorManager.create(sort[i], 10, Integer.MAX_VALUE); + TopFieldCollectorManager tdc = new TopFieldCollectorManager(sort[i], 10, Integer.MAX_VALUE); TopDocs td = is.search(q, tdc); - TopFieldCollectorManager tsdc = - new TopFieldCollectorManager(sort[i], 10, null, Integer.MAX_VALUE); + TopFieldCollectorManager tsdc = new TopFieldCollectorManager(sort[i], 10, Integer.MAX_VALUE); TopDocs td2 = concurrentSearcher.search(q, tsdc); ScoreDoc[] sd = td.scoreDocs; @@ -186,7 +185,7 @@ public void testSortWithoutTotalHitTracking() throws Exception { // the index is not sorted TopFieldCollectorManager tdc; if (i % 2 == 0) { - tdc = TopFieldCollectorManager.create(sort, 10, 1); + tdc = new TopFieldCollectorManager(sort, 10, 1); } else { FieldDoc fieldDoc = new FieldDoc(1, Float.NaN, new Object[] {1}); tdc = new TopFieldCollectorManager(sort, 10, fieldDoc, 1); @@ -305,7 +304,7 @@ public void testSetMinCompetitiveScore() throws Exception { w.close(); Sort sort = new Sort(FIELD_SCORE, new SortField("foo", SortField.Type.LONG)); - TopFieldCollector collector = new TopFieldCollectorManager(sort, 2, null, 2).newCollector(); + TopFieldCollector collector = new TopFieldCollectorManager(sort, 2, 2).newCollector(); ScoreAndDoc scorer = new ScoreAndDoc(); LeafCollector leafCollector = collector.getLeafCollector(reader.leaves().get(0)); @@ -375,7 +374,7 @@ public void testTotalHitsWithScore() throws Exception { for (int totalHitsThreshold = 0; totalHitsThreshold < 20; ++totalHitsThreshold) { Sort sort = new Sort(FIELD_SCORE, new SortField("foo", SortField.Type.LONG)); TopFieldCollector collector = - new TopFieldCollectorManager(sort, 2, null, totalHitsThreshold).newCollector(); + new TopFieldCollectorManager(sort, 2, totalHitsThreshold).newCollector(); ScoreAndDoc scorer = new ScoreAndDoc(); LeafCollector leafCollector = collector.getLeafCollector(reader.leaves().get(0)); @@ -421,7 +420,7 @@ public void testSortNoResults() throws Exception { Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort()}; for (int i = 0; i < sort.length; i++) { TopDocsCollector tdc = - TopFieldCollectorManager.create(sort[i], 10, Integer.MAX_VALUE).newCollector(); + new TopFieldCollectorManager(sort[i], 10, null, Integer.MAX_VALUE, false).newCollector(); TopDocs td = tdc.topDocs(); assertEquals(0, td.totalHits.value); } @@ -459,7 +458,7 @@ public void testComputeScoresOnlyOnce() throws Exception { query, new CollectorManager<>() { TopFieldCollectorManager topFieldCollectorManager = - TopFieldCollectorManager.create( + new TopFieldCollectorManager( sort, TestUtil.nextInt(random(), 1, 2), Integer.MAX_VALUE); @Override @@ -588,7 +587,7 @@ public void testConcurrentMinScore() throws Exception { Sort sort = new Sort(SortField.FIELD_SCORE, SortField.FIELD_DOC); CollectorManager manager = - new TopFieldCollectorManager(sort, 2, null, 0); + new TopFieldCollectorManager(sort, 2, 0); TopFieldCollector collector = manager.newCollector(); TopFieldCollector collector2 = manager.newCollector(); assertTrue(collector.minScoreAcc == collector2.minScoreAcc); @@ -754,17 +753,18 @@ public void testRelationVsTopDocsCount() throws Exception { try (IndexReader reader = DirectoryReader.open(w)) { IndexSearcher searcher = new IndexSearcher(reader); - TopFieldCollectorManager collectorManager = TopFieldCollectorManager.create(sort, 2, 10); + TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, 2, null, 10, false); TopDocs topDocs = searcher.search(new TermQuery(new Term("f", "foo")), collectorManager); assertEquals(10, topDocs.totalHits.value); assertEquals(TotalHits.Relation.EQUAL_TO, topDocs.totalHits.relation); - collectorManager = TopFieldCollectorManager.create(sort, 2, 2); + collectorManager = new TopFieldCollectorManager(sort, 2, null, 2, false); topDocs = searcher.search(new TermQuery(new Term("f", "foo")), collectorManager); assertTrue(10 >= topDocs.totalHits.value); assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, topDocs.totalHits.relation); - collectorManager = TopFieldCollectorManager.create(sort, 10, 2); + collectorManager = new TopFieldCollectorManager(sort, 10, null, 2, false); topDocs = searcher.search(new TermQuery(new Term("f", "foo")), collectorManager); assertEquals(10, topDocs.totalHits.value); assertEquals(TotalHits.Relation.EQUAL_TO, topDocs.totalHits.relation); diff --git a/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java b/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java index 89448bb096a8..e50367c45963 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java @@ -296,8 +296,9 @@ public DrillSidewaysResult search( limit = 1; // the collector does not alow numHits = 0 } final int fTopN = Math.min(topN, limit); + final boolean supportsConcurrency = searcher.getExecutor() != null; final TopFieldCollectorManager collectorManager = - new TopFieldCollectorManager(sort, fTopN, after, Integer.MAX_VALUE); + new TopFieldCollectorManager(sort, fTopN, after, Integer.MAX_VALUE, supportsConcurrency); ConcurrentDrillSidewaysResult r = search(query, collectorManager); TopDocs topDocs = r.collectorResult; @@ -328,8 +329,9 @@ public DrillSidewaysResult search(ScoreDoc after, DrillDownQuery query, int topN limit = 1; // the collector does not alow numHits = 0 } final int fTopN = Math.min(topN, limit); + final boolean supportsConcurrency = searcher.getExecutor() != null; final TopScoreDocCollectorManager collectorManager = - new TopScoreDocCollectorManager(fTopN, after, Integer.MAX_VALUE); + new TopScoreDocCollectorManager(fTopN, after, Integer.MAX_VALUE, supportsConcurrency); ConcurrentDrillSidewaysResult r = search(query, collectorManager); return new DrillSidewaysResult( r.facets, diff --git a/lucene/luke/src/java/org/apache/lucene/luke/models/search/SearchImpl.java b/lucene/luke/src/java/org/apache/lucene/luke/models/search/SearchImpl.java index a8c515354d5f..bcf347816bb7 100644 --- a/lucene/luke/src/java/org/apache/lucene/luke/models/search/SearchImpl.java +++ b/lucene/luke/src/java/org/apache/lucene/luke/models/search/SearchImpl.java @@ -315,7 +315,8 @@ private SearchResults search() throws IOException { } else { int hitsThreshold = exactHitsCount ? Integer.MAX_VALUE : DEFAULT_TOTAL_HITS_THRESHOLD; TopScoreDocCollectorManager collectorManager = - new TopScoreDocCollectorManager(pageSize, after, hitsThreshold); + new TopScoreDocCollectorManager( + pageSize, after, hitsThreshold, searcher.getExecutor() != null); topDocs = searcher.search(query, collectorManager); } diff --git a/lucene/queries/src/test/org/apache/lucene/queries/spans/TestBasics.java b/lucene/queries/src/test/org/apache/lucene/queries/spans/TestBasics.java index 90c3168c275f..b581a3484e02 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/spans/TestBasics.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/spans/TestBasics.java @@ -510,7 +510,7 @@ public void testBooleanSpanQuery() throws Exception { query.add(sq1, BooleanClause.Occur.SHOULD); query.add(sq2, BooleanClause.Occur.SHOULD); TopScoreDocCollectorManager collectorManager = - TopScoreDocCollectorManager.create(1000, Integer.MAX_VALUE); + new TopScoreDocCollectorManager(1000, Integer.MAX_VALUE); TopDocs topDocs = searcher.search(query.build(), collectorManager); hits = topDocs.scoreDocs.length; for (ScoreDoc scoreDoc : topDocs.scoreDocs) { @@ -546,7 +546,7 @@ public void testDismaxSpanQuery() throws Exception { new SpanTermQuery(new Term(FIELD, "clckwork"))), 1.0f); TopScoreDocCollectorManager collectorManager = - TopScoreDocCollectorManager.create(1000, Integer.MAX_VALUE); + new TopScoreDocCollectorManager(1000, Integer.MAX_VALUE); TopDocs topDocs = searcher.search(query, collectorManager); hits = topDocs.scoreDocs.length; for (ScoreDoc scoreDoc : topDocs.scoreDocs) { diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestCombinedFieldQuery.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestCombinedFieldQuery.java index 27f88292aef3..91c3efc93a3b 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestCombinedFieldQuery.java +++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestCombinedFieldQuery.java @@ -151,7 +151,7 @@ public void testSameScore() throws IOException { .build(); TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager( - Math.min(reader.numDocs(), Integer.MAX_VALUE), null, Integer.MAX_VALUE); + Math.min(reader.numDocs(), Integer.MAX_VALUE), Integer.MAX_VALUE); TopDocs topDocs = searcher.search(query, collectorManager); assertEquals(new TotalHits(11, TotalHits.Relation.EQUAL_TO), topDocs.totalHits); // All docs must have the same score @@ -189,7 +189,7 @@ public void testNormsDisabled() throws IOException { Similarity searchSimilarity = randomCompatibleSimilarity(); searcher.setSimilarity(searchSimilarity); - TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(10, null, 10); + TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(10, 10); CombinedFieldQuery query = new CombinedFieldQuery.Builder() @@ -421,12 +421,12 @@ private static Similarity randomCompatibleSimilarity() { private void checkExpectedHits( IndexSearcher searcher, int numHits, Query firstQuery, Query secondQuery) throws IOException { TopScoreDocCollectorManager collectorManager = - new TopScoreDocCollectorManager(numHits, null, Integer.MAX_VALUE); + new TopScoreDocCollectorManager(numHits, Integer.MAX_VALUE); TopDocs firstTopDocs = searcher.search(firstQuery, collectorManager); assertEquals(numHits, firstTopDocs.totalHits.value); - collectorManager = new TopScoreDocCollectorManager(numHits, null, Integer.MAX_VALUE); + collectorManager = new TopScoreDocCollectorManager(numHits, Integer.MAX_VALUE); TopDocs secondTopDocs = searcher.search(secondQuery, collectorManager); CheckHits.checkEqual(firstQuery, secondTopDocs.scoreDocs, firstTopDocs.scoreDocs); } diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.java index 1b3c138b377f..92e288960065 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.java +++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.java @@ -84,7 +84,7 @@ public void testIllegalArguments() throws IOException { IndexSearcher searcher = newSearcher(reader); LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(15); TopScoreDocCollectorManager regularCollectorManager = - new TopScoreDocCollectorManager(15, null, Integer.MAX_VALUE); + new TopScoreDocCollectorManager(15, Integer.MAX_VALUE); searcher.search(testQuery, largeCollector); TopDocs topDocs = searcher.search(testQuery, regularCollectorManager); @@ -105,7 +105,7 @@ public void testNoPQBuild() throws IOException { IndexSearcher searcher = newSearcher(reader); LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(250_000); TopScoreDocCollectorManager regularCollectorManager = - new TopScoreDocCollectorManager(250_000, null, Integer.MAX_VALUE); + new TopScoreDocCollectorManager(250_000, Integer.MAX_VALUE); searcher.search(testQuery, largeCollector); TopDocs topDocs = searcher.search(testQuery, regularCollectorManager); @@ -120,7 +120,7 @@ public void testPQBuild() throws IOException { IndexSearcher searcher = newSearcher(reader); LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(50); TopScoreDocCollectorManager regularCollectorManager = - new TopScoreDocCollectorManager(50, null, Integer.MAX_VALUE); + new TopScoreDocCollectorManager(50, Integer.MAX_VALUE); searcher.search(testQuery, largeCollector); TopDocs topDocs = searcher.search(testQuery, regularCollectorManager); @@ -135,7 +135,7 @@ public void testNoPQHitsOrder() throws IOException { IndexSearcher searcher = newSearcher(reader); LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(250_000); TopScoreDocCollectorManager regularCollectorManager = - new TopScoreDocCollectorManager(250_000, null, Integer.MAX_VALUE); + new TopScoreDocCollectorManager(250_000, Integer.MAX_VALUE); searcher.search(testQuery, largeCollector); TopDocs topDocs = searcher.search(testQuery, regularCollectorManager); @@ -160,7 +160,7 @@ private void runNumHits(int numHits) throws IOException { IndexSearcher searcher = newSearcher(reader); LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(numHits); TopScoreDocCollectorManager regularCollector = - new TopScoreDocCollectorManager(numHits, null, Integer.MAX_VALUE); + new TopScoreDocCollectorManager(numHits, Integer.MAX_VALUE); searcher.search(testQuery, largeCollector); TopDocs topDocs = searcher.search(testQuery, regularCollector); diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java index 3121a3b27d86..9db90f0d7cae 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java @@ -708,7 +708,6 @@ public List lookup( // System.out.println("finalQuery=" + finalQuery); // Sort by weight, descending: - TopFieldCollectorManager c = TopFieldCollectorManager.create(SORT, num, 1); List results = null; SearcherManager mgr; IndexSearcher searcher; @@ -717,6 +716,8 @@ public List lookup( searcher = mgr.acquire(); } try { + TopFieldCollectorManager c = + new TopFieldCollectorManager(SORT, num, null, 1, searcher.getExecutor() != null); // System.out.println("got searcher=" + searcher); TopFieldDocs hits = searcher.search(finalQuery, c); diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java b/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java index 7e2403851ab6..31df7de32d25 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java @@ -639,9 +639,11 @@ public static void checkTopScores(Random random, Query query, IndexSearcher sear private static void doCheckTopScores(Query query, IndexSearcher searcher, int numHits) throws IOException { TopScoreDocCollectorManager collectorManager1 = - new TopScoreDocCollectorManager(numHits, null, Integer.MAX_VALUE); // COMPLETE + new TopScoreDocCollectorManager( + numHits, null, Integer.MAX_VALUE, searcher.getExecutor() != null); // COMPLETE TopScoreDocCollectorManager collectorManager2 = - new TopScoreDocCollectorManager(numHits, null, 1); // TOP_SCORES + new TopScoreDocCollectorManager( + numHits, null, 1, searcher.getExecutor() != null); // TOP_SCORES TopDocs topDocs1 = searcher.search(query, collectorManager1); TopDocs topDocs2 = searcher.search(query, collectorManager2); checkEqual(query, topDocs1.scoreDocs, topDocs2.scoreDocs); From 86bac5216089855ff66f2194acfaca91f8ae3af4 Mon Sep 17 00:00:00 2001 From: Zach Chen Date: Wed, 18 Aug 2021 21:13:24 -0700 Subject: [PATCH 05/14] Add the same hack used in DrillSideways#search(query, collector) to DrillSideways#searchSequentially(query, collectorManager) to by pass caching This is needed to pass test case from ./gradlew test --tests TestDrillSideways.testRandom -Dtests.seed=69A3EF02D8E3465E -Dtests.nightly=true --- .../search/TopFieldCollectorManager.java | 12 +++++- .../apache/lucene/facet/DrillSideways.java | 41 ++++++++++++++++--- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java index ed97ca775951..c3a7f98efd26 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java @@ -155,8 +155,16 @@ public TopFieldCollector newCollector() { public TopFieldDocs reduce(Collection collectors) throws IOException { final TopFieldDocs[] topDocs = new TopFieldDocs[collectors.size()]; int i = 0; - for (TopFieldCollector collector : collectors) { - topDocs[i++] = collector.topDocs(); + for (Collector collector : collectors) { + if (collector instanceof FilterCollector) { + Collector delegateCollector = ((FilterCollector) collector).in; + + if (delegateCollector instanceof TopFieldCollector) { + topDocs[i++] = ((TopFieldCollector) delegateCollector).topDocs(); + } + } else if (collector instanceof TopFieldCollector) { + topDocs[i++] = ((TopFieldCollector) collector).topDocs(); + } } return TopDocs.merge(sort, 0, numHits, topDocs); } diff --git a/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java b/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java index e50367c45963..45b2e125eb45 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -457,10 +458,39 @@ public ConcurrentDrillSidewaysResult search( } @SuppressWarnings("unchecked") - private ConcurrentDrillSidewaysResult searchSequentially( - final DrillDownQuery query, final CollectorManager hitCollectorManager) + private ConcurrentDrillSidewaysResult searchSequentially( + final DrillDownQuery query, final CollectorManager hitCollectorManager) throws IOException { + // This mirrors a similar hack from DrillSideways#search(query, collector). + // Without this cache, LRU cache will be used, causing acceptDocs to be null during collection + // for drillDown and drillSideways and returning more results + // + // This is a horrible hack in order to make sure IndexSearcher will not + // attempt to cache the DrillSidewaysQuery + final CollectorManager filteredCollectorManager = + new CollectorManager<>() { + @Override + public C newCollector() throws IOException { + Collector hitCollector = hitCollectorManager.newCollector(); + if (hitCollector.scoreMode().needsScores() == false) { + hitCollector = + new FilterCollector(hitCollector) { + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE; + } + }; + } + return (C) hitCollector; + } + + @Override + public R reduce(Collection collectors) throws IOException { + return hitCollectorManager.reduce(collectors); + } + }; + Map drillDownDims = query.getDims(); if (drillDownDims.isEmpty()) { @@ -472,13 +502,14 @@ private ConcurrentDrillSidewaysResult searchSequentially( if (drillDownCollectorManager != null) { Object[] mainResults = searcher.search( - query, new MultiCollectorManager(drillDownCollectorManager, hitCollectorManager)); + query, + new MultiCollectorManager(drillDownCollectorManager, filteredCollectorManager)); // Extract the results: mainFacetsCollector = (FacetsCollector) mainResults[0]; collectorResult = (R) mainResults[1]; } else { mainFacetsCollector = null; - collectorResult = searcher.search(query, hitCollectorManager); + collectorResult = searcher.search(query, filteredCollectorManager); } return new ConcurrentDrillSidewaysResult<>( @@ -516,7 +547,7 @@ private ConcurrentDrillSidewaysResult searchSequentially( drillDownQueries, scoreSubDocsAtOnce()); - R collectorResult = searcher.search(dsq, hitCollectorManager); + R collectorResult = searcher.search(dsq, filteredCollectorManager); FacetsCollector drillDownCollector; if (drillDownCollectorManager != null) { From 3a3a86ab1af6857680b795cefeb62bb608020a6a Mon Sep 17 00:00:00 2001 From: Zach Chen Date: Thu, 19 Aug 2021 23:52:03 -0700 Subject: [PATCH 06/14] Update per feedback --- .../apache/lucene/benchmark/byTask/tasks/ReadTask.java | 8 ++------ .../java/org/apache/lucene/search/IndexSearcher.java | 2 +- .../apache/lucene/document/BaseSpatialTestCase.java | 10 +++++----- .../org/apache/lucene/search/TestBooleanQuery.java | 2 +- .../org/apache/lucene/search/TestPointQueries.java | 6 ++++-- .../java/org/apache/lucene/facet/DrillSideways.java | 2 +- .../lucene/search/highlight/TestHighlighterPhrase.java | 3 ++- .../lucene/spatial/prefix/TestNumberRangeFacets.java | 2 +- .../org/apache/lucene/spatial3d/TestGeo3DPoint.java | 2 +- .../org/apache/lucene/geo/BaseGeoPointTestCase.java | 2 +- .../org/apache/lucene/geo/BaseXYPointTestCase.java | 2 +- .../lucene/search/BaseRangeFieldQueryTestCase.java | 2 +- .../org/apache/lucene/util/FixedBitSetCollector.java | 6 +++--- 13 files changed, 24 insertions(+), 25 deletions(-) diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java index 2ddd33a66dc9..8b4d3f505ecd 100644 --- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java +++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java @@ -31,7 +31,6 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopFieldCollectorManager; import org.apache.lucene.search.TopScoreDocCollector; -import org.apache.lucene.search.TopScoreDocCollectorManager; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; @@ -118,11 +117,8 @@ public int doLogic() throws Exception { hits = searcher.search(q, numHits); } } else { - int totalHitsThreshold = withTotalHits() ? Integer.MAX_VALUE : 1; - TopScoreDocCollectorManager collectorManager = - new TopScoreDocCollectorManager( - numHits(), null, totalHitsThreshold, searcher.getExecutor() != null); - searcher.search(q, collectorManager); + Collector collector = createCollector(); + searcher.search(q, collector); // hits = collectorManager.topDocs(); } diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java index d082d46e1f75..172d2bbc957b 100644 --- a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java +++ b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java @@ -614,7 +614,7 @@ private TopFieldDocs searchAfter( */ public T search(Query query, CollectorManager collectorManager) throws IOException { - if (executor == null || leafSlices.length == 0) { + if (executor == null || leafSlices.length <= 1) { final C collector = collectorManager.newCollector(); final Query rewrittenQuery = rewrite(query); diff --git a/lucene/core/src/test/org/apache/lucene/document/BaseSpatialTestCase.java b/lucene/core/src/test/org/apache/lucene/document/BaseSpatialTestCase.java index 122e8e970618..178ac718f6a3 100644 --- a/lucene/core/src/test/org/apache/lucene/document/BaseSpatialTestCase.java +++ b/lucene/core/src/test/org/apache/lucene/document/BaseSpatialTestCase.java @@ -320,7 +320,7 @@ protected void verifyRandomBBoxQueries(IndexReader reader, Object... shapes) thr System.out.println(" query=" + query + ", relation=" + queryRelation); } - final FixedBitSet hits = s.search(query, FixedBitSetCollector.create(maxDoc)); + final FixedBitSet hits = s.search(query, FixedBitSetCollector.createManager(maxDoc)); boolean fail = false; NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id"); @@ -417,7 +417,7 @@ protected void verifyRandomLineQueries(IndexReader reader, Object... shapes) thr System.out.println(" query=" + query + ", relation=" + queryRelation); } - final FixedBitSet hits = s.search(query, FixedBitSetCollector.create(maxDoc)); + final FixedBitSet hits = s.search(query, FixedBitSetCollector.createManager(maxDoc)); boolean fail = false; NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id"); @@ -490,7 +490,7 @@ protected void verifyRandomPolygonQueries(IndexReader reader, Object... shapes) System.out.println(" query=" + query + ", relation=" + queryRelation); } - final FixedBitSet hits = s.search(query, FixedBitSetCollector.create(maxDoc)); + final FixedBitSet hits = s.search(query, FixedBitSetCollector.createManager(maxDoc)); boolean fail = false; NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id"); @@ -569,7 +569,7 @@ protected void verifyRandomPointQueries(IndexReader reader, Object... shapes) th System.out.println(" query=" + query + ", relation=" + queryRelation); } - final FixedBitSet hits = s.search(query, FixedBitSetCollector.create(maxDoc)); + final FixedBitSet hits = s.search(query, FixedBitSetCollector.createManager(maxDoc)); boolean fail = false; NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id"); @@ -644,7 +644,7 @@ protected void verifyRandomDistanceQueries(IndexReader reader, Object... shapes) System.out.println(" query=" + query + ", relation=" + queryRelation); } - final FixedBitSet hits = s.search(query, FixedBitSetCollector.create(maxDoc)); + final FixedBitSet hits = s.search(query, FixedBitSetCollector.createManager(maxDoc)); boolean fail = false; NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id"); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java index 15c289cba50b..ff97a827db24 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java @@ -412,7 +412,7 @@ public void testMinShouldMatchLeniency() throws Exception { } private static FixedBitSet getMatches(IndexSearcher searcher, Query query) throws IOException { - return searcher.search(query, FixedBitSetCollector.create(searcher.reader.maxDoc())); + return searcher.search(query, FixedBitSetCollector.createManager(searcher.reader.maxDoc())); } public void testFILTERClauseBehavesLikeMUST() throws IOException { diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java index 62341d9ea3aa..3c1f84b2513d 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java @@ -573,7 +573,8 @@ private void _run() throws Exception { System.out.println(Thread.currentThread().getName() + ": using query: " + query); } - final FixedBitSet hits = s.search(query, FixedBitSetCollector.create(r.maxDoc())); + final FixedBitSet hits = + s.search(query, FixedBitSetCollector.createManager(r.maxDoc())); if (VERBOSE) { System.out.println( @@ -850,7 +851,8 @@ private void _run() throws Exception { System.out.println(Thread.currentThread().getName() + ": using query: " + query); } - final FixedBitSet hits = s.search(query, FixedBitSetCollector.create(r.maxDoc())); + final FixedBitSet hits = + s.search(query, FixedBitSetCollector.createManager(r.maxDoc())); if (VERBOSE) { System.out.println( diff --git a/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java b/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java index 45b2e125eb45..2bf5be3ef191 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java @@ -463,7 +463,7 @@ private ConcurrentDrillSidewaysResult searchSequenti throws IOException { // This mirrors a similar hack from DrillSideways#search(query, collector). - // Without this cache, LRU cache will be used, causing acceptDocs to be null during collection + // Without this hack, LRU cache will be used, causing acceptDocs to be null during collection // for drillDown and drillSideways and returning more results // // This is a horrible hack in order to make sure IndexSearcher will not diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighterPhrase.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighterPhrase.java index c19ee0d90913..d42d5e3980b4 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighterPhrase.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighterPhrase.java @@ -121,7 +121,8 @@ public void testConcurrentSpan() throws IOException, InvalidTokenOffsetsExceptio 0, true); final FixedBitSet bitset = - indexSearcher.search(phraseQuery, FixedBitSetCollector.create(indexReader.maxDoc())); + indexSearcher.search( + phraseQuery, FixedBitSetCollector.createManager(indexReader.maxDoc())); assertEquals(1, bitset.cardinality()); final int maxDoc = indexReader.maxDoc(); final Highlighter highlighter = diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestNumberRangeFacets.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestNumberRangeFacets.java index a6133da78121..5ac626695546 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestNumberRangeFacets.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestNumberRangeFacets.java @@ -217,7 +217,7 @@ public void test() throws IOException { private Bits searchForDocBits(Query query) throws IOException { return indexSearcher.search( - query, FixedBitSetCollector.create(indexSearcher.getIndexReader().maxDoc())); + query, FixedBitSetCollector.createManager(indexSearcher.getIndexReader().maxDoc())); } private void preQueryHavoc() { diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index cef9ab041ba6..47c1509b0be1 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -1037,7 +1037,7 @@ private static void verify(double[] lats, double[] lons, final PlanetModel plane System.err.println(" using query: " + query); } - final FixedBitSet hits = s.search(query, FixedBitSetCollector.create(r.maxDoc())); + final FixedBitSet hits = s.search(query, FixedBitSetCollector.createManager(r.maxDoc())); if (VERBOSE) { System.err.println(" hitCount: " + hits.cardinality()); diff --git a/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java index 37885dcddd03..63f569dd4fd9 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java @@ -1273,7 +1273,7 @@ private void indexPoints(double[] lats, double[] lons, Set deleted, Ind } private FixedBitSet searchIndex(IndexSearcher s, Query query, int maxDoc) throws IOException { - return s.search(query, FixedBitSetCollector.create(maxDoc)); + return s.search(query, FixedBitSetCollector.createManager(maxDoc)); } private void buildError( diff --git a/lucene/test-framework/src/java/org/apache/lucene/geo/BaseXYPointTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/geo/BaseXYPointTestCase.java index 3c2f524e739e..caebb4f484c0 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/geo/BaseXYPointTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/geo/BaseXYPointTestCase.java @@ -1160,7 +1160,7 @@ private void indexPoints(float[] xs, float[] ys, Set deleted, IndexWrit } private FixedBitSet searchIndex(IndexSearcher s, Query query, int maxDoc) throws IOException { - return s.search(query, FixedBitSetCollector.create(maxDoc)); + return s.search(query, FixedBitSetCollector.createManager(maxDoc)); } private void buildError( diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java index 6dc9a3989849..9c2e1f1a394f 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/BaseRangeFieldQueryTestCase.java @@ -292,7 +292,7 @@ private void verify(Range[][] ranges) throws Exception { System.out.println(" query=" + query); } - FixedBitSet hits = s.search(query, FixedBitSetCollector.create(maxDoc)); + FixedBitSet hits = s.search(query, FixedBitSetCollector.createManager(maxDoc)); NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id"); for (int docID = 0; docID < maxDoc; ++docID) { diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/FixedBitSetCollector.java b/lucene/test-framework/src/java/org/apache/lucene/util/FixedBitSetCollector.java index 629a389d295e..bd9f41300d28 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/FixedBitSetCollector.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/FixedBitSetCollector.java @@ -24,7 +24,7 @@ /** Test utility collector that uses FixedBitSet to record hits. */ public class FixedBitSetCollector extends SimpleCollector { - private FixedBitSet hits; + private final FixedBitSet hits; private int docBase; public FixedBitSetCollector(int maxDoc) { @@ -50,7 +50,7 @@ public FixedBitSet getHits() { return hits; } - public static CollectorManager create(int maxDoc) { + public static CollectorManager createManager(int maxDoc) { return new CollectorManager<>() { @Override public FixedBitSetCollector newCollector() { @@ -60,7 +60,7 @@ public FixedBitSetCollector newCollector() { @Override public FixedBitSet reduce(Collection collectors) { FixedBitSet result = new FixedBitSet(maxDoc); - collectors.stream().forEach(c -> result.or(c.getHits())); + collectors.forEach(c -> result.or(c.getHits())); return result; } }; From d21e5f9447523405db1cad2ad9357cabfc260180 Mon Sep 17 00:00:00 2001 From: Zach Chen Date: Tue, 24 Aug 2021 20:36:17 -0700 Subject: [PATCH 07/14] Use supportsConcurrency to detect potentially incorrect usage --- .../search/TopFieldCollectorManager.java | 10 ++++ .../search/TopScoreDocCollectorManager.java | 10 ++++ .../lucene/search/TestTopDocsCollector.java | 47 ++++++++++++++++++ .../lucene/search/TestTopFieldCollector.java | 49 +++++++++++++++++++ 4 files changed, 116 insertions(+) diff --git a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java index c3a7f98efd26..d1868b82df7d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java @@ -35,6 +35,8 @@ public class TopFieldCollectorManager implements CollectorManager collectors; + private final boolean supportsConcurrency; + private boolean collectorCreated; /** * Creates a new {@link TopFieldCollectorManager} from the given arguments. @@ -62,6 +64,7 @@ public TopFieldCollectorManager( this.sort = sort; this.numHits = numHits; this.after = after; + this.supportsConcurrency = supportsConcurrency; this.hitsThresholdChecker = supportsConcurrency ? HitsThresholdChecker.createShared(Math.max(totalHitsThreshold, numHits)) @@ -111,6 +114,13 @@ public TopFieldCollectorManager(Sort sort, int numHits, int totalHitsThreshold) @Override public TopFieldCollector newCollector() { + if (collectorCreated && supportsConcurrency == false) { + throw new IllegalStateException( + "The instantiated TopFieldCollectorManager does not support concurrency, but multiple collectors are being created"); + } else { + collectorCreated = true; + } + if (sort.fields.length == 0) { throw new IllegalArgumentException("Sort must contain at least one field"); } diff --git a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java index 37a934e0a94f..3c20c2bb3fcc 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java @@ -31,6 +31,8 @@ public class TopScoreDocCollectorManager private final ScoreDoc after; private final HitsThresholdChecker hitsThresholdChecker; private final MaxScoreAccumulator minScoreAcc; + private final boolean supportsConcurrency; + private boolean collectorCreated; /** * Creates a new {@link TopScoreDocCollectorManager} given the number of hits to collect and the @@ -63,6 +65,7 @@ public TopScoreDocCollectorManager( this.numHits = numHits; this.after = after; + this.supportsConcurrency = supportsConcurrency; this.hitsThresholdChecker = supportsConcurrency ? HitsThresholdChecker.createShared(Math.max(totalHitsThreshold, numHits)) @@ -121,6 +124,13 @@ public TopScoreDocCollectorManager(int numHits, int totalHitsThreshold) { @Override public TopScoreDocCollector newCollector() { + if (collectorCreated && supportsConcurrency == false) { + throw new IllegalStateException( + "The instantiated TopScoreDocCollectorManager does not support concurrency, but multiple collectors are being created"); + } else { + collectorCreated = true; + } + if (after == null) { return new TopScoreDocCollector.SimpleTopScoreDocCollector( numHits, hitsThresholdChecker, minScoreAcc); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java index f4f6f41d71b8..4ad709fde24c 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -718,4 +719,50 @@ public void testRealisticConcurrentMinimumScore() throws Exception { reader.close(); dir.close(); } + + public void testInvalidConcurrentSearchIncorrectlyConfiguredCollectorManager() + throws IOException { + try (Directory dir = newDirectory(); + IndexWriter w = + new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE))) { + Document doc = new Document(); + w.addDocuments(Arrays.asList(doc, doc, doc, doc)); + w.flush(); + w.addDocuments(Arrays.asList(doc, doc, doc, doc)); + w.flush(); + + try (IndexReader reader = DirectoryReader.open(w)) { + ExecutorService service = + new ThreadPoolExecutor( + 4, + 4, + 0L, + TimeUnit.MILLISECONDS, + new LinkedBlockingQueue(), + new NamedThreadFactory("TestTopDocsCollector")); + try { + expectThrows( + IllegalStateException.class, + () -> { + int maxDocPerSlice = 1; + int maxSegmentsPerSlice = 1; + IndexSearcher searcher = + new IndexSearcher(reader, service) { + @Override + protected LeafSlice[] slices(List leaves) { + return slices(leaves, maxDocPerSlice, maxSegmentsPerSlice); + } + }; + + TopScoreDocCollectorManager collectorManager = + new TopScoreDocCollectorManager(1, null, 1, false); + + searcher.search(new MatchAllDocsQuery(), collectorManager); + }); + } finally { + service.shutdown(); + } + } + } + } } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java index e1acb0417367..d62deb20e81c 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Comparator; +import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -771,4 +772,52 @@ public void testRelationVsTopDocsCount() throws Exception { } } } + + public void testInvalidConcurrentSearchIncorrectlyConfiguredCollectorManager() + throws IOException { + try (Directory dir = newDirectory(); + IndexWriter w = + new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE))) { + Document doc = new Document(); + doc.add(new TextField("f", "foo bar", Store.NO)); + w.addDocuments(Arrays.asList(doc, doc, doc, doc, doc)); + w.flush(); + w.addDocuments(Arrays.asList(doc, doc, doc, doc, doc)); + w.flush(); + + try (IndexReader reader = DirectoryReader.open(w)) { + ExecutorService service = + new ThreadPoolExecutor( + 4, + 4, + 0L, + TimeUnit.MILLISECONDS, + new LinkedBlockingQueue(), + new NamedThreadFactory("TestTopFieldCollector")); + try { + expectThrows( + IllegalStateException.class, + () -> { + int maxDocPerSlice = 1; + int maxSegmentsPerSlice = 1; + IndexSearcher searcher = + new IndexSearcher(reader, service) { + @Override + protected LeafSlice[] slices(List leaves) { + return slices(leaves, maxDocPerSlice, maxSegmentsPerSlice); + } + }; + + Sort sort = new Sort(SortField.FIELD_SCORE, SortField.FIELD_DOC); + TopFieldCollectorManager collectorManager = + new TopFieldCollectorManager(sort, 1, null, 1, false); + + searcher.search(new MatchAllDocsQuery(), collectorManager); + }); + } finally { + service.shutdown(); + } + } + } + } } From be6977d1c4d720c2e506461bcabb011032df97f8 Mon Sep 17 00:00:00 2001 From: Zach Chen Date: Wed, 25 Aug 2021 19:33:02 -0700 Subject: [PATCH 08/14] Revert "Add the same hack used in DrillSideways#search(query, collector) to DrillSideways#searchSequentially(query, collectorManager) to by pass caching" This reverts commit 86bac5216089855ff66f2194acfaca91f8ae3af4. --- .../search/TopFieldCollectorManager.java | 12 +----- .../apache/lucene/facet/DrillSideways.java | 41 +++---------------- 2 files changed, 7 insertions(+), 46 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java index d1868b82df7d..c334042b6c32 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java @@ -165,16 +165,8 @@ public TopFieldCollector newCollector() { public TopFieldDocs reduce(Collection collectors) throws IOException { final TopFieldDocs[] topDocs = new TopFieldDocs[collectors.size()]; int i = 0; - for (Collector collector : collectors) { - if (collector instanceof FilterCollector) { - Collector delegateCollector = ((FilterCollector) collector).in; - - if (delegateCollector instanceof TopFieldCollector) { - topDocs[i++] = ((TopFieldCollector) delegateCollector).topDocs(); - } - } else if (collector instanceof TopFieldCollector) { - topDocs[i++] = ((TopFieldCollector) collector).topDocs(); - } + for (TopFieldCollector collector : collectors) { + topDocs[i++] = collector.topDocs(); } return TopDocs.merge(sort, 0, numHits, topDocs); } diff --git a/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java b/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java index 2bf5be3ef191..e50367c45963 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java @@ -18,7 +18,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -458,39 +457,10 @@ public ConcurrentDrillSidewaysResult search( } @SuppressWarnings("unchecked") - private ConcurrentDrillSidewaysResult searchSequentially( - final DrillDownQuery query, final CollectorManager hitCollectorManager) + private ConcurrentDrillSidewaysResult searchSequentially( + final DrillDownQuery query, final CollectorManager hitCollectorManager) throws IOException { - // This mirrors a similar hack from DrillSideways#search(query, collector). - // Without this hack, LRU cache will be used, causing acceptDocs to be null during collection - // for drillDown and drillSideways and returning more results - // - // This is a horrible hack in order to make sure IndexSearcher will not - // attempt to cache the DrillSidewaysQuery - final CollectorManager filteredCollectorManager = - new CollectorManager<>() { - @Override - public C newCollector() throws IOException { - Collector hitCollector = hitCollectorManager.newCollector(); - if (hitCollector.scoreMode().needsScores() == false) { - hitCollector = - new FilterCollector(hitCollector) { - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE; - } - }; - } - return (C) hitCollector; - } - - @Override - public R reduce(Collection collectors) throws IOException { - return hitCollectorManager.reduce(collectors); - } - }; - Map drillDownDims = query.getDims(); if (drillDownDims.isEmpty()) { @@ -502,14 +472,13 @@ public R reduce(Collection collectors) throws IOException { if (drillDownCollectorManager != null) { Object[] mainResults = searcher.search( - query, - new MultiCollectorManager(drillDownCollectorManager, filteredCollectorManager)); + query, new MultiCollectorManager(drillDownCollectorManager, hitCollectorManager)); // Extract the results: mainFacetsCollector = (FacetsCollector) mainResults[0]; collectorResult = (R) mainResults[1]; } else { mainFacetsCollector = null; - collectorResult = searcher.search(query, filteredCollectorManager); + collectorResult = searcher.search(query, hitCollectorManager); } return new ConcurrentDrillSidewaysResult<>( @@ -547,7 +516,7 @@ public R reduce(Collection collectors) throws IOException { drillDownQueries, scoreSubDocsAtOnce()); - R collectorResult = searcher.search(dsq, filteredCollectorManager); + R collectorResult = searcher.search(dsq, hitCollectorManager); FacetsCollector drillDownCollector; if (drillDownCollectorManager != null) { From 81163def27febd839a01c71cbff4a8c98ddfe915 Mon Sep 17 00:00:00 2001 From: Zach Chen Date: Thu, 16 Sep 2021 21:15:58 -0700 Subject: [PATCH 09/14] address comment --- .../benchmark/byTask/tasks/ReadTask.java | 16 +++---- .../byTask/tasks/SearchWithCollectorTask.java | 16 ------- .../CachingNaiveBayesClassifier.java | 3 +- .../SimpleNaiveBayesClassifier.java | 5 +- .../SimpleNaiveBayesDocumentClassifier.java | 3 +- .../apache/lucene/search/IndexSearcher.java | 21 +-------- .../search/TopFieldCollectorManager.java | 46 ++++++++++--------- .../search/TopScoreDocCollectorManager.java | 5 ++ .../org/apache/lucene/index/TestOmitTf.java | 3 +- .../search/TestFuzzyTermOnShortTerms.java | 2 +- .../lucene/search/TestLRUQueryCache.java | 8 ++-- .../search/TestTotalHitCountCollector.java | 3 +- .../lucene/search/join/TestJoinUtil.java | 2 +- .../TestLargeNumHitsTopDocsCollector.java | 10 ++-- .../prefix/TestHeatmapFacetCounter.java | 3 +- 15 files changed, 52 insertions(+), 94 deletions(-) diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java index 8b4d3f505ecd..fa029ffe20db 100644 --- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java +++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java @@ -23,14 +23,13 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiBits; -import org.apache.lucene.search.Collector; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopFieldCollectorManager; -import org.apache.lucene.search.TopScoreDocCollector; +import org.apache.lucene.search.TopScoreDocCollectorManager; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; @@ -117,9 +116,11 @@ public int doLogic() throws Exception { hits = searcher.search(q, numHits); } } else { - Collector collector = createCollector(); - searcher.search(q, collector); - // hits = collectorManager.topDocs(); + int totalHitsThreshold = withTotalHits() ? Integer.MAX_VALUE : 1; + TopScoreDocCollectorManager collectorManager = + new TopScoreDocCollectorManager( + numHits(), null, totalHitsThreshold, searcher.getExecutor() != null); + hits = searcher.search(q, collectorManager); } if (hits != null) { @@ -181,11 +182,6 @@ protected int withTopDocs(IndexSearcher searcher, Query q, TopDocs hits) throws return res; } - @Deprecated - protected Collector createCollector() throws Exception { - return TopScoreDocCollector.create(numHits(), withTotalHits() ? Integer.MAX_VALUE : 1); - } - protected Document retrieveDoc(IndexReader ir, int id) throws IOException { return ir.document(id); } diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchWithCollectorTask.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchWithCollectorTask.java index 9f79e51d5057..65a5098ab1d0 100644 --- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchWithCollectorTask.java +++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchWithCollectorTask.java @@ -19,8 +19,6 @@ import org.apache.lucene.benchmark.byTask.PerfRunData; import org.apache.lucene.benchmark.byTask.feeds.QueryMaker; import org.apache.lucene.benchmark.byTask.utils.Config; -import org.apache.lucene.search.Collector; -import org.apache.lucene.search.TopScoreDocCollector; /** Does search w/ a custom collector */ public class SearchWithCollectorTask extends SearchTask { @@ -45,20 +43,6 @@ public boolean withCollector() { return true; } - @Override - protected Collector createCollector() throws Exception { - Collector collector = null; - if (clnName.equalsIgnoreCase("topScoreDoc") == true) { - collector = TopScoreDocCollector.create(numHits(), Integer.MAX_VALUE); - } else if (clnName.length() > 0) { - collector = Class.forName(clnName).asSubclass(Collector.class).getConstructor().newInstance(); - - } else { - collector = super.createCollector(); - } - return collector; - } - @Override public QueryMaker getQueryMaker() { return getRunData().getQueryMaker(this); diff --git a/lucene/classification/src/java/org/apache/lucene/classification/CachingNaiveBayesClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/CachingNaiveBayesClassifier.java index 98576e618efe..deff4202167a 100644 --- a/lucene/classification/src/java/org/apache/lucene/classification/CachingNaiveBayesClassifier.java +++ b/lucene/classification/src/java/org/apache/lucene/classification/CachingNaiveBayesClassifier.java @@ -32,7 +32,6 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TotalHitCountCollectorManager; import org.apache.lucene.util.BytesRef; /** @@ -180,7 +179,7 @@ private Map getWordFreqForClassess(String word) throws IOExce booleanQuery.add(query, BooleanClause.Occur.MUST); } - int ret = indexSearcher.search(booleanQuery.build(), new TotalHitCountCollectorManager()); + int ret = indexSearcher.count(booleanQuery.build()); if (ret != 0) { searched.put(cclass, ret); } diff --git a/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java index fc2b909e27d9..2c3610c0bd86 100644 --- a/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java +++ b/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java @@ -35,7 +35,6 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TotalHitCountCollectorManager; import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.util.BytesRef; @@ -178,7 +177,7 @@ protected int countDocsWithClass() throws IOException { if (query != null) { q.add(query, BooleanClause.Occur.MUST); } - docCount = indexSearcher.search(q.build(), new TotalHitCountCollectorManager()); + docCount = indexSearcher.count(q.build()); } else { docCount = terms.getDocCount(); } @@ -274,7 +273,7 @@ private int getWordFreqForClass(String word, Term term) throws IOException { if (query != null) { booleanQuery.add(query, BooleanClause.Occur.MUST); } - return indexSearcher.search(booleanQuery.build(), new TotalHitCountCollectorManager()); + return indexSearcher.count(booleanQuery.build()); } private double calculateLogPrior(Term term, int docsWithClassSize) throws IOException { diff --git a/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java index 184c41b9fcb8..b777f8dc252f 100644 --- a/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java +++ b/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java @@ -40,7 +40,6 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TotalHitCountCollectorManager; import org.apache.lucene.util.BytesRef; /** @@ -263,7 +262,7 @@ private int getWordFreqForClass(String word, String fieldName, Term term) throws if (query != null) { booleanQuery.add(query, BooleanClause.Occur.MUST); } - return indexSearcher.search(booleanQuery.build(), new TotalHitCountCollectorManager()); + return indexSearcher.count(booleanQuery.build()); } private double calculateLogPrior(Term term, int docsWithClassSize) throws IOException { diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java index 172d2bbc957b..367b70a6852f 100644 --- a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java +++ b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -426,25 +425,7 @@ public int count(Query query) throws IOException { return count; } - // general case: create a collector and count matches - final CollectorManager collectorManager = - new CollectorManager() { - - @Override - public TotalHitCountCollector newCollector() throws IOException { - return new TotalHitCountCollector(); - } - - @Override - public Integer reduce(Collection collectors) throws IOException { - int total = 0; - for (TotalHitCountCollector collector : collectors) { - total += collector.getTotalHits(); - } - return total; - } - }; - return search(query, collectorManager); + return search(query, new TotalHitCountCollectorManager()); } /** diff --git a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java index c334042b6c32..59d76ec9628a 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java @@ -61,6 +61,30 @@ public TopFieldCollectorManager( "totalHitsThreshold must be >= 0, got " + totalHitsThreshold); } + if (numHits <= 0) { + throw new IllegalArgumentException( + "numHits must be > 0; please use TotalHitCountCollector if you just need the total hit count"); + } + + if (sort.getSort().length == 0) { + throw new IllegalArgumentException("Sort must contain at least one field"); + } + + if (after != null) { + if (after.fields == null) { + throw new IllegalArgumentException( + "after.fields wasn't set; you must pass fillFields=true for the previous search"); + } + + if (after.fields.length != sort.getSort().length) { + throw new IllegalArgumentException( + "after.fields has " + + after.fields.length + + " values but sort has " + + sort.getSort().length); + } + } + this.sort = sort; this.numHits = numHits; this.after = after; @@ -121,15 +145,6 @@ public TopFieldCollector newCollector() { collectorCreated = true; } - if (sort.fields.length == 0) { - throw new IllegalArgumentException("Sort must contain at least one field"); - } - - if (numHits <= 0) { - throw new IllegalArgumentException( - "numHits must be > 0; please use TotalHitCountCollector if you just need the total hit count"); - } - FieldValueHitQueue queue = FieldValueHitQueue.create(sort.fields, numHits); @@ -139,19 +154,6 @@ public TopFieldCollector newCollector() { new TopFieldCollector.SimpleFieldCollector( sort, queue, numHits, hitsThresholdChecker, minScoreAcc); } else { - if (after.fields == null) { - throw new IllegalArgumentException( - "after.fields wasn't set; you must pass fillFields=true for the previous search"); - } - - if (after.fields.length != sort.getSort().length) { - throw new IllegalArgumentException( - "after.fields has " - + after.fields.length - + " values but sort has " - + sort.getSort().length); - } - collector = new TopFieldCollector.PagingFieldCollector( sort, queue, after, numHits, hitsThresholdChecker, minScoreAcc); diff --git a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java index 3c20c2bb3fcc..642e2380cc12 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java @@ -58,6 +58,11 @@ public class TopScoreDocCollectorManager */ public TopScoreDocCollectorManager( int numHits, ScoreDoc after, int totalHitsThreshold, boolean supportsConcurrency) { + if (totalHitsThreshold < 0) { + throw new IllegalArgumentException( + "totalHitsThreshold must be >= 0, got " + totalHitsThreshold); + } + if (numHits <= 0) { throw new IllegalArgumentException( "numHits must be > 0; please use TotalHitCountCollectorManager if you just need the total hit count"); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java b/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java index bf3cfa7ae42c..32a098ba8c97 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java @@ -36,7 +36,6 @@ import org.apache.lucene.search.SimpleCollector; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermStatistics; -import org.apache.lucene.search.TotalHitCountCollectorManager; import org.apache.lucene.search.similarities.TFIDFSimilarity; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; @@ -368,7 +367,7 @@ public Object reduce(Collection collectors) throws IOExcep bq.add(q1, Occur.MUST); bq.add(q4, Occur.MUST); - int count = searcher.search(bq.build(), new TotalHitCountCollectorManager()); + int count = searcher.count(bq.build()); assertEquals(15, count); reader.close(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestFuzzyTermOnShortTerms.java b/lucene/core/src/test/org/apache/lucene/search/TestFuzzyTermOnShortTerms.java index 57399120b2cd..14b2cf0fb08a 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestFuzzyTermOnShortTerms.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestFuzzyTermOnShortTerms.java @@ -62,7 +62,7 @@ private void countHits(Analyzer analyzer, String[] docs, Query q, int expected) Directory d = getDirectory(analyzer, docs); IndexReader r = DirectoryReader.open(d); IndexSearcher s = new IndexSearcher(r); - int totalHits = s.search(q, new TotalHitCountCollectorManager()); + int totalHits = s.count(q); assertEquals(q.toString(), expected, totalHits); r.close(); d.close(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java index a7d5580ba5eb..84d5d45e8ea8 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java @@ -163,9 +163,7 @@ public void run() { RandomPicks.randomFrom( random(), new String[] {"blue", "red", "yellow", "green"}); final Query q = new TermQuery(new Term("color", value)); - final int totalHits1 = - searcher.search( - q, new TotalHitCountCollectorManager()); // will use the cache + final int totalHits1 = searcher.count(q); // will use the cache TotalHitCountCollector collector2 = new TotalHitCountCollector(); searcher.search( q, @@ -902,7 +900,7 @@ public void onUse(Query query) { searcher.setQueryCache(queryCache); searcher.setQueryCachingPolicy(policy); - searcher.search(query.build(), new TotalHitCountCollectorManager()); + searcher.count(query.build()); reader.close(); dir.close(); @@ -1120,7 +1118,7 @@ public void testDetectMutatedQueries() throws IOException { try { // trigger an eviction - searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager()); + searcher.count(new MatchAllDocsQuery()); fail(); } catch ( @SuppressWarnings("unused") diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java index 57aa69dd0a9c..b3969e77c660 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java @@ -39,8 +39,7 @@ public void testBasics() throws Exception { writer.close(); IndexSearcher searcher = newSearcher(reader); - TotalHitCountCollectorManager c = new TotalHitCountCollectorManager(); - assertEquals(5, searcher.search(new MatchAllDocsQuery(), c).intValue()); + assertEquals(5, searcher.count(new MatchAllDocsQuery())); reader.close(); indexStore.close(); } diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java index 542131dc7f5e..a8927742fc4d 100644 --- a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java +++ b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java @@ -681,7 +681,7 @@ public void testMinMaxDocs() throws Exception { Query joinQuery = JoinUtil.createJoinQuery( "join_field", fromQuery, toQuery, searcher, scoreMode, ordinalMap, min, max); - int totalHits = searcher.search(joinQuery, new TotalHitCountCollectorManager()); + int totalHits = searcher.count(joinQuery); int expectedCount = 0; for (int numChildDocs : childDocsPerParent) { if (numChildDocs >= min && numChildDocs <= max) { diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.java index 92e288960065..e513c3437d41 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.java +++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.java @@ -163,14 +163,12 @@ private void runNumHits(int numHits) throws IOException { new TopScoreDocCollectorManager(numHits, Integer.MAX_VALUE); searcher.search(testQuery, largeCollector); - TopDocs topDocs = searcher.search(testQuery, regularCollector); - - assertEquals(largeCollector.totalHits, topDocs.totalHits.value); - TopDocs firstTopDocs = largeCollector.topDocs(); - assertEquals(firstTopDocs.scoreDocs.length, topDocs.scoreDocs.length); + TopDocs secondTopDocs = searcher.search(testQuery, regularCollector); - CheckHits.checkEqual(testQuery, firstTopDocs.scoreDocs, topDocs.scoreDocs); + assertEquals(largeCollector.totalHits, secondTopDocs.totalHits.value); + assertEquals(firstTopDocs.scoreDocs.length, secondTopDocs.scoreDocs.length); + CheckHits.checkEqual(testQuery, firstTopDocs.scoreDocs, secondTopDocs.scoreDocs); } } diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestHeatmapFacetCounter.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestHeatmapFacetCounter.java index 30294f7b23c2..b3080d6a4ec9 100644 --- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestHeatmapFacetCounter.java +++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/prefix/TestHeatmapFacetCounter.java @@ -24,7 +24,6 @@ import java.util.ArrayList; import java.util.List; import org.apache.lucene.search.Query; -import org.apache.lucene.search.TotalHitCountCollectorManager; import org.apache.lucene.spatial.StrategyTestCase; import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; @@ -282,7 +281,7 @@ private int countMatchingDocsAtLevel(Point pt, int facetLevel) throws IOExceptio Query filter = new IntersectsPrefixTreeQuery( pt, strategy.getFieldName(), grid, facetLevel, grid.getMaxLevels()); - int totalHits = indexSearcher.search(filter, new TotalHitCountCollectorManager()); + int totalHits = indexSearcher.count(filter); cellsValidated++; if (totalHits > 0) { cellValidatedNonZero++; From 06b0f5a7180f84358626ce3f162ad359864670b0 Mon Sep 17 00:00:00 2001 From: Zach Chen Date: Tue, 7 Nov 2023 19:43:43 -0800 Subject: [PATCH 10/14] Revert some changes --- .../lucene/document/BaseSpatialTestCase.java | 15 ++++++++++----- .../lucene/search/TestSearchWithThreads.java | 4 +++- .../lucene/demo/facet/DistanceFacetsExample.java | 1 + .../taxonomy/TestTaxonomyFacetAssociations.java | 5 ++++- .../tests/search/BaseRangeFieldQueryTestCase.java | 2 +- 5 files changed, 19 insertions(+), 8 deletions(-) diff --git a/lucene/core/src/test/org/apache/lucene/document/BaseSpatialTestCase.java b/lucene/core/src/test/org/apache/lucene/document/BaseSpatialTestCase.java index d744ec11789d..f6551a4628b2 100644 --- a/lucene/core/src/test/org/apache/lucene/document/BaseSpatialTestCase.java +++ b/lucene/core/src/test/org/apache/lucene/document/BaseSpatialTestCase.java @@ -35,6 +35,7 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SerialMergeScheduler; import org.apache.lucene.index.Term; +import org.apache.lucene.search.CollectorManager; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; @@ -321,7 +322,7 @@ protected void verifyRandomBBoxQueries(IndexReader reader, Object... shapes) thr System.out.println(" query=" + query + ", relation=" + queryRelation); } - final FixedBitSet hits = s.search(query, FixedBitSetCollector.createManager(maxDoc)); + final FixedBitSet hits = s.search(query, searchIndex(maxDoc)); boolean fail = false; NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id"); @@ -418,7 +419,7 @@ protected void verifyRandomLineQueries(IndexReader reader, Object... shapes) thr System.out.println(" query=" + query + ", relation=" + queryRelation); } - final FixedBitSet hits = s.search(query, FixedBitSetCollector.createManager(maxDoc)); + final FixedBitSet hits = s.search(query, searchIndex(maxDoc)); boolean fail = false; NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id"); @@ -491,7 +492,7 @@ protected void verifyRandomPolygonQueries(IndexReader reader, Object... shapes) System.out.println(" query=" + query + ", relation=" + queryRelation); } - final FixedBitSet hits = s.search(query, FixedBitSetCollector.createManager(maxDoc)); + final FixedBitSet hits = s.search(query, searchIndex(maxDoc)); boolean fail = false; NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id"); @@ -570,7 +571,7 @@ protected void verifyRandomPointQueries(IndexReader reader, Object... shapes) th System.out.println(" query=" + query + ", relation=" + queryRelation); } - final FixedBitSet hits = s.search(query, FixedBitSetCollector.createManager(maxDoc)); + final FixedBitSet hits = s.search(query, searchIndex(maxDoc)); boolean fail = false; NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id"); @@ -645,7 +646,7 @@ protected void verifyRandomDistanceQueries(IndexReader reader, Object... shapes) System.out.println(" query=" + query + ", relation=" + queryRelation); } - final FixedBitSet hits = s.search(query, FixedBitSetCollector.createManager(maxDoc)); + final FixedBitSet hits = s.search(query, searchIndex(maxDoc)); boolean fail = false; NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id"); @@ -694,6 +695,10 @@ protected void verifyRandomDistanceQueries(IndexReader reader, Object... shapes) } } + private CollectorManager searchIndex(int maxDoc) { + return FixedBitSetCollector.createManager(maxDoc); + } + protected abstract Validator getValidator(); protected abstract static class Encoder { diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSearchWithThreads.java b/lucene/core/src/test/org/apache/lucene/search/TestSearchWithThreads.java index e667bf9ce1c6..d4bd95fca7a1 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSearchWithThreads.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSearchWithThreads.java @@ -24,6 +24,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.tests.search.DummyTotalHitCountCollector; import org.apache.lucene.tests.util.LuceneTestCase; public class TestSearchWithThreads extends LuceneTestCase { @@ -54,14 +55,15 @@ public void test() throws Exception { w.close(); final IndexSearcher s = newSearcher(r); - final TotalHitCountCollectorManager collectorManager = new TotalHitCountCollectorManager(); final AtomicBoolean failed = new AtomicBoolean(); final AtomicLong netSearch = new AtomicLong(); + CollectorManager collectorManager = DummyTotalHitCountCollector.createManager(); Thread[] threads = new Thread[numThreads]; for (int threadID = 0; threadID < numThreads; threadID++) { threads[threadID] = new Thread() { + @Override public void run() { try { diff --git a/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java b/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java index 8be167363881..cd49c08dc044 100644 --- a/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java +++ b/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java @@ -214,6 +214,7 @@ public static Query getBoundingBoxQuery( /** User runs a query and counts facets. */ public FacetResult search() throws IOException { + FacetsCollector fc = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); Facets facets = diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetAssociations.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetAssociations.java index 3b6ee27ff4b2..bfc0af91afd9 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetAssociations.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetAssociations.java @@ -222,8 +222,11 @@ public void testIntSumAssociation() throws Exception { } public void testIntAssociationRandom() throws Exception { + + FacetsCollector fc = new FacetsCollector(); + IndexSearcher searcher = newSearcher(reader); - FacetsCollector fc = searcher.search(new MatchAllDocsQuery(), new FacetsCollectorManager()); + searcher.search(new MatchAllDocsQuery(), fc); Map expected; Facets facets; diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/search/BaseRangeFieldQueryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/search/BaseRangeFieldQueryTestCase.java index 405b5671a601..f908859ce9b1 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/search/BaseRangeFieldQueryTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/search/BaseRangeFieldQueryTestCase.java @@ -296,7 +296,7 @@ private void verify(Range[][] ranges) throws Exception { System.out.println(" query=" + query); } - FixedBitSet hits = s.search(query, FixedBitSetCollector.createManager(maxDoc)); + final FixedBitSet hits = s.search(query, FixedBitSetCollector.createManager(maxDoc)); NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id"); for (int docID = 0; docID < maxDoc; ++docID) { From 84641b440f37212553738329141212a69f7ac7f1 Mon Sep 17 00:00:00 2001 From: Zach Chen Date: Tue, 7 Nov 2023 23:53:33 -0800 Subject: [PATCH 11/14] Add back usage check --- .../apache/lucene/search/TopFieldCollectorManager.java | 10 ++++++++++ .../lucene/search/TopScoreDocCollectorManager.java | 10 ++++++++++ .../org/apache/lucene/search/TestTopDocsCollector.java | 6 ++++-- .../apache/lucene/search/TestTopFieldCollector.java | 6 ++++-- 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java index 3ae7651b81c3..249d66f165f0 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java @@ -35,6 +35,8 @@ public class TopFieldCollectorManager implements CollectorManager collectors; + private final boolean supportsConcurrency; + private boolean collectorCreated; /** * Creates a new {@link TopFieldCollectorManager} from the given arguments. @@ -86,6 +88,7 @@ public TopFieldCollectorManager( this.sort = sort; this.numHits = numHits; this.after = after; + this.supportsConcurrency = supportsConcurrency; this.hitsThresholdChecker = supportsConcurrency ? HitsThresholdChecker.createShared(Math.max(totalHitsThreshold, numHits)) @@ -135,6 +138,13 @@ public TopFieldCollectorManager(Sort sort, int numHits, int totalHitsThreshold) @Override public TopFieldCollector newCollector() { + if (collectorCreated && supportsConcurrency == false) { + throw new IllegalStateException( + "The instantiated TopFieldCollectorManager does not support concurrency, but multiple collectors are being created"); + } else { + collectorCreated = true; + } + FieldValueHitQueue queue = FieldValueHitQueue.create(sort.getSort(), numHits); diff --git a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java index 67089d70e5b6..642e2380cc12 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java @@ -31,6 +31,8 @@ public class TopScoreDocCollectorManager private final ScoreDoc after; private final HitsThresholdChecker hitsThresholdChecker; private final MaxScoreAccumulator minScoreAcc; + private final boolean supportsConcurrency; + private boolean collectorCreated; /** * Creates a new {@link TopScoreDocCollectorManager} given the number of hits to collect and the @@ -68,6 +70,7 @@ public TopScoreDocCollectorManager( this.numHits = numHits; this.after = after; + this.supportsConcurrency = supportsConcurrency; this.hitsThresholdChecker = supportsConcurrency ? HitsThresholdChecker.createShared(Math.max(totalHitsThreshold, numHits)) @@ -126,6 +129,13 @@ public TopScoreDocCollectorManager(int numHits, int totalHitsThreshold) { @Override public TopScoreDocCollector newCollector() { + if (collectorCreated && supportsConcurrency == false) { + throw new IllegalStateException( + "The instantiated TopScoreDocCollectorManager does not support concurrency, but multiple collectors are being created"); + } else { + collectorCreated = true; + } + if (after == null) { return new TopScoreDocCollector.SimpleTopScoreDocCollector( numHits, hitsThresholdChecker, minScoreAcc); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java index 6d8128fcf3d0..e28be104ac0c 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java @@ -139,7 +139,8 @@ private TopDocs doSearchWithThreshold( int numResults, int thresHold, Query q, IndexReader indexReader) throws IOException { IndexSearcher searcher = newSearcher(indexReader, true, true, false); TopScoreDocCollectorManager collectorManager = - new TopScoreDocCollectorManager(numResults, null, thresHold, true); + new TopScoreDocCollectorManager( + numResults, null, thresHold, searcher.getSlices().length > 1); return searcher.search(q, collectorManager); } @@ -147,7 +148,8 @@ private static TopDocs doConcurrentSearchWithThreshold( int numResults, int threshold, Query q, IndexReader indexReader) throws IOException { IndexSearcher searcher = newSearcher(indexReader, true, true, true); TopScoreDocCollectorManager collectorManager = - new TopScoreDocCollectorManager(numResults, threshold); + new TopScoreDocCollectorManager( + numResults, null, threshold, searcher.getSlices().length > 1); return searcher.search(q, collectorManager); } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java index 5186b92fd6db..960e63415c74 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java @@ -74,7 +74,8 @@ private static TopDocs doSearchWithThreshold( throws IOException { IndexSearcher searcher = newSearcher(indexReader); TopFieldCollectorManager manager = - new TopFieldCollectorManager(sort, numResults, null, thresHold, false); + new TopFieldCollectorManager( + sort, numResults, null, thresHold, searcher.getSlices().length > 1); return searcher.search(q, manager); } @@ -84,7 +85,8 @@ private static TopDocs doConcurrentSearchWithThreshold( IndexSearcher searcher = newSearcher(indexReader, true, true, true); TopFieldCollectorManager collectorManager = - new TopFieldCollectorManager(sort, numResults, threshold); + new TopFieldCollectorManager( + sort, numResults, null, threshold, searcher.getSlices().length > 1); TopDocs topDoc = searcher.search(q, collectorManager); From a9a682819fa681ace94c074a517ae79054cb6d86 Mon Sep 17 00:00:00 2001 From: Zach Chen Date: Wed, 8 Nov 2023 19:45:34 -0800 Subject: [PATCH 12/14] Address comment --- .../java/org/apache/lucene/search/TopDocs.java | 3 +-- .../apache/lucene/search/TopFieldCollector.java | 6 +++--- .../lucene/search/TopFieldCollectorManager.java | 2 +- .../lucene/search/TopScoreDocCollector.java | 6 +++--- .../search/TopScoreDocCollectorManager.java | 2 +- .../lucene/document/BaseSpatialTestCase.java | 16 ++++++++-------- 6 files changed, 17 insertions(+), 18 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/TopDocs.java b/lucene/core/src/java/org/apache/lucene/search/TopDocs.java index d5a34ce7fbf8..e2c4dce3133f 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopDocs.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopDocs.java @@ -232,8 +232,7 @@ public static TopDocs merge( /** * Returns a new TopFieldDocs, containing topN results across the provided TopFieldDocs, sorting * by the specified {@link Sort}. Each of the TopDocs must have been sorted by the same Sort, and - * sort field values must have been filled (ie, fillFields=true must be passed to the - * constructor of {@link TopFieldCollectorManager}). + * sort field values must have been filled. * * @see #merge(Sort, int, int, TopFieldDocs[]) * @lucene.experimental diff --git a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java index e3b8a63e4a19..fcb14e1feef4 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java @@ -404,7 +404,7 @@ protected void updateMinCompetitiveScore(Scorable scorer) throws IOException { * count of the result will be accurate. {@link Integer#MAX_VALUE} may be used to make the hit * count accurate, but this will also make query processing slower. * @return a {@link TopFieldCollector} instance which will sort the results by the sort criteria. - * @deprecated This method is being deprecated in favor of using the constructor of {@link + * @deprecated This method is deprecated in favor of the constructor of {@link * TopFieldCollectorManager} due to its support for concurrency in IndexSearcher */ @Deprecated @@ -432,7 +432,7 @@ public static TopFieldCollector create(Sort sort, int numHits, int totalHitsThre * field is indexed both with doc values and points. In this case, there is an assumption that * the same data is stored in these points and doc values. * @return a {@link TopFieldCollector} instance which will sort the results by the sort criteria. - * @deprecated This method is being deprecated in favor of using the constructor of {@link + * @deprecated This method is deprecated in favor of the constructor of {@link * TopFieldCollectorManager} due to its support for concurrency in IndexSearcher */ @Deprecated @@ -447,7 +447,7 @@ public static TopFieldCollector create( * shared {@link MaxScoreAccumulator} to propagate the minimum score accross segments if the * primary sort is by relevancy. * - * @deprecated This method is being deprecated in favor of using the constructor of {@link + * @deprecated This method is deprecated in favor of the constructor of {@link * TopFieldCollectorManager} due to its support for concurrency in IndexSearcher */ @Deprecated diff --git a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java index 249d66f165f0..d09a589be975 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollectorManager.java @@ -140,7 +140,7 @@ public TopFieldCollectorManager(Sort sort, int numHits, int totalHitsThreshold) public TopFieldCollector newCollector() { if (collectorCreated && supportsConcurrency == false) { throw new IllegalStateException( - "The instantiated TopFieldCollectorManager does not support concurrency, but multiple collectors are being created"); + "This TopFieldCollectorManager was created without concurrency (supportsConcurrency=false), but multiple collectors are being created"); } else { collectorCreated = true; } diff --git a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java index 2902ec21f092..304eff2f11a0 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java @@ -204,7 +204,7 @@ public void collect(int doc) throws IOException { *

NOTE: The instances returned by this method pre-allocate a full array of length * numHits, and fill the array with sentinel objects. * - * @deprecated This method is being deprecated in favor of using the constructor of {@link + * @deprecated This method is deprecated in favor of the constructor of {@link * TopScoreDocCollectorManager} due to its support for concurrency in IndexSearcher */ @Deprecated @@ -225,7 +225,7 @@ public static TopScoreDocCollector create(int numHits, int totalHitsThreshold) { *

NOTE: The instances returned by this method pre-allocate a full array of length * numHits, and fill the array with sentinel objects. * - * @deprecated This method is being deprecated in favor of using the constructor of {@link + * @deprecated This method is deprecated in favor of the constructor of {@link * TopScoreDocCollectorManager} due to its support for concurrency in IndexSearcher */ @Deprecated @@ -238,7 +238,7 @@ public static TopScoreDocCollector create(int numHits, ScoreDoc after, int total * Create a CollectorManager which uses a shared hit counter to maintain number of hits and a * shared {@link MaxScoreAccumulator} to propagate the minimum score accross segments * - * @deprecated This method is being deprecated in favor of using the constructor of {@link + * @deprecated This method is deprecated in favor of the constructor of {@link * TopScoreDocCollectorManager} due to its support for concurrency in IndexSearcher */ @Deprecated diff --git a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java index 642e2380cc12..4e3181abdf7c 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollectorManager.java @@ -131,7 +131,7 @@ public TopScoreDocCollectorManager(int numHits, int totalHitsThreshold) { public TopScoreDocCollector newCollector() { if (collectorCreated && supportsConcurrency == false) { throw new IllegalStateException( - "The instantiated TopScoreDocCollectorManager does not support concurrency, but multiple collectors are being created"); + "This TopScoreDocCollectorManager was created without concurrency (supportsConcurrency=false), but multiple collectors are being created"); } else { collectorCreated = true; } diff --git a/lucene/core/src/test/org/apache/lucene/document/BaseSpatialTestCase.java b/lucene/core/src/test/org/apache/lucene/document/BaseSpatialTestCase.java index f6551a4628b2..ae05a73843a3 100644 --- a/lucene/core/src/test/org/apache/lucene/document/BaseSpatialTestCase.java +++ b/lucene/core/src/test/org/apache/lucene/document/BaseSpatialTestCase.java @@ -20,6 +20,7 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween; import com.carrotsearch.randomizedtesting.generators.RandomPicks; +import java.io.IOException; import java.util.Arrays; import java.util.HashSet; import java.util.Set; @@ -35,7 +36,6 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SerialMergeScheduler; import org.apache.lucene.index.Term; -import org.apache.lucene.search.CollectorManager; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; @@ -322,7 +322,7 @@ protected void verifyRandomBBoxQueries(IndexReader reader, Object... shapes) thr System.out.println(" query=" + query + ", relation=" + queryRelation); } - final FixedBitSet hits = s.search(query, searchIndex(maxDoc)); + final FixedBitSet hits = searchIndex(s, query, maxDoc); boolean fail = false; NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id"); @@ -419,7 +419,7 @@ protected void verifyRandomLineQueries(IndexReader reader, Object... shapes) thr System.out.println(" query=" + query + ", relation=" + queryRelation); } - final FixedBitSet hits = s.search(query, searchIndex(maxDoc)); + final FixedBitSet hits = searchIndex(s, query, maxDoc); boolean fail = false; NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id"); @@ -492,7 +492,7 @@ protected void verifyRandomPolygonQueries(IndexReader reader, Object... shapes) System.out.println(" query=" + query + ", relation=" + queryRelation); } - final FixedBitSet hits = s.search(query, searchIndex(maxDoc)); + final FixedBitSet hits = searchIndex(s, query, maxDoc); boolean fail = false; NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id"); @@ -571,7 +571,7 @@ protected void verifyRandomPointQueries(IndexReader reader, Object... shapes) th System.out.println(" query=" + query + ", relation=" + queryRelation); } - final FixedBitSet hits = s.search(query, searchIndex(maxDoc)); + final FixedBitSet hits = searchIndex(s, query, maxDoc); boolean fail = false; NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id"); @@ -646,7 +646,7 @@ protected void verifyRandomDistanceQueries(IndexReader reader, Object... shapes) System.out.println(" query=" + query + ", relation=" + queryRelation); } - final FixedBitSet hits = s.search(query, searchIndex(maxDoc)); + final FixedBitSet hits = searchIndex(s, query, maxDoc); boolean fail = false; NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id"); @@ -695,8 +695,8 @@ protected void verifyRandomDistanceQueries(IndexReader reader, Object... shapes) } } - private CollectorManager searchIndex(int maxDoc) { - return FixedBitSetCollector.createManager(maxDoc); + private FixedBitSet searchIndex(IndexSearcher s, Query query, int maxDoc) throws IOException { + return s.search(query, FixedBitSetCollector.createManager(maxDoc)); } protected abstract Validator getValidator(); From 0b4f9cddcfe54e358af92cdacc4a01bafe08a8fb Mon Sep 17 00:00:00 2001 From: Zach Chen Date: Mon, 13 Nov 2023 17:30:06 -0800 Subject: [PATCH 13/14] Revert some changes --- .../lucene/benchmark/byTask/tasks/ReadTask.java | 15 ++++++++++----- .../byTask/tasks/SearchWithCollectorTask.java | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java index b943c0bc2ed4..58cf8e79efae 100644 --- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java +++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java @@ -24,6 +24,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiBits; import org.apache.lucene.index.StoredFields; +import org.apache.lucene.search.Collector; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; @@ -118,11 +119,10 @@ public int doLogic() throws Exception { hits = searcher.search(q, numHits); } } else { - int totalHitsThreshold = withTotalHits() ? Integer.MAX_VALUE : 1; - TopScoreDocCollectorManager collectorManager = - new TopScoreDocCollectorManager( - numHits(), null, totalHitsThreshold, searcher.getSlices().length > 1); - hits = searcher.search(q, collectorManager); + Collector collector = createCollector(); + + searcher.search(q, collector); + // hits = collector.topDocs(); } if (hits != null) { @@ -184,6 +184,11 @@ protected int withTopDocs(IndexSearcher searcher, Query q, TopDocs hits) throws return res; } + protected Collector createCollector() throws Exception { + return new TopScoreDocCollectorManager(numHits(), withTotalHits() ? Integer.MAX_VALUE : 1) + .newCollector(); + } + protected Document retrieveDoc(StoredFields storedFields, int id) throws IOException { return storedFields.document(id); } diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchWithCollectorTask.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchWithCollectorTask.java index 65a5098ab1d0..9f79e51d5057 100644 --- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchWithCollectorTask.java +++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchWithCollectorTask.java @@ -19,6 +19,8 @@ import org.apache.lucene.benchmark.byTask.PerfRunData; import org.apache.lucene.benchmark.byTask.feeds.QueryMaker; import org.apache.lucene.benchmark.byTask.utils.Config; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.TopScoreDocCollector; /** Does search w/ a custom collector */ public class SearchWithCollectorTask extends SearchTask { @@ -43,6 +45,20 @@ public boolean withCollector() { return true; } + @Override + protected Collector createCollector() throws Exception { + Collector collector = null; + if (clnName.equalsIgnoreCase("topScoreDoc") == true) { + collector = TopScoreDocCollector.create(numHits(), Integer.MAX_VALUE); + } else if (clnName.length() > 0) { + collector = Class.forName(clnName).asSubclass(Collector.class).getConstructor().newInstance(); + + } else { + collector = super.createCollector(); + } + return collector; + } + @Override public QueryMaker getQueryMaker() { return getRunData().getQueryMaker(this); From 8f2797f5e5ef7d1928fb401681c095b97753ae68 Mon Sep 17 00:00:00 2001 From: Zach Chen Date: Mon, 27 Nov 2023 12:01:06 -0800 Subject: [PATCH 14/14] Add change entry --- lucene/CHANGES.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 1fd049270541..e4c01252f54c 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -61,6 +61,10 @@ API Changes * GITHUB#12599: Add RandomAccessInput#readBytes method to the RandomAccessInput interface. (Ignacio Vera) +* GITHUB#11041: Deprecate IndexSearch#search(Query, Collector) in favor of + IndexSearcher#search(Query, CollectorManager) for TopFieldCollectorManager + and TopScoreDocCollectorManager. (Zach Chen, Adrien Grand, Michael McCandless, Greg Miller, Luca Cavanna) + New Features --------------------- @@ -172,6 +176,8 @@ API Changes * GITHUB#12799: Make TaskExecutor constructor public and use TaskExecutor for concurrent HNSW graph build. (Shubham Chaudhary) +* + New Features ---------------------