-
Notifications
You must be signed in to change notification settings - Fork 24.4k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Use ProfileCollectorManager in DfsPhase #96689
Changes from 25 commits
9a410bc
4b94099
14797fd
c34a877
82f8242
7c66bdc
55a172c
c77664c
cf9d380
98c25a9
b988bf7
60cc3aa
c5e5be4
dd1b242
b366795
20537e5
22aae35
971fb58
200e6e2
8fa91bf
39ea2ee
748dbb7
33187c2
5ee6598
e141a97
0cdd332
0f6958c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
pr: 96689 | ||
summary: Use a collector manager in DfsPhase Knn Search | ||
area: Search | ||
type: enhancement | ||
issues: [] |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -13,6 +13,7 @@ | |
|
||
import java.io.IOException; | ||
import java.util.Collection; | ||
import java.util.Collections; | ||
import java.util.List; | ||
import java.util.stream.Collectors; | ||
|
||
|
@@ -21,15 +22,15 @@ | |
* in an {@link InternalProfileCollector}. It delegates all the profiling to the generated collectors via {@link #getCollectorTree()} | ||
* and joins them up when its {@link #reduce} method is called. The profile result can | ||
*/ | ||
public final class ProfileCollectorManager implements CollectorManager<InternalProfileCollector, Void> { | ||
public final class ProfileCollectorManager<T> implements CollectorManager<InternalProfileCollector, T> { | ||
|
||
private final CollectorManager<Collector, ?> collectorManager; | ||
private final CollectorManager<Collector, T> collectorManager; | ||
private final String reason; | ||
private CollectorResult collectorTree; | ||
|
||
@SuppressWarnings("unchecked") | ||
public ProfileCollectorManager(CollectorManager<? extends Collector, ?> collectorManager, String reason) { | ||
this.collectorManager = (CollectorManager<Collector, ?>) collectorManager; | ||
public ProfileCollectorManager(CollectorManager<? extends Collector, T> collectorManager, String reason) { | ||
this.collectorManager = (CollectorManager<Collector, T>) collectorManager; | ||
this.reason = reason; | ||
} | ||
|
||
|
@@ -38,22 +39,31 @@ public InternalProfileCollector newCollector() throws IOException { | |
return new InternalProfileCollector(collectorManager.newCollector(), reason); | ||
} | ||
|
||
public Void reduce(Collection<InternalProfileCollector> profileCollectors) throws IOException { | ||
public T reduce(Collection<InternalProfileCollector> profileCollectors) throws IOException { | ||
List<Collector> unwrapped = profileCollectors.stream() | ||
.map(InternalProfileCollector::getWrappedCollector) | ||
.collect(Collectors.toList()); | ||
collectorManager.reduce(unwrapped); | ||
T returnValue = collectorManager.reduce(unwrapped); | ||
|
||
List<CollectorResult> resultsPerProfiler = profileCollectors.stream() | ||
.map(ipc -> ipc.getCollectorTree()) | ||
.collect(Collectors.toList()); | ||
this.collectorTree = new CollectorResult(this.getClass().getSimpleName(), "segment_search", 0, resultsPerProfiler); | ||
return null; | ||
|
||
long totalTime = resultsPerProfiler.stream().map(CollectorResult::getTime).reduce(0L, Long::sum); | ||
String collectorName; | ||
if (resultsPerProfiler.size() == 0) { | ||
// in case no new collector was ever requested, create a new one just to get the name. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think we could assert that this never happens. The contract is that if reduce is called, newCollector was called at least once? |
||
collectorName = newCollector().getName(); | ||
} else { | ||
collectorName = resultsPerProfiler.get(0).getName(); | ||
} | ||
this.collectorTree = new CollectorResult(collectorName, reason, totalTime, Collections.emptyList()); | ||
return returnValue; | ||
} | ||
|
||
public CollectorResult getCollectorTree() { | ||
if (this.collectorTree == null) { | ||
throw new IllegalStateException("A collectorTree hasn't been set yet, call reduce() before attempting to retrieve it"); | ||
throw new IllegalStateException("A collectorTree hasn't been set yet. Call reduce() before attempting to retrieve it"); | ||
} | ||
return this.collectorTree; | ||
} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,117 @@ | ||
/* | ||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
* or more contributor license agreements. Licensed under the Elastic License | ||
* 2.0 and the Server Side Public License, v 1; you may not use this file except | ||
* in compliance with, at your election, the Elastic License 2.0 or the Server | ||
* Side Public License, v 1. | ||
*/ | ||
|
||
package org.elasticsearch.search.dfs; | ||
|
||
import org.apache.lucene.document.Document; | ||
import org.apache.lucene.document.KnnFloatVectorField; | ||
import org.apache.lucene.index.IndexReader; | ||
import org.apache.lucene.index.LeafReaderContext; | ||
import org.apache.lucene.search.IndexSearcher; | ||
import org.apache.lucene.search.KnnFloatVectorQuery; | ||
import org.apache.lucene.search.Query; | ||
import org.apache.lucene.store.Directory; | ||
import org.apache.lucene.tests.index.RandomIndexWriter; | ||
import org.elasticsearch.common.util.concurrent.EsExecutors; | ||
import org.elasticsearch.search.internal.ContextIndexSearcher; | ||
import org.elasticsearch.search.profile.Profilers; | ||
import org.elasticsearch.search.profile.SearchProfileDfsPhaseResult; | ||
import org.elasticsearch.search.profile.query.CollectorResult; | ||
import org.elasticsearch.search.profile.query.QueryProfileShardResult; | ||
import org.elasticsearch.test.ESTestCase; | ||
import org.elasticsearch.threadpool.TestThreadPool; | ||
import org.junit.After; | ||
import org.junit.Before; | ||
|
||
import java.io.IOException; | ||
import java.util.List; | ||
import java.util.concurrent.ThreadPoolExecutor; | ||
|
||
public class DfsPhaseTests extends ESTestCase { | ||
|
||
ThreadPoolExecutor threadPoolExecutor; | ||
private TestThreadPool threadPool; | ||
|
||
@Before | ||
public final void init() { | ||
int numThreads = randomIntBetween(2, 4); | ||
threadPool = new TestThreadPool(DfsPhaseTests.class.getName()); | ||
threadPoolExecutor = EsExecutors.newFixed( | ||
"test", | ||
numThreads, | ||
10, | ||
EsExecutors.daemonThreadFactory("test"), | ||
threadPool.getThreadContext(), | ||
randomBoolean() | ||
); | ||
} | ||
|
||
@After | ||
public void cleanup() { | ||
threadPoolExecutor.shutdown(); | ||
terminate(threadPool); | ||
} | ||
|
||
public void testSingleKnnSearch() throws IOException { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for adding this test! Do we need to test this specific method or could we instead test |
||
try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig())) { | ||
int numDocs = randomIntBetween(900, 1000); | ||
for (int i = 0; i < numDocs; i++) { | ||
Document d = new Document(); | ||
d.add(new KnnFloatVectorField("float_vector", new float[] { i, 0, 0 })); | ||
w.addDocument(d); | ||
} | ||
w.flush(); | ||
|
||
IndexReader reader = w.getReader(); | ||
ContextIndexSearcher searcher = new ContextIndexSearcher( | ||
reader, | ||
IndexSearcher.getDefaultSimilarity(), | ||
IndexSearcher.getDefaultQueryCache(), | ||
IndexSearcher.getDefaultQueryCachingPolicy(), | ||
randomBoolean(), | ||
this.threadPoolExecutor | ||
) { | ||
@Override | ||
protected LeafSlice[] slices(List<LeafReaderContext> leaves) { | ||
// get a thread per segment | ||
return slices(leaves, 1, 1); | ||
} | ||
}; | ||
|
||
Query query = new KnnFloatVectorQuery("float_vector", new float[] { 0, 0, 0 }, numDocs, null); | ||
|
||
int k = 10; | ||
// run without profiling enabled | ||
DfsKnnResults dfsKnnResults = DfsPhase.singleKnnSearch(query, k, null, searcher); | ||
assertEquals(k, dfsKnnResults.scoreDocs().length); | ||
|
||
// run with profiling enabled | ||
Profilers profilers = new Profilers(searcher); | ||
dfsKnnResults = DfsPhase.singleKnnSearch(query, k, profilers, searcher); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. should we have two tests, one with profiling and one without? |
||
assertEquals(k, dfsKnnResults.scoreDocs().length); | ||
SearchProfileDfsPhaseResult searchProfileDfsPhaseResult = profilers.getDfsProfiler().buildDfsPhaseResults(); | ||
List<QueryProfileShardResult> queryProfileShardResult = searchProfileDfsPhaseResult.getQueryProfileShardResult(); | ||
assertNotNull(queryProfileShardResult); | ||
CollectorResult collectorResult = queryProfileShardResult.get(0).getCollectorResult(); | ||
assertEquals("SimpleTopScoreDocCollector", (collectorResult.getName())); | ||
assertEquals("search_top_hits", (collectorResult.getReason())); | ||
assertTrue(collectorResult.getTime() > 0); | ||
List<CollectorResult> children = collectorResult.getCollectorResults(); | ||
if (children.size() > 0) { | ||
long totalTime = 0L; | ||
for (CollectorResult child : children) { | ||
assertEquals("SimpleTopScoreDocCollector", (child.getName())); | ||
assertEquals("search_top_hits", (child.getReason())); | ||
totalTime += child.getTime(); | ||
} | ||
assertEquals(totalTime, collectorResult.getTime()); | ||
} | ||
reader.close(); | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -12,8 +12,6 @@ | |
import org.apache.lucene.document.Field; | ||
import org.apache.lucene.document.StringField; | ||
import org.apache.lucene.index.IndexReader; | ||
import org.apache.lucene.sandbox.search.ProfilerCollectorResult; | ||
import org.apache.lucene.search.Collector; | ||
import org.apache.lucene.search.CollectorManager; | ||
import org.apache.lucene.search.IndexSearcher; | ||
import org.apache.lucene.search.MatchAllDocsQuery; | ||
|
@@ -29,7 +27,6 @@ | |
import java.io.IOException; | ||
import java.util.Collection; | ||
import java.util.Collections; | ||
import java.util.List; | ||
|
||
public class ProfileCollectorManagerTests extends ESTestCase { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. would it make sense to have a test that simulates profiling a search with and without concurrency? if you call There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Good idea, what should be the goal of that test? Would we want to e.g. prep the searcher with data and then see if all results are there (in this case the top docs?) The profiling times wouldn't make very much sense to check except that they are there. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I would test the output of profiling, timing per se does not make sense to check indeed. |
||
|
||
|
@@ -48,26 +45,28 @@ private static class TestCollector extends DummyTotalHitCountCollector { | |
*/ | ||
public void testBasic() throws IOException { | ||
final SetOnce<Boolean> reduceCalled = new SetOnce<>(); | ||
ProfileCollectorManager pcm = new ProfileCollectorManager(new CollectorManager<>() { | ||
ProfileCollectorManager<Integer> pcm = new ProfileCollectorManager<>(new CollectorManager<TestCollector, Integer>() { | ||
|
||
private static int counter = 0; | ||
private int counter = 0; | ||
|
||
@Override | ||
public Collector newCollector() { | ||
public TestCollector newCollector() { | ||
return new TestCollector(counter++); | ||
} | ||
|
||
@Override | ||
public Void reduce(Collection<Collector> collectors) { | ||
public Integer reduce(Collection<TestCollector> collectors) { | ||
reduceCalled.set(true); | ||
return null; | ||
return counter; | ||
} | ||
}, CollectorResult.REASON_SEARCH_TOP_HITS); | ||
for (int i = 0; i < randomIntBetween(5, 10); i++) { | ||
int runs = randomIntBetween(5, 10); | ||
for (int i = 0; i < runs; i++) { | ||
InternalProfileCollector internalProfileCollector = pcm.newCollector(); | ||
assertEquals(i, ((TestCollector) internalProfileCollector.getWrappedCollector()).id); | ||
} | ||
pcm.reduce(Collections.emptyList()); | ||
Integer returnValue = pcm.reduce(Collections.emptyList()); | ||
assertEquals(runs, returnValue.intValue()); | ||
assertTrue(reduceCalled.get()); | ||
} | ||
|
||
|
@@ -96,21 +95,15 @@ public void testManagerWithSearcher() throws IOException { | |
assertEquals(numDocs, topDocs.totalHits.value); | ||
|
||
String profileReason = "profiler_reason"; | ||
ProfileCollectorManager profileCollectorManager = new ProfileCollectorManager(topDocsManager, profileReason); | ||
ProfileCollectorManager<TopDocs> profileCollectorManager = new ProfileCollectorManager<>(topDocsManager, profileReason); | ||
|
||
searcher.search(new MatchAllDocsQuery(), profileCollectorManager); | ||
|
||
CollectorResult parent = profileCollectorManager.getCollectorTree(); | ||
assertEquals("ProfileCollectorManager", parent.getName()); | ||
assertEquals("segment_search", parent.getReason()); | ||
assertEquals(0, parent.getTime()); | ||
List<ProfilerCollectorResult> delegateCollectorResults = parent.getProfiledChildren(); | ||
assertEquals(numSlices, delegateCollectorResults.size()); | ||
for (ProfilerCollectorResult pcr : delegateCollectorResults) { | ||
assertEquals("SimpleTopScoreDocCollector", pcr.getName()); | ||
assertEquals(profileReason, pcr.getReason()); | ||
assertTrue(pcr.getTime() > 0); | ||
} | ||
CollectorResult result = profileCollectorManager.getCollectorTree(); | ||
assertEquals("profiler_reason", result.getReason()); | ||
assertEquals("SimpleTopScoreDocCollector", result.getName()); | ||
assertTrue(result.getTime() > 0); | ||
|
||
reader.close(); | ||
} | ||
directory.close(); | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This looks better than before. I think that you may need to update existing usages of the profile collector manager to address warnings around their unchecked usage now that it's typed?