Skip to content

bugfix (Solr 9.x port): Performance degradation for collapse queries with string sort clauses due to enforced LZ4 decompression #4618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
bartoszfidrysiak wants to merge 5 commits into from
+474 −3
Closed

bugfix (Solr 9.x port): Performance degradation for collapse queries with string sort clauses due to enforced LZ4 decompression #4618

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
813c499
bugfix: lazy string loading while initializing new collapse group & o…
bartoszfidrysiak Jul 6, 2026
8a6d2f4
Added CollapsingSearch benchmark
bartoszfidrysiak Jul 6, 2026
42e018f
Fixed issues in CollapsingSearch: forbidden APIs & code reformatting
bartoszfidrysiak Jul 6, 2026
a36a2a3
fix: deep copy BytesRef from lookupOrd() in LazyStringValue.materiali…
bartoszfidrysiak Jul 6, 2026
e1491d3
Added changelog: PR#4618-perf-fix-for-collapse-sort-by-str.yml
bartoszfidrysiak Jul 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions changelog/unreleased/PR#4618-perf-fix-for-collapse-sort-by-str.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
title: Fix for performance degradation for collapse queries with string sort clauses due to enforced LZ4 decompression
type: fixed # added, changed, fixed, deprecated, removed, dependency_update, security, other
authors:
- name: Bartosz Fidrysiak
links:
- name: PR#4618
url: https://github.com/apache/solr/pull/4618
229 changes: 229 additions & 0 deletions solr/benchmark/src/java/org/apache/solr/bench/search/CollapsingSearch.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,229 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.bench.search;

import java.io.IOException;
import java.time.Instant;
import java.time.format.DateTimeFormatter;
import java.util.Locale;
import java.util.Random;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.apache.solr.bench.BaseBenchState;
import org.apache.solr.bench.MiniClusterState;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.CommonParams;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.annotations.Threads;
import org.openjdk.jmh.annotations.Warmup;

@Fork(value = 1)
@Warmup(time = 5, iterations = 5)
@Measurement(time = 15, iterations = 5)
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
@Threads(value = 1)
public class CollapsingSearch {

static final String COLLECTION = "collapse_bench";

@State(Scope.Benchmark)
public static class BenchState {

@Param({"2000000"})
int numDocs;

@Param({"100000"})
int numGroups;

@Param({"1", "10"})
int numSegments;

final QueryRequest qSimple =
new QueryRequest(new SolrQuery("q", "*:*", "rows", "10", "cache", "false"));

final QueryRequest qCollapseWithoutSort =
new QueryRequest(
new SolrQuery(
"q", "*:*",
"fq", "{!collapse field=group_s_dv cache=false}",
"rows", "10",
"cache", "false"));

final QueryRequest qCollapseByStr =
new QueryRequest(
new SolrQuery(
"q", "*:*",
"fq", "{!collapse field=group_s_dv sort='str_s_dv asc' cache=false}",
"rows", "10",
"cache", "false"));

final QueryRequest qCollapseByDate =
new QueryRequest(
new SolrQuery(
"q", "*:*",
"fq", "{!collapse field=group_s_dv sort='date_dt_dv asc' cache=false}",
"rows", "10",
"cache", "false"));

final QueryRequest qCollapseByLong =
new QueryRequest(
new SolrQuery(
"q", "*:*",
"fq", "{!collapse field=group_s_dv sort='long_l_dv asc' cache=false}",
"rows", "10",
"cache", "false"));

final QueryRequest qCollapseByDateAndStr =
new QueryRequest(
new SolrQuery(
"q", "*:*",
"fq",
"{!collapse field=group_s_dv sort='date_dt_dv asc, str_s_dv asc' cache=false}",
"rows", "10",
"cache", "false"));

@Setup(Level.Trial)
public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState)
throws Exception {
System.setProperty("commitwithin.softcommit", "false");
miniClusterState.startMiniCluster(1);
miniClusterState.createCollection(COLLECTION, 1, 1);

indexDocs(miniClusterState);
miniClusterState.forceMerge(COLLECTION, numSegments);

int actualSegments = getActualSegmentCount(miniClusterState);
BaseBenchState.log(
"CollapsingSearch ready: numDocs="
+ numDocs
+ " numGroups="
+ numGroups
+ " numSegments(requested)="
+ numSegments
+ " numSegments(actual)="
+ actualSegments);
}

@Setup(Level.Iteration)
public void setupIteration(MiniClusterState.MiniClusterBenchState miniClusterState)
throws SolrServerException, IOException {
CollectionAdminRequest.Reload reload = CollectionAdminRequest.reloadCollection(COLLECTION);
miniClusterState.client.request(reload);
}

@TearDown(Level.Trial)
public void teardown(MiniClusterState.MiniClusterBenchState miniClusterState) throws Exception {
CollectionAdminRequest.deleteCollection(COLLECTION).process(miniClusterState.client);
}

private int pickGroup(int docId) {
return docId % numGroups;
}

private int getActualSegmentCount(MiniClusterState.MiniClusterBenchState state)
throws SolrServerException, IOException {
SolrQuery lukeQuery = new SolrQuery();
lukeQuery.set(CommonParams.QT, "/admin/luke");
lukeQuery.set("show", "index");
var resp = state.client.query(COLLECTION, lukeQuery);
Object segCount = resp.getResponse().findRecursive("index", "segmentCount");
return segCount instanceof Number ? ((Number) segCount).intValue() : -1;
}

private void indexDocs(MiniClusterState.MiniClusterBenchState state) throws Exception {
int docsPerSegment = numDocs / numSegments;
String[] groupIds = new String[numGroups];
for (int i = 0; i < numGroups; i++) {
groupIds[i] = String.format(Locale.ROOT, "group_%06d", i);
}
Random rng = new Random(42);
Instant baseDate = Instant.parse("2020-01-01T00:00:00Z");

int docId = 0;
for (int seg = 0; seg < numSegments; seg++) {
int segDocs = (seg < numSegments - 1) ? docsPerSegment : (numDocs - docId);
UpdateRequest req = new UpdateRequest();
for (int i = 0; i < segDocs; i++) {
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", String.valueOf(docId));
doc.addField("group_s_dv", groupIds[pickGroup(docId)]);
docId++;
doc.addField("str_s_dv", UUID.randomUUID().toString());
doc.addField(
"date_dt_dv",
DateTimeFormatter.ISO_INSTANT.format(baseDate.plusSeconds(rng.nextInt(10_000_000))));
doc.addField("long_l_dv", rng.nextInt(10_000_000));
req.add(doc);
}
req.commit(state.client, COLLECTION);
}
}
}

@Benchmark
public Object simple(BenchState b, MiniClusterState.MiniClusterBenchState cluster)
throws SolrServerException, IOException {
return cluster.client.request(b.qSimple, COLLECTION);
}

@Benchmark
public Object collapseWithoutSort(BenchState b, MiniClusterState.MiniClusterBenchState cluster)
throws SolrServerException, IOException {
return cluster.client.request(b.qCollapseWithoutSort, COLLECTION);
}

@Benchmark
public Object collapseByStr(BenchState b, MiniClusterState.MiniClusterBenchState cluster)
throws SolrServerException, IOException {
return cluster.client.request(b.qCollapseByStr, COLLECTION);
}

@Benchmark
public Object collapseByDate(BenchState b, MiniClusterState.MiniClusterBenchState cluster)
throws SolrServerException, IOException {
return cluster.client.request(b.qCollapseByDate, COLLECTION);
}

@Benchmark
public Object collapseByLong(BenchState b, MiniClusterState.MiniClusterBenchState cluster)
throws SolrServerException, IOException {
return cluster.client.request(b.qCollapseByLong, COLLECTION);
}

@Benchmark
public Object collapseByDateAndStr(BenchState b, MiniClusterState.MiniClusterBenchState cluster)
throws SolrServerException, IOException {
return cluster.client.request(b.qCollapseByDateAndStr, COLLECTION);
}
}
1 change: 1 addition & 0 deletions solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
<dynamicField name="*_d_dv" type="double" indexed="true" docValues="true" stored="false"/>
<dynamicField name="*_dt" type="date" indexed="true" stored="false"/>
<dynamicField name="*_dt_dv" type="date" indexed="true" docValues="true" stored="false"/>
<dynamicField name="*_s_dv" type="string" indexed="true" docValues="true" stored="false"/>

<uniqueKey>id</uniqueKey>
</schema>
52 changes: 49 additions & 3 deletions solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3570,6 +3570,9 @@ private static class SortFieldsCompare {
private Object[][] groupHeadValues; // growable
private final Object[] nullGroupValues;

private final SortedDocValues[] stringSortDVs;
private final int[] stringMissingOrd;

/**
* Constructs an instance based on the (raw, un-rewritten) SortFields to be used, and an initial
* number of expected groups (will grow as needed).
Expand All @@ -3581,6 +3584,7 @@ public SortFieldsCompare(SortField[] sorts, int initNumGroups) {
fieldComparators = new FieldComparator[numClauses];
leafFieldComparators = new LeafFieldComparator[numClauses];
reverseMul = new int[numClauses];
stringMissingOrd = new int[numClauses];
for (int clause = 0; clause < numClauses; clause++) {
SortField sf = sorts[clause];
// we only need one slot for every comparator
Expand All @@ -3592,14 +3596,28 @@ public SortFieldsCompare(SortField[] sorts, int initNumGroups) {
: Pruning.NONE);

reverseMul[clause] = sf.getReverse() ? -1 : 1;
if (sf.getType() == SortField.Type.STRING) {
stringMissingOrd[clause] =
(sf.getMissingValue() == SortField.STRING_LAST) ? Integer.MAX_VALUE : -1;
}
}
groupHeadValues = new Object[initNumGroups][];
nullGroupValues = new Object[numClauses];
stringSortDVs = new SortedDocValues[numClauses]; // populated in setNextReader
}

public void setNextReader(LeafReaderContext context) throws IOException {
for (int clause = 0; clause < numClauses; clause++) {
leafFieldComparators[clause] = fieldComparators[clause].getLeafComparator(context);
if (sorts[clause].getType() == SortField.Type.STRING) {
String field = sorts[clause].getField();
FieldInfo fi = context.reader().getFieldInfos().fieldInfo(field);
if (fi != null && fi.getDocValuesType() == DocValuesType.SORTED) {
stringSortDVs[clause] = DocValues.getSorted(context.reader(), field);
} else {
stringSortDVs[clause] = null;
}
}
}
}

Expand Down Expand Up @@ -3657,12 +3675,20 @@ public void setNullGroupValues(int contextDoc) throws IOException {

/**
* Records the SortField values for the specified contextDoc into the values array provided by
* the caller.
* the caller. STRING clauses with SORTED DocValues are stored as {@link LazyStringValue} to
* defer {@code lookupOrd()} until a second document actually competes for the group.
*/
private void setGroupValues(Object[] values, int contextDoc) throws IOException {
for (int clause = 0; clause < numClauses; clause++) {
leafFieldComparators[clause].copy(0, contextDoc);
values[clause] = cloneIfBytesRef(fieldComparators[clause].value(0));
if (stringSortDVs[clause] != null) {
SortedDocValues dv = stringSortDVs[clause];
int missingOrd = stringMissingOrd[clause];
int ord = dv.advanceExact(contextDoc) ? dv.ordValue() : missingOrd;
values[clause] = new LazyStringValue(dv, ord, missingOrd);
} else {
leafFieldComparators[clause].copy(0, contextDoc);
values[clause] = cloneIfBytesRef(fieldComparators[clause].value(0));
}
}
}

Expand Down Expand Up @@ -3701,6 +3727,19 @@ private boolean testAndSetGroupValues(Object[] values, int contextDoc) throws IO
int testClause = 0;
for (
/* testClause */ ; testClause < numClauses; testClause++) {
if (values[testClause] instanceof LazyStringValue) {
LazyStringValue headVal = (LazyStringValue) values[testClause];
SortedDocValues segDV = stringSortDVs[testClause];
if (segDV != null && segDV == headVal.dv) {
int missingOrd = headVal.missingOrd;
int candidateOrd = segDV.advanceExact(contextDoc) ? segDV.ordValue() : missingOrd;
lastCompare = reverseMul[testClause] * Integer.compare(candidateOrd, headVal.ord);
stash[testClause] = new LazyStringValue(segDV, candidateOrd, missingOrd);
if (0 != lastCompare) break;
continue;
}
values[testClause] = headVal.materialize();
}
leafFieldComparators[testClause].copy(0, contextDoc);
FieldComparator fcomp = fieldComparators[testClause];
stash[testClause] = cloneIfBytesRef(fcomp.value(0));
Expand All @@ -3724,6 +3763,13 @@ private boolean testAndSetGroupValues(Object[] values, int contextDoc) throws IO
System.arraycopy(stash, 0, values, 0, testClause);
// read the remaining values we didn't need to test
for (int copyClause = testClause; copyClause < numClauses; copyClause++) {
SortedDocValues segDV = stringSortDVs[copyClause];
if (segDV != null) {
int missingOrd = stringMissingOrd[copyClause];
int candidateOrd = segDV.advanceExact(contextDoc) ? segDV.ordValue() : missingOrd;
values[copyClause] = new LazyStringValue(segDV, candidateOrd, missingOrd);
continue;
}
leafFieldComparators[copyClause].copy(0, contextDoc);
values[copyClause] = cloneIfBytesRef(fieldComparators[copyClause].value(0));
}
Expand Down
41 changes: 41 additions & 0 deletions solr/core/src/java/org/apache/solr/search/LazyStringValue.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search;

import java.io.IOException;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.util.BytesRef;

final class LazyStringValue {

final SortedDocValues dv;
final int ord;
final int missingOrd;

LazyStringValue(SortedDocValues dv, int ord, int missingOrd) {
this.dv = dv;
this.ord = ord;
this.missingOrd = missingOrd;
}

BytesRef materialize() throws IOException {
if (ord == missingOrd || ord < 0) {
return null;
}
return BytesRef.deepCopyOf(dv.lookupOrd(ord));
}
}
Loading