Skip to content

[9.0] Fix NPE in flat_bbq scorer when all vectors are missing (#129548) #129623

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 18, 2025
Merged

[9.0] Fix NPE in flat_bbq scorer when all vectors are missing (#129548) #129623

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
021ae41
Fix NPE in flat_bbq scorer when all vectors are missing (#129548)
benwtrent Jun 17, 2025
bb4b884
fixing comp
benwtrent Jun 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/changelog/129548.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 129548
summary: Fix NPE in `flat_bbq` scorer when all vectors are missing
area: Vector Search
type: bug
issues: []
3 changes: 3 additions & 0 deletions ...c/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryFlatVectorsScorer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ public RandomVectorScorer getRandomVectorScorer(
float[] target
) throws IOException {
if (vectorValues instanceof BinarizedByteVectorValues binarizedVectors) {
assert binarizedVectors.getQuantizer() != null
: "BinarizedByteVectorValues must have a quantizer for ES816BinaryFlatVectorsScorer";
assert binarizedVectors.size() > 0 : "BinarizedByteVectorValues must have at least one vector for ES816BinaryFlatVectorsScorer";
BinaryQuantizer quantizer = binarizedVectors.getQuantizer();
float[] centroid = binarizedVectors.getCentroid();
// FIXME: precompute this once?
Expand Down
2 changes: 1 addition & 1 deletion ...n/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ static void validateFieldEntry(FieldInfo info, FieldEntry fieldEntry) {
@Override
public RandomVectorScorer getRandomVectorScorer(String field, float[] target) throws IOException {
FieldEntry fi = fields.get(field);
if (fi == null) {
if (fi == null || fi.size() == 0) {
return null;
}
return vectorScorer.getRandomVectorScorer(
Expand Down
3 changes: 3 additions & 0 deletions ...c/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryFlatVectorsScorer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,9 @@ public RandomVectorScorer getRandomVectorScorer(
float[] target
) throws IOException {
if (vectorValues instanceof BinarizedByteVectorValues binarizedVectors) {
assert binarizedVectors.getQuantizer() != null
: "BinarizedByteVectorValues must have a quantizer for ES816BinaryFlatVectorsScorer";
assert binarizedVectors.size() > 0 : "BinarizedByteVectorValues must have at least one vector for ES816BinaryFlatVectorsScorer";
OptimizedScalarQuantizer quantizer = binarizedVectors.getQuantizer();
float[] centroid = binarizedVectors.getCentroid();
// We make a copy as the quantization process mutates the input
Expand Down
2 changes: 1 addition & 1 deletion ...n/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ static void validateFieldEntry(FieldInfo info, FieldEntry fieldEntry) {
@Override
public RandomVectorScorer getRandomVectorScorer(String field, float[] target) throws IOException {
FieldEntry fi = fields.get(field);
if (fi == null) {
if (fi == null || fi.size() == 0) {
return null;
}
return vectorScorer.getRandomVectorScorer(
Expand Down
65 changes: 65 additions & 0 deletions ...a/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsFormatTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.lucene101.Lucene101Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FloatVectorValues;
Expand All @@ -32,18 +33,30 @@
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.KnnVectorValues;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.SoftDeletesRetentionMergePolicy;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.search.FieldExistsQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.KnnFloatVectorQuery;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.search.join.BitSetProducer;
import org.apache.lucene.search.join.CheckJoinIndex;
import org.apache.lucene.search.join.DiversifyingChildrenFloatKnnVectorQuery;
import org.apache.lucene.search.join.QueryBitSetProducer;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
import org.elasticsearch.common.logging.LogConfigurator;
import org.elasticsearch.index.codec.vectors.BQVectorUtils;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;

import static java.lang.String.format;
Expand All @@ -68,6 +81,58 @@ public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
};
}

static String encodeInts(int[] i) {
return Arrays.toString(i);
}

static BitSetProducer parentFilter(IndexReader r) throws IOException {
// Create a filter that defines "parent" documents in the index
BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "_parent")));
CheckJoinIndex.check(r, parentsFilter);
return parentsFilter;
}

Document makeParent(int[] children) {
Document parent = new Document();
parent.add(newStringField("docType", "_parent", Field.Store.NO));
parent.add(newStringField("id", encodeInts(children), Field.Store.YES));
return parent;
}

public void testEmptyDiversifiedChildSearch() throws Exception {
String fieldName = "field";
int dims = random().nextInt(4, 65);
float[] vector = randomVector(dims);
VectorSimilarityFunction similarityFunction = VectorSimilarityFunction.EUCLIDEAN;
try (Directory d = newDirectory()) {
IndexWriterConfig iwc = newIndexWriterConfig().setCodec(getCodec());
iwc.setMergePolicy(new SoftDeletesRetentionMergePolicy("soft_delete", MatchAllDocsQuery::new, iwc.getMergePolicy()));
try (IndexWriter w = new IndexWriter(d, iwc)) {
List<Document> toAdd = new ArrayList<>();
for (int j = 1; j <= 5; j++) {
Document doc = new Document();
doc.add(new KnnFloatVectorField(fieldName, vector, similarityFunction));
doc.add(newStringField("id", Integer.toString(j), Field.Store.YES));
toAdd.add(doc);
}
toAdd.add(makeParent(new int[] { 1, 2, 3, 4, 5 }));
w.addDocuments(toAdd);
w.addDocuments(List.of(makeParent(new int[] { 6, 7, 8, 9, 10 })));
w.deleteDocuments(new FieldExistsQuery(fieldName), new TermQuery(new Term("id", encodeInts(new int[] { 1, 2, 3, 4, 5 }))));
w.flush();
w.commit();
w.forceMerge(1);
try (IndexReader reader = DirectoryReader.open(w)) {
IndexSearcher searcher = new IndexSearcher(reader);
BitSetProducer parentFilter = parentFilter(searcher.getIndexReader());
Query query = new DiversifyingChildrenFloatKnnVectorQuery(fieldName, vector, null, 1, parentFilter);
assertTrue(searcher.search(query, 1).scoreDocs.length == 0);
}
}

}
}

public void testSearch() throws Exception {
String fieldName = "field";
int numVectors = random().nextInt(99, 500);
Expand Down
65 changes: 65 additions & 0 deletions ...a/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsFormatTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.lucene101.Lucene101Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FloatVectorValues;
Expand All @@ -32,18 +33,30 @@
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.KnnVectorValues;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.SoftDeletesRetentionMergePolicy;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.search.FieldExistsQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.KnnFloatVectorQuery;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.search.join.BitSetProducer;
import org.apache.lucene.search.join.CheckJoinIndex;
import org.apache.lucene.search.join.DiversifyingChildrenFloatKnnVectorQuery;
import org.apache.lucene.search.join.QueryBitSetProducer;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
import org.elasticsearch.common.logging.LogConfigurator;
import org.elasticsearch.index.codec.vectors.BQVectorUtils;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;

import static java.lang.String.format;
Expand All @@ -68,6 +81,58 @@ public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
};
}

static String encodeInts(int[] i) {
return Arrays.toString(i);
}

static BitSetProducer parentFilter(IndexReader r) throws IOException {
// Create a filter that defines "parent" documents in the index
BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "_parent")));
CheckJoinIndex.check(r, parentsFilter);
return parentsFilter;
}

Document makeParent(int[] children) {
Document parent = new Document();
parent.add(newStringField("docType", "_parent", Field.Store.NO));
parent.add(newStringField("id", encodeInts(children), Field.Store.YES));
return parent;
}

public void testEmptyDiversifiedChildSearch() throws Exception {
String fieldName = "field";
int dims = random().nextInt(4, 65);
float[] vector = randomVector(dims);
VectorSimilarityFunction similarityFunction = VectorSimilarityFunction.EUCLIDEAN;
try (Directory d = newDirectory()) {
IndexWriterConfig iwc = newIndexWriterConfig().setCodec(getCodec());
iwc.setMergePolicy(new SoftDeletesRetentionMergePolicy("soft_delete", MatchAllDocsQuery::new, iwc.getMergePolicy()));
try (IndexWriter w = new IndexWriter(d, iwc)) {
List<Document> toAdd = new ArrayList<>();
for (int j = 1; j <= 5; j++) {
Document doc = new Document();
doc.add(new KnnFloatVectorField(fieldName, vector, similarityFunction));
doc.add(newStringField("id", Integer.toString(j), Field.Store.YES));
toAdd.add(doc);
}
toAdd.add(makeParent(new int[] { 1, 2, 3, 4, 5 }));
w.addDocuments(toAdd);
w.addDocuments(List.of(makeParent(new int[] { 6, 7, 8, 9, 10 })));
w.deleteDocuments(new FieldExistsQuery(fieldName), new TermQuery(new Term("id", encodeInts(new int[] { 1, 2, 3, 4, 5 }))));
w.flush();
w.commit();
w.forceMerge(1);
try (IndexReader reader = DirectoryReader.open(w)) {
IndexSearcher searcher = new IndexSearcher(reader);
BitSetProducer parentFilter = parentFilter(searcher.getIndexReader());
Query query = new DiversifyingChildrenFloatKnnVectorQuery(fieldName, vector, null, 1, parentFilter);
assertTrue(searcher.search(query, 1).scoreDocs.length == 0);
}
}

}
}

public void testSearch() throws Exception {
String fieldName = "field";
int numVectors = random().nextInt(99, 500);
Expand Down