diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java index 0f3179b546a88..7a77f370979f4 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java @@ -8,7 +8,6 @@ */ package org.elasticsearch.index.fielddata.fieldcomparator; -import org.apache.lucene.index.DocValuesSkipper; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.search.DocIdSetIterator; @@ -16,7 +15,6 @@ import org.apache.lucene.search.LeafFieldComparator; import org.apache.lucene.search.LongValues; import org.apache.lucene.search.Pruning; -import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.util.BitSet; import org.elasticsearch.common.time.DateUtils; @@ -31,7 +29,6 @@ import org.elasticsearch.index.fielddata.SortedNumericLongValues; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.lucene.comparators.XLongComparator; -import org.elasticsearch.lucene.comparators.XNumericComparator; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.sort.BucketedSort; @@ -114,40 +111,6 @@ public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws I protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException { return wrap(getLongValues(context, lMissingValue), maxDoc); } - - @Override - protected XNumericComparator.CompetitiveDISIBuilder buildCompetitiveDISIBuilder(LeafReaderContext context) - throws IOException { - Sort indexSort = context.reader().getMetaData().sort(); - if (indexSort == null) { - return super.buildCompetitiveDISIBuilder(context); - } - SortField[] sortFields = indexSort.getSort(); - if (sortFields.length != 2) { - return super.buildCompetitiveDISIBuilder(context); - } - if (sortFields[1].getField().equals(field) == false) { - return super.buildCompetitiveDISIBuilder(context); - } - DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field); - DocValuesSkipper primaryFieldSkipper = context.reader().getDocValuesSkipper(sortFields[0].getField()); - if (primaryFieldSkipper == null || skipper.docCount() != maxDoc || primaryFieldSkipper.docCount() != maxDoc) { - return super.buildCompetitiveDISIBuilder(context); - } - return new CompetitiveDISIBuilder(this) { - @Override - protected int docCount() { - return skipper.docCount(); - } - - @Override - protected void doUpdateCompetitiveIterator() { - competitiveIterator.update( - new SecondarySortIterator(docValues, skipper, primaryFieldSkipper, minValueAsLong, maxValueAsLong) - ); - } - }; - } }; } }; diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/SecondarySortIterator.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/SecondarySortIterator.java deleted file mode 100644 index 405cc58080751..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/SecondarySortIterator.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.fielddata.fieldcomparator; - -import org.apache.lucene.index.DocValuesSkipper; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.search.DocIdSetIterator; - -import java.io.IOException; - -/** - * A competitive DocIdSetIterator that examines the values of a secondary - * sort field and tries to exclude documents with values outside a given - * range, using DocValueSkippers on the primary sort field to advance rapidly - * to the next block of values. - */ -class SecondarySortIterator extends DocIdSetIterator { - - final NumericDocValues values; - - final DocValuesSkipper valueSkipper; - final DocValuesSkipper primaryFieldSkipper; - final long minValue; - final long maxValue; - - int docID = -1; - boolean skipperMatch; - int primaryFieldUpTo = -1; - int valueFieldUpTo = -1; - - SecondarySortIterator( - NumericDocValues values, - DocValuesSkipper valueSkipper, - DocValuesSkipper primaryFieldSkipper, - long minValue, - long maxValue - ) { - this.values = values; - this.valueSkipper = valueSkipper; - this.primaryFieldSkipper = primaryFieldSkipper; - this.minValue = minValue; - this.maxValue = maxValue; - - valueFieldUpTo = valueSkipper.maxDocID(0); - primaryFieldUpTo = primaryFieldSkipper.maxDocID(0); - } - - @Override - public int docID() { - return docID; - } - - @Override - public int nextDoc() throws IOException { - return advance(docID + 1); - } - - @Override - public int advance(int target) throws IOException { - skipperMatch = false; - target = values.advance(target); - if (target == DocIdSetIterator.NO_MORE_DOCS) { - return docID = target; - } - while (true) { - if (target > valueFieldUpTo) { - valueSkipper.advance(target); - valueFieldUpTo = valueSkipper.maxDocID(0); - long minValue = valueSkipper.minValue(0); - long maxValue = valueSkipper.maxValue(0); - if (minValue > this.maxValue || maxValue < this.minValue) { - // outside the desired range, skip forward - for (int level = 1; level < valueSkipper.numLevels(); level++) { - minValue = valueSkipper.minValue(level); - maxValue = valueSkipper.maxValue(level); - if (minValue > this.maxValue || maxValue < this.minValue) { - valueFieldUpTo = valueSkipper.maxDocID(level); - } else { - break; - } - } - - int upTo = valueFieldUpTo; - if (maxValue < this.minValue) { - // We've moved past the end of the valid values in the secondary sort field - // for this primary value. Advance the primary skipper to find the starting point - // for the next primary value, where the secondary field values will have reset - primaryFieldSkipper.advance(target); - primaryFieldUpTo = primaryFieldSkipper.maxDocID(0); - if (primaryFieldSkipper.minValue(0) == primaryFieldSkipper.maxValue(0)) { - for (int level = 1; level < primaryFieldSkipper.numLevels(); level++) { - if (primaryFieldSkipper.minValue(level) == primaryFieldSkipper.maxValue(level)) { - primaryFieldUpTo = primaryFieldSkipper.maxDocID(level); - } else { - break; - } - } - } - if (primaryFieldUpTo > upTo) { - upTo = primaryFieldUpTo; - } - } - - target = values.advance(upTo + 1); - if (target == DocIdSetIterator.NO_MORE_DOCS) { - return docID = target; - } - } else if (minValue >= this.minValue && maxValue <= this.maxValue) { - assert valueSkipper.docCount(0) == valueSkipper.maxDocID(0) - valueSkipper.minDocID(0) + 1; - skipperMatch = true; - return docID = target; - } - } - - long value = values.longValue(); - if (value < minValue && target > primaryFieldUpTo) { - primaryFieldSkipper.advance(target); - primaryFieldUpTo = primaryFieldSkipper.maxDocID(0); - if (primaryFieldSkipper.minValue(0) == primaryFieldSkipper.maxValue(0)) { - for (int level = 1; level < primaryFieldSkipper.numLevels(); level++) { - if (primaryFieldSkipper.minValue(level) == primaryFieldSkipper.maxValue(level)) { - primaryFieldUpTo = primaryFieldSkipper.maxDocID(level); - } else { - break; - } - } - target = values.advance(primaryFieldUpTo + 1); - if (target == DocIdSetIterator.NO_MORE_DOCS) { - return docID = target; - } - } else { - target = values.nextDoc(); - if (target == DocIdSetIterator.NO_MORE_DOCS) { - return docID = target; - } - } - } else if (value >= minValue && value <= maxValue) { - return docID = target; - } else { - target = values.nextDoc(); - if (target == DocIdSetIterator.NO_MORE_DOCS) { - return docID = target; - } - } - } - } - - @Override - public int docIDRunEnd() throws IOException { - if (skipperMatch) { - return valueFieldUpTo + 1; - } - return super.docIDRunEnd(); - } - - @Override - public long cost() { - return values.cost(); - } - -} diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/SecondarySortIteratorTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/SecondarySortIteratorTests.java deleted file mode 100644 index 54c7892b7b2b3..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/SecondarySortIteratorTests.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.fielddata.fieldcomparator; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.NumericDocValuesField; -import org.apache.lucene.document.SortedDocValuesField; -import org.apache.lucene.index.DocValuesSkipper; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.DocValuesRangeIterator; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TwoPhaseIterator; -import org.apache.lucene.store.Directory; -import org.apache.lucene.tests.analysis.MockAnalyzer; -import org.apache.lucene.tests.index.RandomIndexWriter; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.test.ESTestCase; - -import java.io.IOException; - -public class SecondarySortIteratorTests extends ESTestCase { - - public void testAgainstDocValuesRangeIterator() throws IOException { - - Directory dir = newDirectory(); - Sort indexSort = new Sort(new SortField("hostname", SortField.Type.STRING), new SortField("@timestamp", SortField.Type.LONG, true)); - IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())).setIndexSort(indexSort); - RandomIndexWriter indexWriter = new RandomIndexWriter(random(), dir, iwc); - - int numDocs = atLeast(1000); - for (int i = 0; i < numDocs; i++) { - Document doc = new Document(); - doc.add(SortedDocValuesField.indexedField("hostname", new BytesRef("host1"))); - doc.add(NumericDocValuesField.indexedField("@timestamp", 1_000_000 + i)); - indexWriter.addDocument(doc); - } - - indexWriter.forceMerge(1); - - IndexReader reader = indexWriter.getReader(); - long start = 1_000_400; - long end = 1_000_499; - DocIdSetIterator dvIt = docValuesRangeIterator(reader.leaves().getFirst(), start, end); - DocIdSetIterator ssIt = secondarySortIterator(reader.leaves().getFirst(), start, end); - - // Because the primary sort field has only a single value, we should get exactly the same - // results from the secondary sort iterator as from a standard DVRangeIterator over the - // secondary field - for (int doc = dvIt.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = dvIt.nextDoc()) { - assertEquals(doc, ssIt.nextDoc()); - } - assertEquals(DocIdSetIterator.NO_MORE_DOCS, ssIt.nextDoc()); - - reader.close(); - indexWriter.close(); - dir.close(); - } - - private static DocIdSetIterator secondarySortIterator(LeafReaderContext ctx, long start, long end) throws IOException { - NumericDocValues timeStampDV = ctx.reader().getNumericDocValues("@timestamp"); - DocValuesSkipper primarySkipper = ctx.reader().getDocValuesSkipper("hostname"); - DocValuesSkipper secondarySkipper = ctx.reader().getDocValuesSkipper("@timestamp"); - return new SecondarySortIterator(timeStampDV, secondarySkipper, primarySkipper, start, end); - } - - private static DocIdSetIterator docValuesRangeIterator(LeafReaderContext ctx, long start, long end) throws IOException { - NumericDocValues timeStampDV = ctx.reader().getNumericDocValues("@timestamp"); - TwoPhaseIterator twoPhaseIterator = new TwoPhaseIterator(timeStampDV) { - @Override - public boolean matches() throws IOException { - return timeStampDV.longValue() >= start && timeStampDV.longValue() <= end; - } - - @Override - public float matchCost() { - return 2; - } - }; - DocValuesSkipper skipper = ctx.reader().getDocValuesSkipper("@timestamp"); - return TwoPhaseIterator.asDocIdSetIterator(new DocValuesRangeIterator(twoPhaseIterator, skipper, start, end, false)); - } -} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index c750ca7ee3e1a..9d5c88b10ee7d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -41,6 +41,7 @@ import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.XContentBuilder; +import org.junit.Assert; import java.io.IOException; import java.math.BigDecimal; @@ -934,7 +935,7 @@ protected List getSortShortcutSupport() { b -> b.field("type", "date").field("ignore_malformed", false), b -> b.startObject("host.name").field("type", "keyword").endObject(), b -> b.field("@timestamp", "2025-10-30T00:00:00").field("host.name", "foo"), - checkClass + Assert::assertNotNull ), new SortShortcutSupport(b -> b.field("type", "date"), b -> b.field("field", "2025-10-30T00:00:00"), true), new SortShortcutSupport(b -> b.field("type", "date_nanos"), b -> b.field("field", "2025-10-30T00:00:00"), true), diff --git a/server/src/test/java/org/elasticsearch/lucene/comparators/SkipperPruningTests.java b/server/src/test/java/org/elasticsearch/lucene/comparators/SkipperPruningTests.java new file mode 100644 index 0000000000000..5e0a932fa9f7d --- /dev/null +++ b/server/src/test/java/org/elasticsearch/lucene/comparators/SkipperPruningTests.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.lucene.comparators; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource; +import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; +import org.elasticsearch.index.mapper.IndexType; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +public class SkipperPruningTests extends ESTestCase { + + public void testCompetitiveIterator() throws IOException { + + Directory dir = newDirectory(); + Sort indexSort = new Sort(new SortField("hostname", SortField.Type.STRING), new SortField("@timestamp", SortField.Type.LONG, true)); + IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())).setIndexSort(indexSort); + RandomIndexWriter indexWriter = new RandomIndexWriter(random(), dir, iwc); + + int numDocs = atLeast(1000); + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + doc.add(SortedDocValuesField.indexedField("hostname", new BytesRef("host1"))); + doc.add(NumericDocValuesField.indexedField("@timestamp", 1_000_000 + i)); + indexWriter.addDocument(doc); + } + + indexWriter.forceMerge(1); + + IndexReader reader = indexWriter.getReader(); + + SortedNumericIndexFieldData fd = new SortedNumericIndexFieldData( + "@timestamp", + IndexNumericFieldData.NumericType.LONG, + CoreValuesSourceType.NUMERIC, + null, + IndexType.skippers() + ); + LongValuesComparatorSource source = new LongValuesComparatorSource( + fd, + null, + MultiValueMode.MAX, + null, + IndexNumericFieldData.NumericType.LONG + ); + + var comparator = (XLongComparator) source.newComparator("@timestamp", 1, Pruning.GREATER_THAN_OR_EQUAL_TO, true); + comparator.queueFull = true; + comparator.hitsThresholdReached = true; + comparator.bottom = Long.MAX_VALUE; + var leafComparator = comparator.getLeafComparator(reader.leaves().getFirst()); + assertThat(leafComparator.competitiveIterator().nextDoc(), equalTo(DocIdSetIterator.NO_MORE_DOCS)); + + reader.close(); + indexWriter.close(); + dir.close(); + } +}