diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java index 4bfa1a51f3f80..b7f34be35cadf 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java @@ -868,6 +868,148 @@ public void testSortMissingStrings() throws IOException { assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3")); } + public void testSortMissingDates() throws IOException { + for (String type : new String[]{"date", "date_nanos"}) { + String index = "test_" + type; + assertAcked( + prepareCreate(index).addMapping( + "_doc", + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("mydate") + .field("type", type) + .endObject() + .endObject() + .endObject() + .endObject() + ) + ); + ensureGreen(); + client().prepareIndex(index, "_doc").setId("1").setSource("mydate", "2021-01-01").get(); + client().prepareIndex(index, "_doc").setId("2").setSource("mydate", "2021-02-01").get(); + client().prepareIndex(index, "_doc").setId("3").setSource("other_field", "value").get(); + + refresh(); + + for (boolean withFormat : new boolean[] {true, false}) { + String format = null; + if (withFormat) { + format = type.equals("date") ? "strict_date_optional_time" : "strict_date_optional_time_nanos"; + } + + SearchResponse searchResponse = client().prepareSearch(index) + .addSort(SortBuilders.fieldSort("mydate").order(SortOrder.ASC).setFormat(format)) + .get(); + assertHitsInOrder(searchResponse, new String[] { "1", "2", "3" }); + + searchResponse = client().prepareSearch(index) + .addSort(SortBuilders.fieldSort("mydate").order(SortOrder.ASC).missing("_first").setFormat(format)) + .get(); + assertHitsInOrder(searchResponse, new String[] { "3", "1", "2" }); + + searchResponse = client().prepareSearch(index) + .addSort(SortBuilders.fieldSort("mydate").order(SortOrder.DESC).setFormat(format)) + .get(); + assertHitsInOrder(searchResponse, new String[] { "2", "1", "3" }); + + searchResponse = client().prepareSearch(index) + .addSort(SortBuilders.fieldSort("mydate").order(SortOrder.DESC).missing("_first").setFormat(format)) + .get(); + assertHitsInOrder(searchResponse, new String[] { "3", "2", "1" }); + } + } + } + + /** + * Sort across two indices with both "date" and "date_nanos" type using "numeric_type" set to "date_nanos" + */ + public void testSortMissingDatesMixedTypes() throws IOException { + for (String type : new String[] { "date", "date_nanos" }) { + String index = "test_" + type; + assertAcked( + prepareCreate(index).addMapping( + "_doc", + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("mydate") + .field("type", type) + .endObject() + .endObject() + .endObject() + .endObject() + ) + ); + + } + ensureGreen(); + + client().prepareIndex("test_date", "_doc").setId("1").setSource("mydate", "2021-01-01").get(); + client().prepareIndex("test_date", "_doc").setId("2").setSource("mydate", "2021-02-01").get(); + client().prepareIndex("test_date", "_doc").setId("3").setSource("other_field", 1).get(); + client().prepareIndex("test_date_nanos", "_doc").setId("4").setSource("mydate", "2021-03-01").get(); + client().prepareIndex("test_date_nanos", "_doc").setId("5").setSource("mydate", "2021-04-01").get(); + client().prepareIndex("test_date_nanos", "_doc").setId("6").setSource("other_field", 2).get(); + refresh(); + + for (boolean withFormat : new boolean[] {true, false}) { + String format = null; + if (withFormat) { + format = "strict_date_optional_time_nanos"; + } + + String index = "test*"; + SearchResponse searchResponse = client().prepareSearch(index) + .addSort(SortBuilders.fieldSort("mydate").order(SortOrder.ASC).setFormat(format).setNumericType("date_nanos")) + .addSort(SortBuilders.fieldSort("other_field").order(SortOrder.ASC)) + .get(); + assertHitsInOrder(searchResponse, new String[] { "1", "2", "4", "5", "3", "6" }); + + searchResponse = client().prepareSearch(index) + .addSort( + SortBuilders.fieldSort("mydate") + .order(SortOrder.ASC) + .missing("_first") + .setFormat(format) + .setNumericType("date_nanos") + ) + .addSort(SortBuilders.fieldSort("other_field").order(SortOrder.ASC)) + .get(); + assertHitsInOrder(searchResponse, new String[] { "3", "6", "1", "2", "4", "5" }); + + searchResponse = client().prepareSearch(index) + .addSort(SortBuilders.fieldSort("mydate").order(SortOrder.DESC).setFormat(format).setNumericType("date_nanos")) + .addSort(SortBuilders.fieldSort("other_field").order(SortOrder.ASC)) + .get(); + assertHitsInOrder(searchResponse, new String[] { "5", "4", "2", "1", "3", "6" }); + + searchResponse = client().prepareSearch(index) + .addSort( + SortBuilders.fieldSort("mydate") + .order(SortOrder.DESC) + .missing("_first") + .setFormat(format) + .setNumericType("date_nanos") + ) + .addSort(SortBuilders.fieldSort("other_field").order(SortOrder.ASC)) + .get(); + assertHitsInOrder(searchResponse, new String[] { "3", "6", "5", "4", "2", "1" }); + } + } + + private void assertHitsInOrder(SearchResponse response, String[] expectedIds) { + SearchHit[] hits = response.getHits().getHits(); + assertEquals(expectedIds.length, hits.length); + int i = 0; + for (String id : expectedIds) { + assertEquals(id, hits[i].getId()); + i++; + } + } + public void testIgnoreUnmapped() throws Exception { createIndex("test"); diff --git a/server/src/main/java/org/elasticsearch/common/time/DateUtils.java b/server/src/main/java/org/elasticsearch/common/time/DateUtils.java index b77d85aefecca..6f399c1ca6985 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateUtils.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateUtils.java @@ -207,6 +207,8 @@ public static ZoneId of(String zoneId) { static final long MAX_NANOSECOND_IN_MILLIS = MAX_NANOSECOND_INSTANT.toEpochMilli(); + public static final long MAX_NANOSECOND = toLong(MAX_NANOSECOND_INSTANT); + /** * convert a java time instant to a long value which is stored in lucene * the long value resembles the nanoseconds since the epoch diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java index f50fe5f023290..4c3a8fbc5fdbf 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java @@ -23,8 +23,8 @@ import org.apache.lucene.util.BitDocIdSet; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.core.Nullable; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.DocValueFormat; @@ -142,17 +142,17 @@ public DocIdSetIterator innerDocs(LeafReaderContext ctx) throws IOException { } /** Whether missing values should be sorted first. */ - public final boolean sortMissingFirst(Object missingValue) { + public static final boolean sortMissingFirst(Object missingValue) { return "_first".equals(missingValue); } /** Whether missing values should be sorted last, this is the default. */ - public final boolean sortMissingLast(Object missingValue) { + public static final boolean sortMissingLast(Object missingValue) { return missingValue == null || "_last".equals(missingValue); } /** Return the missing object value according to the reduced type of the comparator. */ - public final Object missingObject(Object missingValue, boolean reversed) { + public Object missingObject(Object missingValue, boolean reversed) { if (sortMissingFirst(missingValue) || sortMissingLast(missingValue)) { final boolean min = sortMissingFirst(missingValue) ^ reversed; switch (reducedType()) { @@ -199,7 +199,7 @@ public final Object missingObject(Object missingValue, boolean reversed) { case STRING: case STRING_VAL: if (missingValue instanceof BytesRef) { - return (BytesRef) missingValue; + return missingValue; } else if (missingValue instanceof byte[]) { return new BytesRef((byte[]) missingValue); } else { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java index 72b69215508a3..8dfa8a1e14502 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java @@ -12,9 +12,9 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortedNumericSelector; import org.apache.lucene.search.SortedNumericSortField; -import org.elasticsearch.core.Nullable; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource; import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource; @@ -156,16 +156,31 @@ private XFieldComparatorSource comparatorSource( return dateNanosComparatorSource(missingValue, sortMode, nested); default: assert targetNumericType.isFloatingPoint() == false; - return new LongValuesComparatorSource(this, missingValue, sortMode, nested); + return new LongValuesComparatorSource(this, missingValue, sortMode, nested, targetNumericType); } } - protected XFieldComparatorSource dateComparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) { - return new LongValuesComparatorSource(this, missingValue, sortMode, nested); + protected XFieldComparatorSource dateComparatorSource( + @Nullable Object missingValue, + MultiValueMode sortMode, + Nested nested + ) { + return new LongValuesComparatorSource(this, missingValue, sortMode, nested, NumericType.DATE); } - protected XFieldComparatorSource dateNanosComparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) { - return new LongValuesComparatorSource(this, missingValue, sortMode, nested, dvs -> convertNumeric(dvs, DateUtils::toNanoSeconds)); + protected XFieldComparatorSource dateNanosComparatorSource( + @Nullable Object missingValue, + MultiValueMode sortMode, + Nested nested + ) { + return new LongValuesComparatorSource( + this, + missingValue, + sortMode, + nested, + dvs -> convertNumeric(dvs, DateUtils::toNanoSeconds), + NumericType.DATE_NANOSECONDS + ); } /** diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java index 6c144a914e2b5..d237819502006 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java @@ -16,12 +16,14 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.comparators.LongComparator; import org.apache.lucene.util.BitSet; -import org.elasticsearch.core.Nullable; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.index.fielddata.LeafNumericFieldData; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; +import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; @@ -38,18 +40,20 @@ public class LongValuesComparatorSource extends IndexFieldData.XFieldComparatorS private final IndexNumericFieldData indexFieldData; private final Function converter; + private final NumericType targetNumericType; public LongValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, - MultiValueMode sortMode, Nested nested) { - this(indexFieldData, missingValue, sortMode, nested, null); + MultiValueMode sortMode, Nested nested, NumericType targetNumericType) { + this(indexFieldData, missingValue, sortMode, nested, null, targetNumericType); } public LongValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue, MultiValueMode sortMode, Nested nested, - Function converter) { + Function converter, NumericType targetNumericType) { super(missingValue, sortMode, nested); this.indexFieldData = indexFieldData; this.converter = converter; + this.targetNumericType = targetNumericType; } @Override @@ -128,4 +132,16 @@ protected long docValue() { } }; } + + @Override + public Object missingObject(Object missingValue, boolean reversed) { + if (targetNumericType == NumericType.DATE_NANOSECONDS) { + // special case to prevent negative values that would cause invalid nanosecond ranges + if (sortMissingFirst(missingValue) || sortMissingLast(missingValue)) { + final boolean min = sortMissingFirst(missingValue) ^ reversed; + return min ? 0L : DateUtils.MAX_NANOSECOND; + } + } + return super.missingObject(missingValue, reversed); + } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java index af4c3e9197be6..a3c7e649ad100 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java @@ -87,23 +87,37 @@ protected boolean sortRequiresCustomComparator() { } @Override - protected XFieldComparatorSource dateComparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) { + protected XFieldComparatorSource dateComparatorSource( + Object missingValue, + MultiValueMode sortMode, + Nested nested + ) { if (numericType == NumericType.DATE_NANOSECONDS) { // converts date_nanos values to millisecond resolution return new LongValuesComparatorSource(this, missingValue, - sortMode, nested, dvs -> convertNumeric(dvs, DateUtils::toMilliSeconds)); + sortMode, nested, dvs -> convertNumeric(dvs, DateUtils::toMilliSeconds), NumericType.DATE); } - return new LongValuesComparatorSource(this, missingValue, sortMode, nested); + return new LongValuesComparatorSource(this, missingValue, sortMode, nested, NumericType.DATE); } @Override - protected XFieldComparatorSource dateNanosComparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) { + protected XFieldComparatorSource dateNanosComparatorSource( + Object missingValue, + MultiValueMode sortMode, + Nested nested + ) { if (numericType == NumericType.DATE) { // converts date values to nanosecond resolution - return new LongValuesComparatorSource(this, missingValue, - sortMode, nested, dvs -> convertNumeric(dvs, DateUtils::toNanoSeconds)); - } - return new LongValuesComparatorSource(this, missingValue, sortMode, nested); + return new LongValuesComparatorSource( + this, + missingValue, + sortMode, + nested, + dvs -> convertNumeric(dvs, DateUtils::toNanoSeconds), + NumericType.DATE_NANOSECONDS + ); + } + return new LongValuesComparatorSource(this, missingValue, sortMode, nested, NumericType.DATE_NANOSECONDS); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetOrdinalsIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetOrdinalsIndexFieldData.java index afac4c68aa9da..8b394c35ea107 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetOrdinalsIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetOrdinalsIndexFieldData.java @@ -14,8 +14,8 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortedSetSelector; import org.apache.lucene.search.SortedSetSortField; -import org.elasticsearch.core.Nullable; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.IndexFieldDataCache; @@ -31,6 +31,9 @@ import java.util.function.Function; +import static org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.sortMissingFirst; +import static org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.sortMissingLast; + public class SortedSetOrdinalsIndexFieldData extends AbstractIndexOrdinalsFieldData { public static class Builder implements IndexFieldData.Builder { @@ -76,12 +79,12 @@ public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMod */ if (nested != null || (sortMode != MultiValueMode.MAX && sortMode != MultiValueMode.MIN) || - (source.sortMissingLast(missingValue) == false && source.sortMissingFirst(missingValue) == false)) { + (sortMissingLast(missingValue) == false && sortMissingFirst(missingValue) == false)) { return new SortField(getFieldName(), source, reverse); } SortField sortField = new SortedSetSortField(getFieldName(), reverse, sortMode == MultiValueMode.MAX ? SortedSetSelector.Type.MAX : SortedSetSelector.Type.MIN); - sortField.setMissingValue(source.sortMissingLast(missingValue) ^ reverse ? + sortField.setMissingValue(sortMissingLast(missingValue) ^ reverse ? SortedSetSortField.STRING_LAST : SortedSetSortField.STRING_FIRST); return sortField; } diff --git a/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java b/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java index 18790002a2e71..7f5fe7a09b1c9 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java @@ -48,8 +48,8 @@ import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.core.Tuple; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.core.Tuple; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; @@ -680,7 +680,7 @@ private static Tuple randomSortFieldCustomComparatorSource switch(randomIntBetween(0, 3)) { case 0: comparatorSource = new LongValuesComparatorSource(null, randomBoolean() ? randomLong() : null, - randomFrom(MultiValueMode.values()), null); + randomFrom(MultiValueMode.values()), null, null); break; case 1: comparatorSource = new DoubleValuesComparatorSource(null, randomBoolean() ? randomDouble() : null, diff --git a/server/src/test/java/org/elasticsearch/index/search/nested/LongNestedSortingTests.java b/server/src/test/java/org/elasticsearch/index/search/nested/LongNestedSortingTests.java index d996acabbef04..e4d8ed03b0526 100644 --- a/server/src/test/java/org/elasticsearch/index/search/nested/LongNestedSortingTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/nested/LongNestedSortingTests.java @@ -23,10 +23,14 @@ protected String getFieldDataType() { } @Override - protected IndexFieldData.XFieldComparatorSource createFieldComparator(String fieldName, MultiValueMode sortMode, - Object missingValue, Nested nested) { + protected IndexFieldData.XFieldComparatorSource createFieldComparator( + String fieldName, + MultiValueMode sortMode, + Object missingValue, + Nested nested + ) { IndexNumericFieldData fieldData = getForField(fieldName); - return new LongValuesComparatorSource(fieldData, missingValue, sortMode, nested); + return new LongValuesComparatorSource(fieldData, missingValue, sortMode, nested, null); } @Override