Merge branch 'main' into lucene_snapshot

docs/changelog/107158.yaml

@@ -0,0 +1,5 @@
+pr: 107158
+summary: "ESQL: allow sorting by expressions and not only regular fields"
+area: ES|QL
+type: feature
+issues: []

docs/reference/docs/bulk.asciidoc

@@ -414,9 +414,7 @@ This parameter is only returned for successful actions.
 `result`::
 (string)
 Result of the operation. Successful values are `created`, `deleted`, and
-`updated`.
-+
-This parameter is only returned for successful operations.
+`updated`. Other valid values are `noop` and `not_found`.
 
 `_shards`::
 (object)

docs/reference/esql/esql-functions-operators.asciidoc

@@ -1,40 +1,71 @@
 [[esql-functions-operators]]
 === {esql} functions and operators
-
 ++++
 <titleabbrev>Functions and operators</titleabbrev>
 ++++
 
 {esql} provides a comprehensive set of functions and operators for working with data.
-The functions are divided into the following categories:
+The reference documentation is divided into the following categories:
 
 [[esql-functions]]
-<<esql-agg-functions>>::
+==== Functions overview
+
+.*Aggregate functions*
+[%collapsible]
+====
 include::functions/aggregation-functions.asciidoc[tag=agg_list]
+====
 
-<<esql-math-functions>>::
+.*Math functions*
+[%collapsible]
+====
 include::functions/math-functions.asciidoc[tag=math_list]
+====
 
-<<esql-string-functions>>::
+.*String functions*
+[%collapsible]
+====
 include::functions/string-functions.asciidoc[tag=string_list]
+====
 
-<<esql-date-time-functions>>::
+.*Date and time functions*
+[%collapsible]
+====
 include::functions/date-time-functions.asciidoc[tag=date_list]
+====
 
-<<esql-spatial-functions>>::
+.*Spatial functions*
+[%collapsible]
+====
 include::functions/spatial-functions.asciidoc[tag=spatial_list]
+====
 
-<<esql-type-conversion-functions>>::
+.*Type conversion functions*
+[%collapsible]
+====
 include::functions/type-conversion-functions.asciidoc[tag=type_list]
+====
 
-<<esql-conditional-functions-and-expressions>>::
+.*Conditional functions and expressions*
+[%collapsible]
+====
 include::functions/conditional-functions-and-expressions.asciidoc[tag=cond_list]
+====
 
-<<esql-mv-functions>>::
+.*Multi value functions*
+[%collapsible]
+====
 include::functions/mv-functions.asciidoc[tag=mv_list]
+====
+
+[[esql-operators-overview]]
+==== Operators overview
 
-<<esql-operators>>::
+.*Operators*
+[%collapsible]
+====
 include::functions/operators.asciidoc[tag=op_list]
+====
 
 include::functions/aggregation-functions.asciidoc[]
 include::functions/math-functions.asciidoc[]

docs/reference/esql/esql-language.asciidoc

@@ -1,11 +1,10 @@
 [[esql-language]]
-== Learning {esql}
-
+== {esql} reference
 ++++
-<titleabbrev>Learning {esql}</titleabbrev>
+<titleabbrev>{esql} reference</titleabbrev>
 ++++
 
-Detailed information about the {esql} language:
+Detailed reference documentation for the {esql} language:
 
 * <<esql-syntax>>
 * <<esql-commands>>

docs/reference/esql/functions/types/auto_bucket.asciidoc

@@ -5,5 +5,40 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 field | buckets | from | to | result
-
+datetime | integer | datetime | datetime | datetime
+datetime | integer | datetime | keyword | datetime
+datetime | integer | datetime | text | datetime
+datetime | integer | keyword | datetime | datetime
+datetime | integer | keyword | keyword | datetime
+datetime | integer | keyword | text | datetime
+datetime | integer | text | datetime | datetime
+datetime | integer | text | keyword | datetime
+datetime | integer | text | text | datetime
+double | integer | double | double | double
+double | integer | double | integer | double
+double | integer | double | long | double
+double | integer | integer | double | double
+double | integer | integer | integer | double
+double | integer | integer | long | double
+double | integer | long | double | double
+double | integer | long | integer | double
+double | integer | long | long | double
+integer | integer | double | double | double
+integer | integer | double | integer | double
+integer | integer | double | long | double
+integer | integer | integer | double | double
+integer | integer | integer | integer | double
+integer | integer | integer | long | double
+integer | integer | long | double | double
+integer | integer | long | integer | double
+integer | integer | long | long | double
+long | integer | double | double | double
+long | integer | double | integer | double
+long | integer | double | long | double
+long | integer | integer | double | double
+long | integer | integer | integer | double
+long | integer | integer | long | double
+long | integer | long | double | double
+long | integer | long | integer | double
+long | integer | long | long | double
 |===

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java

@@ -340,7 +340,8 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
                         valuesSourceType,
                         (dv, n) -> {
                             throw new UnsupportedOperationException();
-                        }
+                        },
+                        isIndexed()
                     ).build(cache, breakerService);
                     return new ScaledFloatIndexFieldData(scaledValues, scalingFactor, ScaledFloatDocValuesField::new);
                 };
@@ -608,6 +609,11 @@ protected boolean sortRequiresCustomComparator() {
             return true;
         }
 
+        @Override
+        protected boolean isIndexed() {
+            return false; // We don't know how to take advantage of the index with half floats anyway
+        }
+
         @Override
         public NumericType getNumericType() {
             /*

modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java

@@ -267,7 +267,7 @@ protected S3Repository createRepository(
         ) {
             return new S3Repository(metadata, registry, getService(), clusterService, bigArrays, recoverySettings, s3RepositoriesMetrics) {
                 @Override
-                protected void assertSnapshotOrGenericThread() {
+                protected void assertSnapshotOrStatelessPermittedThreadPool() {
                     // eliminate thread name check as we create repo manually on test/main threads
                 }
             };

modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java

@@ -132,7 +132,7 @@ private S3Repository createS3Repo(RepositoryMetadata metadata) {
             S3RepositoriesMetrics.NOOP
         ) {
             @Override
-            protected void assertSnapshotOrGenericThread() {
+            protected void assertSnapshotOrStatelessPermittedThreadPool() {
                 // eliminate thread name check as we create repo manually on test/main threads
             }
         };

modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java

@@ -43,7 +43,7 @@ private URLRepository createRepository(Settings baseSettings, RepositoryMetadata
             mock(URLHttpClient.Factory.class)
         ) {
             @Override
-            protected void assertSnapshotOrGenericThread() {
+            protected void assertSnapshotOrStatelessPermittedThreadPool() {
                 // eliminate thread name check as we create repo manually on test/main threads
             }
         };

plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java

@@ -80,7 +80,7 @@ public String typeName() {
         @Override
         public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) {
             failIfNoDocValues();
-            return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG, Murmur3DocValueField::new);
+            return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG, Murmur3DocValueField::new, isIndexed());
         }
 
         @Override

server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java

@@ -33,6 +33,9 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
+/**
+ * A collection of static methods to help create different ES Executor types.
+ */
 public class EsExecutors {
 
     // although the available processors may technically change, for node sizing we use the number available at launch

server/src/main/java/org/elasticsearch/index/fielddata/BooleanScriptFieldData.java

@@ -90,6 +90,11 @@ protected boolean sortRequiresCustomComparator() {
         return true;
     }
 
+    @Override
+    protected boolean isIndexed() {
+        return false;
+    }
+
     public static class BooleanScriptLeafFieldData extends LeafLongFieldData {
         private final BooleanScriptDocValues booleanScriptDocValues;
         protected final ToScriptFieldFactory<SortedNumericDocValues> toScriptFieldFactory;

server/src/main/java/org/elasticsearch/index/fielddata/DateScriptFieldData.java

@@ -90,6 +90,11 @@ protected boolean sortRequiresCustomComparator() {
         return true;
     }
 
+    @Override
+    protected boolean isIndexed() {
+        return false;
+    }
+
     public static class DateScriptLeafFieldData extends LeafLongFieldData {
         private final LongScriptDocValues longScriptDocValues;
         protected final ToScriptFieldFactory<SortedNumericDocValues> toScriptFieldFactory;

server/src/main/java/org/elasticsearch/index/fielddata/DoubleScriptFieldData.java

@@ -89,6 +89,11 @@ protected boolean sortRequiresCustomComparator() {
         return true;
     }
 
+    @Override
+    protected boolean isIndexed() {
+        return false;
+    }
+
     public static class DoubleScriptLeafFieldData extends LeafDoubleFieldData {
         private final DoubleScriptDocValues doubleScriptDocValues;
         protected final ToScriptFieldFactory<SortedNumericDoubleValues> toScriptFieldFactory;

server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java

@@ -121,6 +121,7 @@ public final SortField sortField(
             case LONG:
             case DOUBLE:
                 // longs, doubles and dates use the same type for doc-values and points.
+                sortField.setOptimizeSortWithPoints(isIndexed());
                 break;
 
             default:
@@ -132,12 +133,18 @@ public final SortField sortField(
     }
 
     /**
-     * Does {@link #sortField} require a custom comparator because of the way
-     * the data is stored in doc values ({@code true}) or are the docs values
-     * stored such that they can be sorted without decoding ({@code false}).
+     * Should sorting use a custom comparator source vs. rely on a Lucene {@link SortField}. Using a Lucene {@link SortField} when possible
+     * is important because index sorting cannot be configured with a custom comparator, and because it gives better performance by
+     * dynamically pruning irrelevant hits. On the other hand, Lucene {@link SortField}s are less flexible and make stronger assumptions
+     * about how the data is indexed. Therefore, they cannot be used in all cases.
      */
     protected abstract boolean sortRequiresCustomComparator();
 
+    /**
+     * Return true if, and only if the field is indexed with points that match the content of doc values.
+     */
+    protected abstract boolean isIndexed();
+
     @Override
     public final SortField sortField(Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
         return sortField(getNumericType(), missingValue, sortMode, nested, reverse);

server/src/main/java/org/elasticsearch/index/fielddata/LongScriptFieldData.java

@@ -92,6 +92,11 @@ protected boolean sortRequiresCustomComparator() {
         return true;
     }
 
+    @Override
+    protected boolean isIndexed() {
+        return false;
+    }
+
     public static class LongScriptLeafFieldData extends LeafLongFieldData {
         private final LongScriptDocValues longScriptDocValues;
         protected final ToScriptFieldFactory<SortedNumericDocValues> toScriptFieldFactory;

server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedDoublesIndexFieldData.java

@@ -42,41 +42,47 @@ public static class Builder implements IndexFieldData.Builder {
         private final NumericType numericType;
         private final ValuesSourceType valuesSourceType;
         protected final ToScriptFieldFactory<SortedNumericDoubleValues> toScriptFieldFactory;
+        private final boolean indexed;
 
         public Builder(
             String name,
             NumericType numericType,
             ValuesSourceType valuesSourceType,
-            ToScriptFieldFactory<SortedNumericDoubleValues> toScriptFieldFactory
+            ToScriptFieldFactory<SortedNumericDoubleValues> toScriptFieldFactory,
+            boolean indexed
         ) {
             this.name = name;
             this.numericType = numericType;
             this.valuesSourceType = valuesSourceType;
             this.toScriptFieldFactory = toScriptFieldFactory;
+            this.indexed = indexed;
         }
 
         @Override
         public SortedDoublesIndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) {
-            return new SortedDoublesIndexFieldData(name, numericType, valuesSourceType, toScriptFieldFactory);
+            return new SortedDoublesIndexFieldData(name, numericType, valuesSourceType, toScriptFieldFactory, indexed);
         }
     }
 
     private final NumericType numericType;
     protected final String fieldName;
     protected final ValuesSourceType valuesSourceType;
     protected final ToScriptFieldFactory<SortedNumericDoubleValues> toScriptFieldFactory;
+    protected final boolean indexed;
 
     public SortedDoublesIndexFieldData(
         String fieldName,
         NumericType numericType,
         ValuesSourceType valuesSourceType,
-        ToScriptFieldFactory<SortedNumericDoubleValues> toScriptFieldFactory
+        ToScriptFieldFactory<SortedNumericDoubleValues> toScriptFieldFactory,
+        boolean indexed
     ) {
         this.fieldName = fieldName;
         this.numericType = Objects.requireNonNull(numericType);
         assert this.numericType.isFloatingPoint();
         this.valuesSourceType = valuesSourceType;
         this.toScriptFieldFactory = toScriptFieldFactory;
+        this.indexed = indexed;
     }
 
     @Override
@@ -94,6 +100,11 @@ protected boolean sortRequiresCustomComparator() {
         return numericType == NumericType.HALF_FLOAT;
     }
 
+    @Override
+    public boolean isIndexed() {
+        return indexed;
+    }
+
     @Override
     public NumericType getNumericType() {
         return numericType;