Merge remote-tracking branch 'origin/master' into snapshot-lifecycle-…

…management

.ci/java-versions.properties

@@ -4,7 +4,7 @@
 # build and test Elasticsearch for this branch. Valid Java versions
 # are 'java' or 'openjdk' followed by the major release number.
 
-ES_BUILD_JAVA=java11
+ES_BUILD_JAVA=openjdk12
 ES_RUNTIME_JAVA=java8
 GRADLE_TASK=build
 

.ci/matrix-build-javas.yml

@@ -6,5 +6,4 @@
 # or 'openjdk' followed by the major release number.
 
 ES_BUILD_JAVA:
-  - java11
   - openjdk12

CONTRIBUTING.md

@@ -92,11 +92,11 @@ Contributing to the Elasticsearch codebase
 
 **Repository:** [https://github.com/elastic/elasticsearch](https://github.com/elastic/elasticsearch)
 
-JDK 11 is required to build Elasticsearch. You must have a JDK 11 installation
+JDK 12 is required to build Elasticsearch. You must have a JDK 12 installation
 with the environment variable `JAVA_HOME` referencing the path to Java home for
-your JDK 11 installation. By default, tests use the same runtime as `JAVA_HOME`.
+your JDK 12 installation. By default, tests use the same runtime as `JAVA_HOME`.
 However, since Elasticsearch supports JDK 8, the build supports compiling with
-JDK 11 and testing on a JDK 8 runtime; to do this, set `RUNTIME_JAVA_HOME`
+JDK 12 and testing on a JDK 8 runtime; to do this, set `RUNTIME_JAVA_HOME`
 pointing to the Java home of a JDK 8 installation. Note that this mechanism can
 be used to test against other JDKs as well, this is not only limited to JDK 8.
 

buildSrc/src/main/resources/minimumCompilerVersion

@@ -1 +1 @@
-1.11
+1.12

distribution/bwc/build.gradle

@@ -169,8 +169,9 @@ bwcVersions.forPreviousUnreleased { BwcVersions.UnreleasedVersionInfo unreleased
                         'JAVA_HOME',
                         getJavaHome(it, Integer.parseInt(
                                 lines
-                                        .findAll({ it.startsWith("ES_BUILD_JAVA=java") })
+                                        .findAll({ it.startsWith("ES_BUILD_JAVA=")})
                                         .collect({ it.replace("ES_BUILD_JAVA=java", "").trim() })
+                                        .collect({ it.replace("ES_BUILD_JAVA=openjdk", "").trim() })
                                         .join("!!")
                         ))
                 )

docs/reference/sql/language/syntax/commands/select.asciidoc

@@ -303,7 +303,7 @@ Represents an input column, an output column or an ordinal number of the positio
 The direction, if not specified, is by default `ASC` (ascending).
 Regardless of the ordering specified, null values are ordered last (at the end).
 
-IMPORTANT: When used along-side, `GROUP BY` expression can point _only_ to the columns used for grouping.
+IMPORTANT: When used along-side, `GROUP BY` expression can point _only_ to the columns used for grouping or aggregate functions.
 
 For example, the following query sorts by an arbitrary input field (`page_count`):
 
@@ -312,6 +312,37 @@ For example, the following query sorts by an arbitrary input field (`page_count`
 include-tagged::{sql-specs}/docs.csv-spec[orderByBasic]
 ----
 
+[[sql-syntax-order-by-grouping]]
+==== Order By and Grouping
+
+For queries that perform grouping, ordering can be applied either on the grouping columns (by default ascending) or on aggregate functions. 
+
+NOTE: With `GROUP BY`, make sure the ordering targets the resulting group - applying it to individual elements inside the group will have no impact on the results since regardless of the order, values inside the group are aggregated.
+
+For example, to order groups simply indicate the grouping key:
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[orderByGroup]
+----
+
+Multiple keys can be specified of course:
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[groupByMulti]
+----
+
+Further more, it is possible to order groups based on aggregations of their values:
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[orderByAgg]
+----
+
+IMPORTANT: Ordering by aggregation is possible for up to 512 entries for memory consumption reasons.
+In cases where the results pass this threshold, use <<`LIMIT`, sql-syntax-limit>> to reduce the number
+of results.
+
 [[sql-syntax-order-by-score]]
 ==== Order By Score
 

docs/reference/upgrade/rolling_upgrade.asciidoc

@@ -178,4 +178,10 @@ In the unlikely case of a network malfunction during the upgrade process that
 isolates all remaining old nodes from the cluster, you must take the
 old nodes offline and upgrade them to enable them to join the cluster.
 
+Similarly, if you run a testing/development environment with only one master
+node, the master node should be upgraded last. Restarting a single master node
+forces the cluster to be reformed. The new cluster will initially only have the
+upgraded master node and will thus reject the older nodes when they re-join the
+cluster. Nodes that have already been upgraded will successfully re-join the 
+upgraded master. 
 ====================================================

modules/lang-painless/src/test/java/org/elasticsearch/painless/StandardCastTests.java

@@ -1768,4 +1768,166 @@ public void testFloatCasts() {
         expectScriptThrows(ClassCastException.class, () -> exec("Float o = Float.valueOf((float)0); ArrayList b = o;"));
         expectScriptThrows(ClassCastException.class, () -> exec("Float o = Float.valueOf((float)0); ArrayList b = (ArrayList)o;"));
     }
+
+    public void testPrimitiveDoubleCasts() {
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; Object n = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; Object n = (Object)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; Number n = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; Number n = (Number)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; String n = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; String n = (String)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; boolean b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; boolean b = (boolean)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; byte b = o;"));
+        exec("double o = 0; byte b = (byte)o;");
+
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; short b = o;"));
+        exec("double o = 0; short b = (short)o;");
+
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; char b = o;"));
+        exec("double o = 0; char b = (char)o;");
+
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; int b = o;"));
+        exec("double o = 0; int b = (int)o;");
+
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; long b = o;"));
+        exec("double o = 0; long b = (long)o;");
+
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; float b = o;"));
+        exec("double o = 0; float b = (float)o;");
+
+        exec("double o = 0; double b = o;");
+        exec("double o = 0; double b = (double)o;");
+
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; Boolean b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; Boolean b = (Boolean)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; Byte b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; Byte b = (Byte)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; Short b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; Short b = (Short)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; Character b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; Character b = (Character)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; Integer b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; Integer b = (Integer)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; Long b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; Long b = (Long)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; Float b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; Float b = (Float)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; Double b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; Double b = (Double)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; ArrayList b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("double o = 0; ArrayList b = (ArrayList)o;"));
+    }
+
+    public void testDoubleCasts() {
+        exec("Double o = Double.valueOf((double)0); Object n = o;");
+        exec("Double o = null; Object n = o;");
+        exec("Double o = Double.valueOf((double)0); Object n = (Object)o;");
+        exec("Double o = null; Object n = (Object)o;");
+
+        exec("Double o = Double.valueOf((double)0); Number n = o;");
+        exec("Double o = null; Number n = o;");
+        exec("Double o = Double.valueOf((double)0); Number n = (Number)o;");
+        exec("Double o = null; Number n = (Number)o;");
+
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); String n = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; String n = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); String n = (String)o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; String n = (String)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); boolean b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; boolean b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); boolean b = (boolean)o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; boolean b = (boolean)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); byte b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; byte b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); byte b = (byte)o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; byte b = (byte)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); short b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; short b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); short b = (short)o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; short b = (short)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); char b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; char b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); char b = (char)o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; char b = (char)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); int b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; int b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); int b = (int)o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; int b = (int)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); long b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; long b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); long b = (long)o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; long b = (long)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); float b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; float b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); float b = (float)o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; float b = (float)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); double b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; double b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); double b = (double)o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; double b = (double)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); Boolean b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; Boolean b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); Boolean b = (Boolean)o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; Boolean b = (Boolean)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); Byte b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; Byte b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); Byte b = (Byte)o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; Byte b = (Byte)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); Short b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; Short b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); Short b = (Short)o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; Short b = (Short)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); Character b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; Character b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); Character b = (Character)o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; Character b = (Character)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); Integer b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; Integer b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); Integer b = (Integer)o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; Integer b = (Integer)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); Long b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; Long b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); Long b = (Long)o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; Long b = (Long)o;"));
+
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); Float b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; Float b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); Float b = (Float)o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = null; Float b = (Float)o;"));
+
+        exec("Double o = Double.valueOf((double)0); Double b = o;");
+        exec("Double o = null; Double b = o;");
+        exec("Double o = Double.valueOf((double)0); Double b = (Double)o;");
+        exec("Double o = null; Double b = (Double)o;");
+
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); ArrayList b = o;"));
+        expectScriptThrows(ClassCastException.class, () -> exec("Double o = Double.valueOf((double)0); ArrayList b = (ArrayList)o;"));
+    }
 }

modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufStreamInput.java

@@ -28,7 +28,7 @@
 import java.io.IOException;
 
 /**
- * A Netty {@link io.netty.buffer.ByteBuf} based {@link org.elasticsearch.common.io.stream.StreamInput}.
+ * A Netty {@link ByteBuf} based {@link StreamInput}.
  */
 class ByteBufStreamInput extends StreamInput {
 
@@ -109,6 +109,39 @@ public int read(byte[] b, int off, int len) throws IOException {
         return len;
     }
 
+    @Override
+    public short readShort() throws IOException {
+        try {
+            return buffer.readShort();
+        } catch (IndexOutOfBoundsException ex) {
+            EOFException eofException = new EOFException();
+            eofException.initCause(ex);
+            throw eofException;
+        }
+    }
+
+    @Override
+    public int readInt() throws IOException {
+        try {
+            return buffer.readInt();
+        } catch (IndexOutOfBoundsException ex) {
+            EOFException eofException = new EOFException();
+            eofException.initCause(ex);
+            throw eofException;
+        }
+    }
+
+    @Override
+    public long readLong() throws IOException {
+        try {
+            return buffer.readLong();
+        } catch (IndexOutOfBoundsException ex) {
+            EOFException eofException = new EOFException();
+            eofException.initCause(ex);
+            throw eofException;
+        }
+    }
+
     @Override
     public void reset() throws IOException {
         buffer.resetReaderIndex();

plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/ByteBufUtils.java

@@ -226,6 +226,39 @@ public int read(byte[] b, int off, int len) throws IOException {
             return len;
         }
 
+        @Override
+        public short readShort() throws IOException {
+            try {
+                return buffer.readShort();
+            } catch (IndexOutOfBoundsException ex) {
+                EOFException eofException = new EOFException();
+                eofException.initCause(ex);
+                throw eofException;
+            }
+        }
+
+        @Override
+        public int readInt() throws IOException {
+            try {
+                return buffer.readInt();
+            } catch (IndexOutOfBoundsException ex) {
+                EOFException eofException = new EOFException();
+                eofException.initCause(ex);
+                throw eofException;
+            }
+        }
+
+        @Override
+        public long readLong() throws IOException {
+            try {
+                return buffer.readLong();
+            } catch (IndexOutOfBoundsException ex) {
+                EOFException eofException = new EOFException();
+                eofException.initCause(ex);
+                throw eofException;
+            }
+        }
+
         @Override
         public void reset() throws IOException {
             buffer.resetReaderIndex();

rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/30_segments.yml

@@ -1,5 +1,9 @@
 ---
 setup:
+  - skip:
+      version: "all"
+      reason: "AwaitsFix: https://github.com/elastic/elasticsearch/issues/40331"
+
   - do:
       indices.create:
           index: test