Merge branch 'master' into ccr

* master:
  ingest: Add ignore_missing property to foreach filter (#22147) (#31578)
  Fix a formatting issue in the docvalue_fields documentation. (#31563)
  reduce log level at gradle configuration time
  [TEST] Close additional clients created while running yaml tests (#31575)
  Docs: Clarify sensitive fields watcher encryption (#31551)
  Watcher: Remove never executed code (#31135)
  Add support for switching distribution for all integration tests (#30874)
  Improve robustness of geo shape parser for malformed shapes (#31449)
  QA: Create xpack yaml features (#31403)
  Improve test times for tests using `RandomObjects::addFields` (#31556)
  [Test] Add full cluster restart test for Rollup (#31533)
  Enhance thread context uniqueness assertion
  [DOCS] Fix heading format errors (#31483)
  fix writeIndex evaluation for aliases (#31562)
  Add x-opaque-id to search slow logs (#31539)
  Watcher: Fix put watch action (#31524)
  Add package pre-install check for java binary (#31343)
  Reduce number of raw types warnings (#31523)
  Migrate scripted metric aggregation scripts to ScriptContext design (#30111)
  turn GetFieldMappingsResponse to ToXContentObject (#31544)
  Close xcontent parsers (partial) (#31513)
  Ingest Attachment: Upgrade Tika to 1.18 (#31252)
  TEST: Correct the assertion arguments order (#31540)

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -71,7 +71,9 @@ public class PluginBuildPlugin extends BuildPlugin {
             if (isModule) {
                 project.integTestCluster.module(project)
                 project.tasks.run.clusterConfig.module(project)
-                project.tasks.run.clusterConfig.distribution = 'integ-test-zip'
+                project.tasks.run.clusterConfig.distribution = System.getProperty(
+                        'run.distribution', 'integ-test-zip'
+                )
             } else {
                 project.integTestCluster.plugin(project.path)
                 project.tasks.run.clusterConfig.plugin(project.path)
@@ -111,7 +113,7 @@ public class PluginBuildPlugin extends BuildPlugin {
     private static void createIntegTestTask(Project project) {
         RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class)
         integTest.mustRunAfter(project.precommit, project.test)
-        project.integTestCluster.distribution = 'integ-test-zip'
+        project.integTestCluster.distribution = System.getProperty('tests.distribution', 'integ-test-zip')
         project.check.dependsOn(integTest)
     }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -88,6 +88,9 @@ class ClusterFormationTasks {
         Configuration currentDistro = project.configurations.create("${prefix}_elasticsearchDistro")
         Configuration bwcDistro = project.configurations.create("${prefix}_elasticsearchBwcDistro")
         Configuration bwcPlugins = project.configurations.create("${prefix}_elasticsearchBwcPlugins")
+        if (System.getProperty('tests.distribution', 'oss-zip') == 'integ-test-zip') {
+            throw new Exception("tests.distribution=integ-test-zip is not supported")
+        }
         configureDistributionDependency(project, config.distribution, currentDistro, VersionProperties.elasticsearch)
         if (config.numBwcNodes > 0) {
             if (config.bwcVersion == null) {
@@ -533,7 +536,8 @@ class ClusterFormationTasks {
 
     static Task configureInstallModuleTask(String name, Project project, Task setup, NodeInfo node, Project module) {
         if (node.config.distribution != 'integ-test-zip') {
-            throw new GradleException("Module ${module.path} not allowed be installed distributions other than integ-test-zip because they should already have all modules bundled!")
+            project.logger.info("Not installing modules for $name, ${node.config.distribution} already has them")
+            return setup
         }
         if (module.plugins.hasPlugin(PluginBuildPlugin) == false) {
             throw new GradleException("Task ${name} cannot include module ${module.path} which is not an esplugin")
@@ -643,6 +647,9 @@ class ClusterFormationTasks {
             BuildPlugin.requireJavaHome(start, node.javaVersion)
         }
         start.doLast(elasticsearchRunner)
+        start.doFirst {
+            project.logger.info("Starting node in ${node.clusterName} distribution: ${node.config.distribution}")
+        }
         return start
     }
 

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/NoopSearchRequestBuilder.java

@@ -329,7 +329,7 @@ public NoopSearchRequestBuilder addSort(String field, SortOrder order) {
      *
      * @see org.elasticsearch.search.sort.SortBuilders
      */
-    public NoopSearchRequestBuilder addSort(SortBuilder sort) {
+    public NoopSearchRequestBuilder addSort(SortBuilder<?> sort) {
         sourceBuilder().sort(sort);
         return this;
     }
@@ -415,7 +415,7 @@ public NoopSearchRequestBuilder setRescorer(RescorerBuilder<?> rescorer) {
      * @param window   rescore window
      * @return this for chaining
      */
-    public NoopSearchRequestBuilder setRescorer(RescorerBuilder rescorer, int window) {
+    public NoopSearchRequestBuilder setRescorer(RescorerBuilder<?> rescorer, int window) {
         sourceBuilder().clearRescorers();
         return addRescorer(rescorer.windowSize(window));
     }

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java

@@ -113,7 +113,7 @@ public void testIndex() throws Exception {
                     .source(jsonMap); // <1>
             //end::index-request-map
             IndexResponse indexResponse = client.index(indexRequest, RequestOptions.DEFAULT);
-            assertEquals(indexResponse.getResult(), DocWriteResponse.Result.CREATED);
+            assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult());
         }
         {
             //tag::index-request-xcontent
@@ -129,7 +129,7 @@ public void testIndex() throws Exception {
                     .source(builder);  // <1>
             //end::index-request-xcontent
             IndexResponse indexResponse = client.index(indexRequest, RequestOptions.DEFAULT);
-            assertEquals(indexResponse.getResult(), DocWriteResponse.Result.UPDATED);
+            assertEquals(DocWriteResponse.Result.UPDATED, indexResponse.getResult());
         }
         {
             //tag::index-request-shortcut
@@ -139,7 +139,7 @@ public void testIndex() throws Exception {
                             "message", "trying out Elasticsearch"); // <1>
             //end::index-request-shortcut
             IndexResponse indexResponse = client.index(indexRequest, RequestOptions.DEFAULT);
-            assertEquals(indexResponse.getResult(), DocWriteResponse.Result.UPDATED);
+            assertEquals(DocWriteResponse.Result.UPDATED, indexResponse.getResult());
         }
         {
             //tag::index-request-string
@@ -158,7 +158,7 @@ public void testIndex() throws Exception {
             // tag::index-execute
             IndexResponse indexResponse = client.index(request, RequestOptions.DEFAULT);
             // end::index-execute
-            assertEquals(indexResponse.getResult(), DocWriteResponse.Result.UPDATED);
+            assertEquals(DocWriteResponse.Result.UPDATED, indexResponse.getResult());
 
             // tag::index-response
             String index = indexResponse.getIndex();
@@ -269,7 +269,7 @@ public void testUpdate() throws Exception {
         {
             IndexRequest indexRequest = new IndexRequest("posts", "doc", "1").source("field", 0);
             IndexResponse indexResponse = client.index(indexRequest, RequestOptions.DEFAULT);
-            assertSame(indexResponse.status(), RestStatus.CREATED);
+            assertSame(RestStatus.CREATED, indexResponse.status());
 
             Request request = new Request("POST", "/_scripts/increment-field");
             request.setJsonEntity(Strings.toString(JsonXContent.contentBuilder()
@@ -280,7 +280,7 @@ public void testUpdate() throws Exception {
                         .endObject()
                     .endObject()));
             Response response = client().performRequest(request);
-            assertEquals(response.getStatusLine().getStatusCode(), RestStatus.OK.getStatus());
+            assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode());
         }
         {
             //tag::update-request
@@ -298,7 +298,7 @@ public void testUpdate() throws Exception {
             request.script(inline);  // <3>
             //end::update-request-with-inline-script
             UpdateResponse updateResponse = client.update(request, RequestOptions.DEFAULT);
-            assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED);
+            assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
             assertEquals(4, updateResponse.getGetResult().getSource().get("field"));
 
             request = new UpdateRequest("posts", "doc", "1").fetchSource(true);
@@ -308,7 +308,7 @@ public void testUpdate() throws Exception {
             request.script(stored);  // <2>
             //end::update-request-with-stored-script
             updateResponse = client.update(request, RequestOptions.DEFAULT);
-            assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED);
+            assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
             assertEquals(8, updateResponse.getGetResult().getSource().get("field"));
         }
         {
@@ -320,7 +320,7 @@ public void testUpdate() throws Exception {
                     .doc(jsonMap); // <1>
             //end::update-request-with-doc-as-map
             UpdateResponse updateResponse = client.update(request, RequestOptions.DEFAULT);
-            assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED);
+            assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
         }
         {
             //tag::update-request-with-doc-as-xcontent
@@ -335,7 +335,7 @@ public void testUpdate() throws Exception {
                     .doc(builder);  // <1>
             //end::update-request-with-doc-as-xcontent
             UpdateResponse updateResponse = client.update(request, RequestOptions.DEFAULT);
-            assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED);
+            assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
         }
         {
             //tag::update-request-shortcut
@@ -344,7 +344,7 @@ public void testUpdate() throws Exception {
                          "reason", "daily update"); // <1>
             //end::update-request-shortcut
             UpdateResponse updateResponse = client.update(request, RequestOptions.DEFAULT);
-            assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED);
+            assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
         }
         {
             //tag::update-request-with-doc-as-string
@@ -359,7 +359,7 @@ public void testUpdate() throws Exception {
             // tag::update-execute
             UpdateResponse updateResponse = client.update(request, RequestOptions.DEFAULT);
             // end::update-execute
-            assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED);
+            assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
 
             // tag::update-response
             String index = updateResponse.getIndex();
@@ -434,7 +434,7 @@ public void testUpdate() throws Exception {
             request.fetchSource(true); // <1>
             //end::update-request-no-source
             UpdateResponse updateResponse = client.update(request, RequestOptions.DEFAULT);
-            assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED);
+            assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
             assertNotNull(updateResponse.getGetResult());
             assertEquals(3, updateResponse.getGetResult().sourceAsMap().size());
         }
@@ -446,7 +446,7 @@ public void testUpdate() throws Exception {
             request.fetchSource(new FetchSourceContext(true, includes, excludes)); // <1>
             //end::update-request-source-include
             UpdateResponse updateResponse = client.update(request, RequestOptions.DEFAULT);
-            assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED);
+            assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
             Map<String, Object> sourceAsMap = updateResponse.getGetResult().sourceAsMap();
             assertEquals(2, sourceAsMap.size());
             assertEquals("source includes", sourceAsMap.get("reason"));
@@ -460,7 +460,7 @@ public void testUpdate() throws Exception {
             request.fetchSource(new FetchSourceContext(true, includes, excludes)); // <1>
             //end::update-request-source-exclude
             UpdateResponse updateResponse = client.update(request, RequestOptions.DEFAULT);
-            assertEquals(updateResponse.getResult(), DocWriteResponse.Result.UPDATED);
+            assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
             Map<String, Object> sourceAsMap = updateResponse.getGetResult().sourceAsMap();
             assertEquals(2, sourceAsMap.size());
             assertEquals("source excludes", sourceAsMap.get("reason"));
@@ -538,7 +538,7 @@ public void testDelete() throws Exception {
         {
             IndexRequest indexRequest = new IndexRequest("posts", "doc", "1").source("field", "value");
             IndexResponse indexResponse = client.index(indexRequest, RequestOptions.DEFAULT);
-            assertSame(indexResponse.status(), RestStatus.CREATED);
+            assertSame(RestStatus.CREATED, indexResponse.status());
         }
 
         {
@@ -552,7 +552,7 @@ public void testDelete() throws Exception {
             // tag::delete-execute
             DeleteResponse deleteResponse = client.delete(request, RequestOptions.DEFAULT);
             // end::delete-execute
-            assertSame(deleteResponse.getResult(), DocWriteResponse.Result.DELETED);
+            assertSame(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
 
             // tag::delete-response
             String index = deleteResponse.getIndex();
@@ -605,7 +605,7 @@ public void testDelete() throws Exception {
         {
             IndexResponse indexResponse = client.index(new IndexRequest("posts", "doc", "1").source("field", "value")
                     , RequestOptions.DEFAULT);
-            assertSame(indexResponse.status(), RestStatus.CREATED);
+            assertSame(RestStatus.CREATED, indexResponse.status());
 
             // tag::delete-conflict
             try {
@@ -621,7 +621,7 @@ public void testDelete() throws Exception {
         {
             IndexResponse indexResponse = client.index(new IndexRequest("posts", "doc", "async").source("field", "value"),
                     RequestOptions.DEFAULT);
-            assertSame(indexResponse.status(), RestStatus.CREATED);
+            assertSame(RestStatus.CREATED, indexResponse.status());
 
             DeleteRequest request = new DeleteRequest("posts", "doc", "async");
 
@@ -666,7 +666,7 @@ public void testBulk() throws Exception {
             // tag::bulk-execute
             BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
             // end::bulk-execute
-            assertSame(bulkResponse.status(), RestStatus.OK);
+            assertSame(RestStatus.OK, bulkResponse.status());
             assertFalse(bulkResponse.hasFailures());
         }
         {
@@ -679,7 +679,7 @@ public void testBulk() throws Exception {
                     .source(XContentType.JSON,"field", "baz"));
             // end::bulk-request-with-mixed-operations
             BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
-            assertSame(bulkResponse.status(), RestStatus.OK);
+            assertSame(RestStatus.OK, bulkResponse.status());
             assertFalse(bulkResponse.hasFailures());
 
             // tag::bulk-response
@@ -778,7 +778,7 @@ public void testGet() throws Exception {
                             "postDate", new Date(),
                             "message", "trying out Elasticsearch");
             IndexResponse indexResponse = client.index(indexRequest, RequestOptions.DEFAULT);
-            assertEquals(indexResponse.getResult(), DocWriteResponse.Result.CREATED);
+            assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult());
         }
         {
             //tag::get-request

client/rest/src/main/java/org/elasticsearch/client/HasAttributeNodeSelector.java

@@ -22,6 +22,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  * A {@link NodeSelector} that selects nodes that have a particular value
@@ -49,6 +50,24 @@ public void select(Iterable<Node> nodes) {
         }
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        HasAttributeNodeSelector that = (HasAttributeNodeSelector) o;
+        return Objects.equals(key, that.key) &&
+                Objects.equals(value, that.value);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(key, value);
+    }
+
     @Override
     public String toString() {
         return key + "=" + value;

distribution/packages/src/common/scripts/preinst

@@ -9,6 +9,18 @@
 #       $1=1       : indicates an new install
 #       $1=2       : indicates an upgrade
 
+# Check for these at preinst time due to failures in postinst if they do not exist
+if [ -x "$JAVA_HOME/bin/java" ]; then
+    JAVA="$JAVA_HOME/bin/java"
+else
+    JAVA=`which java`
+fi
+
+if [ -z "$JAVA" ]; then
+    echo "could not find java; set JAVA_HOME or ensure java is in PATH"
+    exit 1
+fi
+
 case "$1" in
 
     # Debian ####################################################

docs/java-api/aggregations/metrics/scripted-metric-aggregation.asciidoc

@@ -13,8 +13,8 @@ Here is an example on how to create the aggregation request:
 --------------------------------------------------
 ScriptedMetricAggregationBuilder aggregation = AggregationBuilders
     .scriptedMetric("agg")
-    .initScript(new Script("params._agg.heights = []"))
-    .mapScript(new Script("params._agg.heights.add(doc.gender.value == 'male' ? doc.height.value : -1.0 * doc.height.value)"));
+    .initScript(new Script("state.heights = []"))
+    .mapScript(new Script("state.heights.add(doc.gender.value == 'male' ? doc.height.value : -1.0 * doc.height.value)"));
 --------------------------------------------------
 
 You can also specify a `combine` script which will be executed on each shard:
@@ -23,9 +23,9 @@ You can also specify a `combine` script which will be executed on each shard:
 --------------------------------------------------
 ScriptedMetricAggregationBuilder aggregation = AggregationBuilders
     .scriptedMetric("agg")
-    .initScript(new Script("params._agg.heights = []"))
-    .mapScript(new Script("params._agg.heights.add(doc.gender.value == 'male' ? doc.height.value : -1.0 * doc.height.value)"))
-    .combineScript(new Script("double heights_sum = 0.0; for (t in params._agg.heights) { heights_sum += t } return heights_sum"));
+    .initScript(new Script("state.heights = []"))
+    .mapScript(new Script("state.heights.add(doc.gender.value == 'male' ? doc.height.value : -1.0 * doc.height.value)"))
+    .combineScript(new Script("double heights_sum = 0.0; for (t in state.heights) { heights_sum += t } return heights_sum"));
 --------------------------------------------------
 
 You can also specify a `reduce` script which will be executed on the node which gets the request:
@@ -34,10 +34,10 @@ You can also specify a `reduce` script which will be executed on the node which
 --------------------------------------------------
 ScriptedMetricAggregationBuilder aggregation = AggregationBuilders
     .scriptedMetric("agg")
-    .initScript(new Script("params._agg.heights = []"))
-    .mapScript(new Script("params._agg.heights.add(doc.gender.value == 'male' ? doc.height.value : -1.0 * doc.height.value)"))
-    .combineScript(new Script("double heights_sum = 0.0; for (t in params._agg.heights) { heights_sum += t } return heights_sum"))
-    .reduceScript(new Script("double heights_sum = 0.0; for (a in params._aggs) { heights_sum += a } return heights_sum"));
+    .initScript(new Script("state.heights = []"))
+    .mapScript(new Script("state.heights.add(doc.gender.value == 'male' ? doc.height.value : -1.0 * doc.height.value)"))
+    .combineScript(new Script("double heights_sum = 0.0; for (t in state.heights) { heights_sum += t } return heights_sum"))
+    .reduceScript(new Script("double heights_sum = 0.0; for (a in states) { heights_sum += a } return heights_sum"));
 --------------------------------------------------
 
 

docs/reference/aggregations/bucket/composite-aggregation.asciidoc

@@ -224,7 +224,7 @@ Time values can also be specified via abbreviations supported by <<time-units,ti
 Note that fractional time values are not supported, but you can address this by shifting to another
 time unit (e.g., `1.5h` could instead be specified as `90m`).
 
-====== Format
+*Format*
 
 Internally, a date is represented as a 64 bit number representing a timestamp in milliseconds-since-the-epoch.
 These timestamps are returned as the bucket keys. It is possible to return a formatted date string instead using
@@ -257,7 +257,7 @@ GET /_search
 
 <1> Supports expressive date <<date-format-pattern,format pattern>>
 
-====== Time Zone
+*Time Zone*
 
 Date-times are stored in Elasticsearch in UTC.  By default, all bucketing and
 rounding is also done in UTC. The `time_zone` parameter can be used to indicate