Merge branch 'upstream/master' into tsdb-required-range-settings

* upstream/master: (55 commits)
  Fix ComposableIndexTemplate equals when composed_of is null (elastic#80864)
  Optimize DLS bitset building for matchAll query (elastic#81030)
  URL option for BaseRunAsSuperuserCommand (elastic#81025)
  Less Verbose Serialization of Snapshot Failure in SLM Metadata (elastic#80942)
  Fix shadowed vars pt7 (elastic#80996)
  Fail shards early when we can detect a type missmatch (elastic#79869)
  Delegate Ref Counting to ByteBuf in Netty Transport (elastic#81096)
  Clarify `unassigned.reason` docs (elastic#81017)
  Strip blocks from settings for reindex targets (elastic#80887)
  Split off the values supplier for ScriptDocValues (elastic#80635)
  [ML] Switch message and detail for model snapshot deprecations (elastic#81108)
  [DOCS] Update xrefs for snapshot restore docs (elastic#81023)
  [ML] Updates visiblity of validate API (elastic#81061)
  Track histogram of transport handling times (elastic#80581)
  [ML] Fix datafeed preview with remote indices (elastic#81099)
  [ML] Fix acceptable model snapshot versions in ML deprecation checker (elastic#81060)
  [ML] Add logging for failing PyTorch test (elastic#81044)
  Extending the timeout waiting for snapshot to be ready (elastic#81018)
  [ML] Fix incorrect logging of unexpected model size error (elastic#81089)
  [ML] Make inference timeout test more reliable (elastic#81094)
  ...

build-conventions/src/main/java/org/elasticsearch/gradle/internal/checkstyle/HiddenFieldCheck.java

@@ -71,9 +71,15 @@ public class HiddenFieldCheck extends AbstractCheck {
     /** Control whether to ignore constructor parameters. */
     private boolean ignoreConstructorParameter;
 
+    /** Control whether to ignore variables in constructor bodies. */
+    private boolean ignoreConstructorBody;
+
     /** Control whether to ignore parameters of abstract methods. */
     private boolean ignoreAbstractMethods;
 
+    /** If set, specifies a regex of method names that should be ignored */
+    private String ignoredMethodNames;
+
     @Override
     public int[] getDefaultTokens() {
         return getAcceptableTokens();
@@ -224,7 +230,8 @@ private void processVariable(DetailAST ast) {
 
             if ((frame.containsStaticField(name) || isInstanceField(ast, name))
                 && isMatchingRegexp(name) == false
-                && isIgnoredParam(ast, name) == false) {
+                && isIgnoredParam(ast, name) == false
+                && isIgnoredVariable(ast, name) == false) {
                 log(nameAST, MSG_KEY, name);
             }
         }
@@ -238,7 +245,14 @@ && isIgnoredParam(ast, name) == false) {
      * @return true if parameter is ignored.
      */
     private boolean isIgnoredParam(DetailAST ast, String name) {
-        return isIgnoredSetterParam(ast, name) || isIgnoredConstructorParam(ast) || isIgnoredParamOfAbstractMethod(ast);
+        return isVariableInIgnoredMethod(ast, name)
+            || isIgnoredSetterParam(ast, name)
+            || isIgnoredConstructorParam(ast)
+            || isIgnoredParamOfAbstractMethod(ast);
+    }
+
+    private boolean isIgnoredVariable(DetailAST ast, String name) {
+        return isIgnoredVariableInConstructorBody(ast, name);
     }
 
     /**
@@ -410,6 +424,42 @@ private boolean isIgnoredParamOfAbstractMethod(DetailAST ast) {
         return result;
     }
 
+    /**
+     * Decides whether to ignore an AST node that is the parameter of a method whose
+     * name matches the {@link #ignoredMethodNames} regex, if set.
+     * @param ast the AST to check
+     * @return true is the ast should be ignored because the parameter belongs to a
+     *      method whose name matches the regex.
+     */
+    private boolean isVariableInIgnoredMethod(DetailAST ast, String name) {
+        boolean result = false;
+        if (ignoredMethodNames != null && (ast.getType() == TokenTypes.PARAMETER_DEF || ast.getType() == TokenTypes.VARIABLE_DEF)) {
+            DetailAST method = ast.getParent();
+            while (method != null && method.getType() != TokenTypes.METHOD_DEF) {
+                method = method.getParent();
+            }
+            if (method != null && method.getType() == TokenTypes.METHOD_DEF) {
+                final String methodName = method.findFirstToken(TokenTypes.IDENT).getText();
+                result = methodName.matches(ignoredMethodNames);
+            }
+        }
+        return result;
+    }
+
+    private boolean isIgnoredVariableInConstructorBody(DetailAST ast, String name) {
+        boolean result = false;
+
+        if (ignoreConstructorBody && ast.getType() == TokenTypes.VARIABLE_DEF) {
+            DetailAST method = ast.getParent();
+            while (method != null && method.getType() != TokenTypes.CTOR_DEF) {
+                method = method.getParent();
+            }
+            result = method != null && method.getType() == TokenTypes.CTOR_DEF;
+        }
+
+        return result;
+    }
+
     /**
      * Setter to define the RegExp for names of variables and parameters to ignore.
      *
@@ -463,6 +513,14 @@ public void setIgnoreAbstractMethods(boolean ignoreAbstractMethods) {
         this.ignoreAbstractMethods = ignoreAbstractMethods;
     }
 
+    public void setIgnoredMethodNames(String ignoredMethodNames) {
+        this.ignoredMethodNames = ignoredMethodNames;
+    }
+
+    public void setIgnoreConstructorBody(boolean ignoreConstructorBody) {
+        this.ignoreConstructorBody = ignoreConstructorBody;
+    }
+
     /**
      * Holds the names of static and instance fields of a type.
      */

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -97,6 +97,7 @@ public void execute(Task t) {
                 "-Xmx" + System.getProperty("tests.heap.size", "512m"),
                 "-Xms" + System.getProperty("tests.heap.size", "512m"),
                 "--illegal-access=deny",
+                "-Djava.security.manager=allow",
                 // TODO: only open these for mockito when it is modularized
                 "--add-opens=java.base/java.security.cert=ALL-UNNAMED",
                 "--add-opens=java.base/java.nio.channels=ALL-UNNAMED",

build-tools-internal/src/main/resources/checkstyle.xml

@@ -111,10 +111,12 @@
     <!--
     <module name="org.elasticsearch.gradle.internal.checkstyle.HiddenFieldCheck">
         <property name="ignoreConstructorParameter" value="true" />
+        <property name="ignoreConstructorBody" value="true"/>
         <property name="ignoreSetter" value="true" />
         <property name="setterCanReturnItsClass" value="true"/>
-        <property name="ignoreFormat" value="^(threadPool)$"/>
+        <property name="ignoreFormat" value="^(?:threadPool)$"/>
         <property name="ignoreAbstractMethods" value="true"/>
+        <property name="ignoredMethodNames" value="^(?:createParser)$"/>
         <message key="hidden.field" value="''{0}'' hides a field." />
     </module>
     -->

build-tools/src/main/java/org/elasticsearch/gradle/Version.java

@@ -39,7 +39,9 @@ public enum Mode {
 
     private static final Pattern pattern = Pattern.compile("(\\d+)\\.(\\d+)\\.(\\d+)(?:-(alpha\\d+|beta\\d+|rc\\d+|SNAPSHOT))?");
 
-    private static final Pattern relaxedPattern = Pattern.compile("v?(\\d+)\\.(\\d+)\\.(\\d+)(?:-([a-zA-Z0-9_]+(?:-[a-zA-Z0-9]+)*))?");
+    private static final Pattern relaxedPattern = Pattern.compile(
+        "v?(\\d+)\\.(\\d+)\\.(\\d+)(?:[\\-+]+([a-zA-Z0-9_]+(?:-[a-zA-Z0-9]+)*))?"
+    );
 
     public Version(int major, int minor, int revision) {
         this(major, minor, revision, null);

build-tools/src/test/java/org/elasticsearch/gradle/VersionTests.java

@@ -41,6 +41,7 @@ public void testRelaxedVersionParsing() {
         assertVersionEquals("6.1.2-foo", 6, 1, 2, Version.Mode.RELAXED);
         assertVersionEquals("6.1.2-foo-bar", 6, 1, 2, Version.Mode.RELAXED);
         assertVersionEquals("16.01.22", 16, 1, 22, Version.Mode.RELAXED);
+        assertVersionEquals("20.10.10+dfsg1", 20, 10, 10, Version.Mode.RELAXED);
     }
 
     public void testCompareWithStringVersions() {

client/rest-high-level/src/main/java/org/elasticsearch/client/TasksRequestConverters.java

@@ -34,15 +34,15 @@ static Request cancelTasks(CancelTasksRequest req) {
     }
 
     static Request listTasks(ListTasksRequest listTaskRequest) {
-        if (listTaskRequest.getTaskId() != null && listTaskRequest.getTaskId().isSet()) {
-            throw new IllegalArgumentException("TaskId cannot be used for list tasks request");
+        if (listTaskRequest.getTargetTaskId() != null && listTaskRequest.getTargetTaskId().isSet()) {
+            throw new IllegalArgumentException("TargetTaskId cannot be used for list tasks request");
         }
         Request request = new Request(HttpGet.METHOD_NAME, "/_tasks");
         RequestConverters.Params params = new RequestConverters.Params();
         params.withTimeout(listTaskRequest.getTimeout())
             .withDetailed(listTaskRequest.getDetailed())
             .withWaitForCompletion(listTaskRequest.getWaitForCompletion())
-            .withParentTaskId(listTaskRequest.getParentTaskId())
+            .withParentTaskId(listTaskRequest.getTargetParentTaskId())
             .withNodes(listTaskRequest.getNodes())
             .withActions(listTaskRequest.getActions())
             .putParam("group_by", "none");

client/rest-high-level/src/test/java/org/elasticsearch/client/TasksRequestConvertersTests.java

@@ -72,7 +72,7 @@ public void testListTasks() {
             if (randomBoolean()) {
                 if (randomBoolean()) {
                     TaskId taskId = new TaskId(randomAlphaOfLength(5), randomNonNegativeLong());
-                    request.setParentTaskId(taskId);
+                    request.setTargetParentTaskId(taskId);
                     expectedParams.put("parent_task_id", taskId.toString());
                 } else {
                     request.setParentTask(TaskId.EMPTY_TASK_ID);
@@ -102,12 +102,12 @@ public void testListTasks() {
         }
         {
             ListTasksRequest request = new ListTasksRequest();
-            request.setTaskId(new TaskId(randomAlphaOfLength(5), randomNonNegativeLong()));
+            request.setTargetTaskId(new TaskId(randomAlphaOfLength(5), randomNonNegativeLong()));
             IllegalArgumentException exception = expectThrows(
                 IllegalArgumentException.class,
                 () -> TasksRequestConverters.listTasks(request)
             );
-            assertEquals("TaskId cannot be used for list tasks request", exception.getMessage());
+            assertEquals("TargetTaskId cannot be used for list tasks request", exception.getMessage());
         }
     }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/TasksClientDocumentationIT.java

@@ -67,7 +67,7 @@ public void testListTasks() throws IOException {
             // tag::list-tasks-request-filter
             request.setActions("cluster:*"); // <1>
             request.setNodes("nodeId1", "nodeId2"); // <2>
-            request.setParentTaskId(new TaskId("parentTaskId", 42)); // <3>
+            request.setTargetParentTaskId(new TaskId("parentTaskId", 42)); // <3>
             // end::list-tasks-request-filter
 
             // tag::list-tasks-request-detailed

distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/BaseKeyStoreCommand.java

@@ -12,9 +12,9 @@
 import joptsimple.OptionSpec;
 
 import org.elasticsearch.cli.ExitCodes;
-import org.elasticsearch.cli.KeyStoreAwareCommand;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
+import org.elasticsearch.common.cli.KeyStoreAwareCommand;
 import org.elasticsearch.common.settings.KeyStoreWrapper;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.env.Environment;

distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/CreateKeyStoreCommand.java

@@ -12,9 +12,9 @@
 import joptsimple.OptionSpec;
 
 import org.elasticsearch.cli.ExitCodes;
-import org.elasticsearch.cli.KeyStoreAwareCommand;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
+import org.elasticsearch.common.cli.KeyStoreAwareCommand;
 import org.elasticsearch.common.settings.KeyStoreWrapper;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.env.Environment;

distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/HasPasswordKeyStoreCommand.java

@@ -10,9 +10,9 @@
 
 import joptsimple.OptionSet;
 
-import org.elasticsearch.cli.KeyStoreAwareCommand;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
+import org.elasticsearch.common.cli.KeyStoreAwareCommand;
 import org.elasticsearch.common.settings.KeyStoreWrapper;
 import org.elasticsearch.env.Environment;
 

distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/KeyStoreCli.java

@@ -8,8 +8,8 @@
 
 package org.elasticsearch.cli.keystore;
 
-import org.elasticsearch.cli.LoggingAwareMultiCommand;
 import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.common.cli.LoggingAwareMultiCommand;
 
 /**
  * A cli tool for managing secrets in the elasticsearch keystore.

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/SystemJvmOptions.java

@@ -27,6 +27,8 @@ static List<String> systemJvmOptions() {
              * networkaddress.cache.negative ttl; set to -1 to cache forever.
              */
             "-Des.networkaddress.cache.negative.ttl=10",
+            // Allow to set the security manager.
+            "-Djava.security.manager=allow",
             // pre-touch JVM emory pages during initialization
             "-XX:+AlwaysPreTouch",
             // explicitly set the stack size

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginCommand.java

@@ -11,8 +11,8 @@
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
 
-import org.elasticsearch.cli.EnvironmentAwareCommand;
 import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.common.cli.EnvironmentAwareCommand;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.PluginInfo;
 

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ListPluginsCommand.java

@@ -11,8 +11,8 @@
 import joptsimple.OptionSet;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.cli.EnvironmentAwareCommand;
 import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.common.cli.EnvironmentAwareCommand;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.PluginInfo;
 

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginCli.java

@@ -9,8 +9,8 @@
 package org.elasticsearch.plugins.cli;
 
 import org.elasticsearch.cli.Command;
-import org.elasticsearch.cli.LoggingAwareMultiCommand;
 import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.common.cli.LoggingAwareMultiCommand;
 import org.elasticsearch.core.internal.io.IOUtils;
 
 import java.io.IOException;

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginCommand.java

@@ -11,8 +11,8 @@
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
 
-import org.elasticsearch.cli.EnvironmentAwareCommand;
 import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.common.cli.EnvironmentAwareCommand;
 import org.elasticsearch.env.Environment;
 
 import java.util.Arrays;

docs/reference/cat/shards.asciidoc

@@ -249,7 +249,9 @@ Time at which the shard became unassigned in
 Time (UTC)].
 
 `unassigned.details`, `ud`::
-Details about why the shard became unassigned.
+Details about why the shard became unassigned. This does not explain why the
+shard is currently unassigned. To understand why a shard is not assigned, use
+the <<cluster-allocation-explain>> API.
 
 `unassigned.for`, `uf`::
 Time at which the shard was requested to be unassigned in
@@ -258,16 +260,24 @@ Time (UTC)].
 
 [[reason-unassigned]]
 `unassigned.reason`, `ur`::
-Reason the shard is unassigned. Returned values are:
+Indicates the reason for the last change to the state of this unassigned shard.
+This does not explain why the shard is currently unassigned. To understand why
+a shard is not assigned, use the <<cluster-allocation-explain>> API. Returned
+values include:
 +
 * `ALLOCATION_FAILED`: Unassigned as a result of a failed allocation of the shard.
 * `CLUSTER_RECOVERED`: Unassigned as a result of a full cluster recovery.
 * `DANGLING_INDEX_IMPORTED`: Unassigned as a result of importing a dangling index.
 * `EXISTING_INDEX_RESTORED`: Unassigned as a result of restoring into a closed index.
+* `FORCED_EMPTY_PRIMARY`: The shard's allocation was last modified by forcing an empty primary using the <<cluster-reroute>> API.
+* `INDEX_CLOSED`: Unassigned because the index was closed.
 * `INDEX_CREATED`: Unassigned as a result of an API creation of an index.
 * `INDEX_REOPENED`: Unassigned as a result of opening a closed index.
+* `MANUAL_ALLOCATION`: The shard's allocation was last modified by the <<cluster-reroute>> API.
 * `NEW_INDEX_RESTORED`: Unassigned as a result of restoring into a new index.
 * `NODE_LEFT`: Unassigned as a result of the node hosting it leaving the cluster.
+* `NODE_RESTARTING`: Similar to `NODE_LEFT`, except that the node was registered as restarting using the <<node-lifecycle-api,Node shutdown API>>. 
+* `PRIMARY_FAILED`: The shard was initializing as a replica, but the primary shard failed before the initialization completed.
 * `REALLOCATED_REPLICA`: A better replica location is identified and causes the existing replica allocation to be cancelled.
 * `REINITIALIZED`: When a shard moves from started back to initializing.
 * `REPLICA_ADDED`: Unassigned as a result of explicit addition of a replica.

docs/reference/cluster/nodes-stats.asciidoc

@@ -1899,6 +1899,54 @@ Size of TX packets sent by the node during internal cluster communication.
 (integer)
 Size, in bytes, of TX packets sent by the node during internal cluster
 communication.
+
+`inbound_handling_time_histogram`::
+(array)
+The distribution of the time spent handling each inbound message on a transport
+thread, represented as a histogram.
++
+.Properties of `inbound_handling_time_histogram`
+[%collapsible]
+=======
+`ge_millis`::
+(integer)
+The inclusive lower bound of the bucket in milliseconds. Omitted on the first
+bucket since this bucket has no lower bound.
+
+`lt_millis`::
+(integer)
+The exclusive upper bound of the bucket in milliseconds. Omitted on the last
+bucket since this bucket has no upper bound.
+
+`count`::
+(integer)
+The number of times a transport thread took a period of time within the bounds
+of this bucket to handle an inbound message.
+=======
+
+`outbound_handling_time_histogram`::
+(array)
+The distribution of the time spent sending each outbound transport message on a
+transport thread, represented as a histogram.
++
+.Properties of `outbound_handling_time_histogram`
+[%collapsible]
+=======
+`ge_millis`::
+(integer)
+The inclusive lower bound of the bucket in milliseconds. Omitted on the first
+bucket since this bucket has no lower bound.
+
+`lt_millis`::
+(integer)
+The exclusive upper bound of the bucket in milliseconds. Omitted on the last
+bucket since this bucket has no upper bound.
+
+`count`::
+(integer)
+The number of times a transport thread took a period of time within the bounds
+of this bucket to send a transport message.
+=======
 ======
 
 [[cluster-nodes-stats-api-response-body-http]]