Merge branch 'master' into ccr-overview

* master: (22 commits)
  Introduce CCR getting started guide (elastic#35434)
  Correct typo in BuildPlugin exception message (elastic#35458)
  Correct implemented interface of ParsedReverseNested (elastic#35455)
  [ML] Fix find_file_structure NPE with should_trim_fields (elastic#35465)
  Handle OS pretty name on old OS without OS release (elastic#35453)
  Register remote cluster compress setting (elastic#35464)
  [DOCS] Clarify results_index_name description (elastic#35463)
  [TEST] add missing doc tests for HLRC GetRollupIndexCaps API
  [HLRC] Add GetRollupIndexCaps API (elastic#35102)
  Geo: enables coerce support in WKT polygon parser (elastic#35414)
  [ILM] fix retry so it picks up latest policy and executes async action (elastic#35406)
  Address handling of OS pretty name on some OS (elastic#35451)
  HLRC support for getTask (elastic#35166)
  upgrade to lucene-8.0.0-snapshot-6d9c714052 (elastic#35428)
  Add docs for CCR stats API (elastic#35439)
  Fix the names of CCR stats endpoints in usage API (elastic#35438)
  Switch to default to no build qualifier (elastic#35415)
  Clarify S3 repository storage class parameter (elastic#35400)
  SQL: Fix query translation for scripted queries (elastic#35408)
  [TEST] Instead of ignoring the ccr downgrade to basic license qa test avoid the assertions that check the log files, because that does not work on Windows. The rest of the test is still useful and should work on Windows CI.
  ...

buildSrc/build.gradle

@@ -245,7 +245,7 @@ class VersionPropertiesLoader {
                       elasticsearch
       )
     }
-    String qualifier = systemProperties.getProperty("build.version_qualifier", "alpha1");
+    String qualifier = systemProperties.getProperty("build.version_qualifier", "");
     if (qualifier.isEmpty() == false) {
       if (qualifier.matches("(alpha|beta|rc)\\d+") == false) {
         throw new IllegalStateException("Invalid qualifier: " + qualifier)

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -66,7 +66,7 @@ class BuildPlugin implements Plugin<Project> {
     void apply(Project project) {
         if (project.pluginManager.hasPlugin('elasticsearch.standalone-rest-test')) {
               throw new InvalidUserDataException('elasticsearch.standalone-test, '
-                + 'elasticearch.standalone-rest-test, and elasticsearch.build '
+                + 'elasticsearch.standalone-rest-test, and elasticsearch.build '
                 + 'are mutually exclusive')
         }
         final String minimumGradleVersion

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 7.0.0
-lucene            = 8.0.0-snapshot-31d7dfe6b1
+lucene            = 8.0.0-snapshot-6d9c714052
 
 # optional dependencies
 spatial4j         = 0.7
@@ -16,6 +16,7 @@ slf4j             = 1.6.2
 jna               = 4.5.1
 
 netty             = 4.1.30.Final
+joda              = 2.10.1
 
 # test dependencies
 randomizedrunner  = 2.7.0

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -1417,6 +1417,38 @@ private <Req, Resp> Resp internalPerformRequest(Req request,
             throw new IOException("Unable to parse response body for " + response, e);
         }
     }
+
+    /**
+     * Defines a helper method for requests that can 404 and in which case will return an empty Optional 
+     * otherwise tries to parse the response body
+     */
+    protected final <Req extends Validatable, Resp> Optional<Resp> performRequestAndParseOptionalEntity(Req request,
+                                                                  CheckedFunction<Req, Request, IOException> requestConverter,
+                                                                  RequestOptions options,
+                                                                  CheckedFunction<XContentParser, Resp, IOException> entityParser
+                                                                  ) throws IOException {
+        Optional<ValidationException> validationException = request.validate();
+        if (validationException != null && validationException.isPresent()) {
+            throw validationException.get();
+        }
+        Request req = requestConverter.apply(request);
+        req.setOptions(options);
+        Response response;
+        try {
+            response = client.performRequest(req);
+        } catch (ResponseException e) {
+            if (RestStatus.NOT_FOUND.getStatus() == e.getResponse().getStatusLine().getStatusCode()) {
+                return Optional.empty();
+            }
+            throw parseResponseException(e);
+        }
+
+        try {
+            return Optional.of(parseEntity(response.getEntity(), entityParser));
+        } catch (Exception e) {
+            throw new IOException("Unable to parse response body for " + response, e);
+        }
+    }      
 
     /**
      * @deprecated If creating a new HLRC ReST API call, consider creating new actions instead of reusing server actions. The Validation
@@ -1538,6 +1570,62 @@ public void onFailure(Exception exception) {
             }
         };
     }
+
+    /**
+     * Async request which returns empty Optionals in the case of 404s or parses entity into an Optional
+     */
+    protected final <Req extends Validatable, Resp> void performRequestAsyncAndParseOptionalEntity(Req request,
+            CheckedFunction<Req, Request, IOException> requestConverter,
+            RequestOptions options,
+            CheckedFunction<XContentParser, Resp, IOException> entityParser,
+            ActionListener<Optional<Resp>> listener) {
+        Optional<ValidationException> validationException = request.validate();
+        if (validationException != null && validationException.isPresent()) {
+            listener.onFailure(validationException.get());
+            return;
+        }
+        Request req;
+        try {
+            req = requestConverter.apply(request);
+        } catch (Exception e) {
+            listener.onFailure(e);
+            return;
+        }
+        req.setOptions(options);
+        ResponseListener responseListener = wrapResponseListener404sOptional(response -> parseEntity(response.getEntity(), 
+                entityParser), listener);
+        client.performRequestAsync(req, responseListener);        
+    }    
+
+    final <Resp> ResponseListener wrapResponseListener404sOptional(CheckedFunction<Response, Resp, IOException> responseConverter,
+            ActionListener<Optional<Resp>> actionListener) {
+        return new ResponseListener() {
+            @Override
+            public void onSuccess(Response response) {
+                try {
+                    actionListener.onResponse(Optional.of(responseConverter.apply(response)));
+                } catch (Exception e) {
+                    IOException ioe = new IOException("Unable to parse response body for " + response, e);
+                    onFailure(ioe);
+                }
+            }
+
+            @Override
+            public void onFailure(Exception exception) {
+                if (exception instanceof ResponseException) {
+                    ResponseException responseException = (ResponseException) exception;
+                    Response response = responseException.getResponse();
+                    if (RestStatus.NOT_FOUND.getStatus() == response.getStatusLine().getStatusCode()) {
+                            actionListener.onResponse(Optional.empty());
+                    } else {
+                        actionListener.onFailure(parseResponseException(responseException));
+                    }
+                } else {
+                    actionListener.onFailure(exception);
+                }
+            }
+        };
+    }    
 
     /**
      * Converts a {@link ResponseException} obtained from the low level REST client into an {@link ElasticsearchException}.

client/rest-high-level/src/main/java/org/elasticsearch/client/RollupClient.java

@@ -22,6 +22,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.rollup.DeleteRollupJobRequest;
 import org.elasticsearch.client.rollup.DeleteRollupJobResponse;
+import org.elasticsearch.client.rollup.GetRollupIndexCapsRequest;
+import org.elasticsearch.client.rollup.GetRollupIndexCapsResponse;
 import org.elasticsearch.client.rollup.GetRollupJobRequest;
 import org.elasticsearch.client.rollup.GetRollupJobResponse;
 import org.elasticsearch.client.rollup.GetRollupCapsRequest;
@@ -219,4 +221,40 @@ public void getRollupCapabilitiesAsync(GetRollupCapsRequest request, RequestOpti
             listener,
             Collections.emptySet());
     }
+
+    /**
+     * Get the Rollup Index Capabilities of a rollup index or pattern
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/master/rollup-get-rollup-index-caps.html">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public GetRollupIndexCapsResponse getRollupIndexCapabilities(GetRollupIndexCapsRequest request,
+                                                                 RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+            RollupRequestConverters::getRollupIndexCaps,
+            options,
+            GetRollupIndexCapsResponse::fromXContent,
+            Collections.emptySet());
+    }
+
+    /**
+     * Asynchronously Get the Rollup Index Capabilities of a rollup index or pattern
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/master/rollup-get-rollup-index-caps.html">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void getRollupIndexCapabilitiesAsync(GetRollupIndexCapsRequest request, RequestOptions options,
+                                           ActionListener<GetRollupIndexCapsResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request,
+            RollupRequestConverters::getRollupIndexCaps,
+            options,
+            GetRollupIndexCapsResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/RollupRequestConverters.java

@@ -24,6 +24,7 @@
 import org.apache.http.client.methods.HttpPut;
 import org.elasticsearch.client.rollup.DeleteRollupJobRequest;
 import org.elasticsearch.client.rollup.GetRollupCapsRequest;
+import org.elasticsearch.client.rollup.GetRollupIndexCapsRequest;
 import org.elasticsearch.client.rollup.GetRollupJobRequest;
 import org.elasticsearch.client.rollup.PutRollupJobRequest;
 import org.elasticsearch.client.rollup.StartRollupJobRequest;
@@ -85,4 +86,14 @@ static Request getRollupCaps(final GetRollupCapsRequest getRollupCapsRequest) th
         request.setEntity(createEntity(getRollupCapsRequest, REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
+
+    static Request getRollupIndexCaps(final GetRollupIndexCapsRequest getRollupIndexCapsRequest) throws IOException {
+        String endpoint = new RequestConverters.EndpointBuilder()
+            .addCommaSeparatedPathParts(getRollupIndexCapsRequest.indices())
+            .addPathPartAsIs("_xpack", "rollup", "data")
+            .build();
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        request.setEntity(createEntity(getRollupIndexCapsRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/TasksClient.java

@@ -24,8 +24,11 @@
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
+import org.elasticsearch.client.tasks.GetTaskRequest;
+import org.elasticsearch.client.tasks.GetTaskResponse;
 
 import java.io.IOException;
+import java.util.Optional;
 
 import static java.util.Collections.emptySet;
 
@@ -67,6 +70,34 @@ public void listAsync(ListTasksRequest request, RequestOptions options, ActionLi
         restHighLevelClient.performRequestAsyncAndParseEntity(request, TasksRequestConverters::listTasks, options,
                 ListTasksResponse::fromXContent, listener, emptySet());
     }
+
+    /**
+     * Get a task using the Task Management API.
+     * See
+     * <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html"> Task Management API on elastic.co</a>
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public Optional<GetTaskResponse> get(GetTaskRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseOptionalEntity(request, TasksRequestConverters::getTask, options,
+                GetTaskResponse::fromXContent);        
+    }   
+
+    /**
+     * Get a task using the Task Management API.
+     * See
+     * <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html"> Task Management API on elastic.co</a>
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener an actionlistener that takes an optional response (404s are returned as an empty Optional)
+     */
+    public void getAsync(GetTaskRequest request, RequestOptions options, ActionListener<Optional<GetTaskResponse>> listener) {
+
+        restHighLevelClient.performRequestAsyncAndParseOptionalEntity(request, TasksRequestConverters::getTask, options,
+                GetTaskResponse::fromXContent, listener);
+    }    
 
     /**
      * Cancel one or more cluster tasks using the Task Management API.

client/rest-high-level/src/main/java/org/elasticsearch/client/TasksRequestConverters.java

@@ -23,6 +23,8 @@
 import org.apache.http.client.methods.HttpPost;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest;
+import org.elasticsearch.client.RequestConverters.EndpointBuilder;
+import org.elasticsearch.client.tasks.GetTaskRequest;
 
 final class TasksRequestConverters {
 
@@ -54,4 +56,16 @@ static Request listTasks(ListTasksRequest listTaskRequest) {
             .putParam("group_by", "none");
         return request;
     }
+
+    static Request getTask(GetTaskRequest getTaskRequest) {
+        String endpoint = new EndpointBuilder().addPathPartAsIs("_tasks")
+                .addPathPartAsIs(getTaskRequest.getNodeId() + ":" + Long.toString(getTaskRequest.getTaskId()))
+                .build();
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        RequestConverters.Params params = new RequestConverters.Params(request);
+        params.withTimeout(getTaskRequest.getTimeout())
+            .withWaitForCompletion(getTaskRequest.getWaitForCompletion());
+        return request;
+    }
+
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/GetRollupCapsResponse.java

@@ -18,10 +18,6 @@
  */
 package org.elasticsearch.client.rollup;
 
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.ToXContentObject;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -30,7 +26,7 @@
 import java.util.Map;
 import java.util.Objects;
 
-public class GetRollupCapsResponse implements ToXContentObject {
+public class GetRollupCapsResponse {
 
     private final Map<String, RollableIndexCaps> jobs;
 
@@ -42,16 +38,6 @@ public Map<String, RollableIndexCaps> getJobs() {
         return jobs;
     }
 
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
-        builder.startObject();
-        for (Map.Entry<String, RollableIndexCaps> entry : jobs.entrySet()) {
-            entry.getValue().toXContent(builder, params);
-        }
-        builder.endObject();
-        return builder;
-    }
-
     public static GetRollupCapsResponse fromXContent(final XContentParser parser) throws IOException {
         Map<String, RollableIndexCaps> jobs = new HashMap<>();
         XContentParser.Token token = parser.nextToken();
@@ -84,9 +70,4 @@ public boolean equals(Object obj) {
         GetRollupCapsResponse other = (GetRollupCapsResponse) obj;
         return Objects.equals(jobs, other.jobs);
     }
-
-    @Override
-    public final String toString() {
-        return Strings.toString(this);
-    }
 }