Merge branch '6.6' into docker-from-artifacts-6.6

* 6.6: (121 commits)
  [DOCS] Add warning about bypassing ML PUT APIs (elastic#38608)
  fix dissect doc "ip" --> "clientip" (elastic#38512)
  bad formatted JSON object (elastic#38515)
  SQL: Fix issue with IN not resolving to underlying keyword field (elastic#38440)
  Update ilm-api.asciidoc, point to REMOVE policy (elastic#38235)
  Backport changes to the release notes script. (elastic#38347)
  Change the milliseconds precision to 3 digits for intervals. (elastic#38297)
  SecuritySettingsSource license.self_generated: trial (elastic#38233) (elastic#38398)
  Fix IndexAuditTrail rolling upgrade on rollover edge 2 (elastic#38286) (elastic#38381)
  Cleanup construction of interceptors (elastic#38388)
  Skip unsupported languages for tests (elastic#38328) (elastic#38385)
  [ILM][TEST] increase assertBusy timeout (elastic#36864) (elastic#38354)
  Docs: Drop inline callout from scroll example (elastic#38340) (elastic#38365)
  Preserve ILM operation mode when creating new lifecycles (elastic#38134) (elastic#38230)
  [ML] Add explanation so far to file structure finder exceptions (elastic#38337)
  ML: Fix error race condition on stop _all datafeeds and close _all jobs (elastic#38113) (elastic#38211) (elastic#38222)
  SQL: Generate relevant error message when grouping functions are not used in GROUP BY (elastic#38017)
  Fix NPE in Logfile Audit Filter (elastic#38120) (elastic#38273)
  Enable trace log in FollowerFailOverIT (elastic#38148)
  Replace awaitBusy with assertBusy in atLeastDocsIndexed (elastic#38190)
  ...

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -99,7 +99,7 @@ public class PluginBuildPlugin extends BuildPlugin {
                 generatePOMTask.ext.pomFileName = "${project.archivesBaseName}-client-${project.versions.elasticsearch}.pom"
             }
         } else {
-            project.plugins.withType(MavenPublishPlugin).whenPluginAdded {
+            if (project.plugins.hasPlugin(MavenPublishPlugin)) {
                 project.publishing.publications.nebula(MavenPublication).artifactId(
                         project.pluginProperties.extension.name
                 )

buildSrc/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-5.0
+5.1.1

buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java

@@ -40,13 +40,15 @@
 
 public class BuildExamplePluginsIT extends GradleIntegrationTestCase {
 
-    private static List<File> EXAMPLE_PLUGINS = Collections.unmodifiableList(
+    private static final List<File> EXAMPLE_PLUGINS = Collections.unmodifiableList(
         Arrays.stream(
             Objects.requireNonNull(System.getProperty("test.build-tools.plugin.examples"))
                 .split(File.pathSeparator)
         ).map(File::new).collect(Collectors.toList())
     );
 
+    private static final String BUILD_TOOLS_VERSION = Objects.requireNonNull(System.getProperty("test.version_under_test"));
+
     @Rule
     public TemporaryFolder tmpDir = new TemporaryFolder();
 
@@ -96,7 +98,8 @@ public void testCurrentExamplePlugin() throws IOException {
 
     private void adaptBuildScriptForTest() throws IOException {
         // Add the local repo as a build script URL so we can pull in build-tools and apply the plugin under test
-        // + is ok because we have no other repo and just want to pick up latest
+        // we need to specify the exact version of build-tools because gradle automatically adds its plugin portal
+        // which appears to mirror jcenter, opening us up to pulling a "later" version of build-tools
         writeBuildScript(
             "buildscript {\n" +
                 "    repositories {\n" +
@@ -105,7 +108,7 @@ private void adaptBuildScriptForTest() throws IOException {
                 "        }\n" +
                 "    }\n" +
                 "    dependencies {\n" +
-                "        classpath \"org.elasticsearch.gradle:build-tools:+\"\n" +
+                "        classpath \"org.elasticsearch.gradle:build-tools:" + BUILD_TOOLS_VERSION + "\"\n" +
                 "    }\n" +
                 "}\n"
         );

buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java

@@ -21,9 +21,11 @@
 import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
 import org.gradle.testkit.runner.BuildResult;
 import org.gradle.testkit.runner.GradleRunner;
+import org.junit.Ignore;
 
 import java.util.Arrays;
 
+@Ignore // https://github.com/elastic/elasticsearch/issues/37218
 public class TestClustersPluginIT extends GradleIntegrationTestCase {
 
     public void testListClusters() {

buildSrc/version.properties

@@ -1,4 +1,4 @@
-elasticsearch     = 6.6.0
+elasticsearch     = 6.6.1
 lucene            = 7.6.0
 
 # optional dependencies

client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/AutoFollowStats.java

@@ -45,7 +45,9 @@ public final class AutoFollowStats {
     static final ParseField LAST_SEEN_METADATA_VERSION = new ParseField("last_seen_metadata_version");
 
     @SuppressWarnings("unchecked")
-    static final ConstructingObjectParser<AutoFollowStats, Void> STATS_PARSER = new ConstructingObjectParser<>("auto_follow_stats",
+    static final ConstructingObjectParser<AutoFollowStats, Void> STATS_PARSER = new ConstructingObjectParser<>(
+        "auto_follow_stats",
+        true,
         args -> new AutoFollowStats(
             (Long) args[0],
             (Long) args[1],
@@ -63,11 +65,13 @@ public final class AutoFollowStats {
     private static final ConstructingObjectParser<Map.Entry<String, ElasticsearchException>, Void> AUTO_FOLLOW_EXCEPTIONS_PARSER =
         new ConstructingObjectParser<>(
             "auto_follow_stats_errors",
+            true,
             args -> new AbstractMap.SimpleEntry<>((String) args[0], (ElasticsearchException) args[1]));
 
     private static final ConstructingObjectParser<Map.Entry<String, AutoFollowedCluster>, Void> AUTO_FOLLOWED_CLUSTERS_PARSER =
         new ConstructingObjectParser<>(
             "auto_followed_clusters",
+            true,
             args -> new AbstractMap.SimpleEntry<>((String) args[0], new AutoFollowedCluster((Long) args[1], (Long) args[2])));
 
     static {

client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/CcrStatsResponse.java

@@ -28,7 +28,9 @@ public final class CcrStatsResponse {
     static final ParseField AUTO_FOLLOW_STATS_FIELD = new ParseField("auto_follow_stats");
     static final ParseField FOLLOW_STATS_FIELD = new ParseField("follow_stats");
 
-    private static final ConstructingObjectParser<CcrStatsResponse, Void> PARSER = new ConstructingObjectParser<>("indices",
+    private static final ConstructingObjectParser<CcrStatsResponse, Void> PARSER = new ConstructingObjectParser<>(
+        "indices",
+        true,
         args -> {
             AutoFollowStats autoFollowStats = (AutoFollowStats) args[0];
             IndicesFollowStats indicesFollowStats = (IndicesFollowStats) args[1];

client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponse.java

@@ -42,15 +42,15 @@ public final class GetAutoFollowPatternResponse {
     static final ParseField PATTERN_FIELD = new ParseField("pattern");
 
     private static final ConstructingObjectParser<Map.Entry<String, Pattern>, Void> ENTRY_PARSER = new ConstructingObjectParser<>(
-        "get_auto_follow_pattern_response", args -> new AbstractMap.SimpleEntry<>((String) args[0], (Pattern) args[1]));
+        "get_auto_follow_pattern_response", true, args -> new AbstractMap.SimpleEntry<>((String) args[0], (Pattern) args[1]));
 
     static {
         ENTRY_PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD);
         ENTRY_PARSER.declareObject(ConstructingObjectParser.constructorArg(), Pattern.PARSER, PATTERN_FIELD);
     }
 
     private static final ConstructingObjectParser<GetAutoFollowPatternResponse, Void> PARSER = new ConstructingObjectParser<>(
-        "get_auto_follow_pattern_response", args -> {
+        "get_auto_follow_pattern_response", true, args -> {
             @SuppressWarnings("unchecked")
             List<Map.Entry<String, Pattern>> entries = (List<Map.Entry<String, Pattern>>) args[0];
             return new GetAutoFollowPatternResponse(new TreeMap<>(entries.stream()
@@ -92,7 +92,7 @@ public static class Pattern extends FollowConfig {
 
         @SuppressWarnings("unchecked")
         private static final ConstructingObjectParser<Pattern, Void> PARSER = new ConstructingObjectParser<>(
-            "pattern", args -> new Pattern((String) args[0], (List<String>) args[1], (String) args[2]));
+            "pattern", true, args -> new Pattern((String) args[0], (List<String>) args[1], (String) args[2]));
 
         static {
             PARSER.declareString(ConstructingObjectParser.constructorArg(), PutFollowRequest.REMOTE_CLUSTER_FIELD);

client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/IndicesFollowStats.java

@@ -41,6 +41,7 @@ public final class IndicesFollowStats {
     private static final ConstructingObjectParser<Tuple<String, List<ShardFollowStats>>, Void> ENTRY_PARSER =
         new ConstructingObjectParser<>(
             "entry",
+            true,
             args -> {
                 String index = (String) args[0];
                 @SuppressWarnings("unchecked")
@@ -54,7 +55,9 @@ public final class IndicesFollowStats {
         ENTRY_PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), ShardFollowStats.PARSER, SHARDS_FIELD);
     }
 
-    static final ConstructingObjectParser<IndicesFollowStats, Void> PARSER = new ConstructingObjectParser<>("indices",
+    static final ConstructingObjectParser<IndicesFollowStats, Void> PARSER = new ConstructingObjectParser<>(
+        "indices",
+        true,
         args -> {
             @SuppressWarnings("unchecked")
             List<Tuple<String, List<ShardFollowStats>>> entries = (List<Tuple<String, List<ShardFollowStats>>>) args[0];
@@ -116,6 +119,7 @@ public static final class ShardFollowStats {
         static final ConstructingObjectParser<ShardFollowStats, Void> PARSER =
             new ConstructingObjectParser<>(
                 "shard-follow-stats",
+                true,
                 args -> new ShardFollowStats(
                     (String) args[0],
                     (String) args[1],
@@ -152,6 +156,7 @@ public static final class ShardFollowStats {
         static final ConstructingObjectParser<Map.Entry<Long, Tuple<Integer, ElasticsearchException>>, Void> READ_EXCEPTIONS_ENTRY_PARSER =
             new ConstructingObjectParser<>(
                 "shard-follow-stats-read-exceptions-entry",
+                true,
                 args -> new AbstractMap.SimpleEntry<>((long) args[0], Tuple.tuple((Integer) args[1], (ElasticsearchException)args[2])));
 
         static {

client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutFollowResponse.java

@@ -33,7 +33,7 @@ public final class PutFollowResponse {
     static final ParseField INDEX_FOLLOWING_STARTED = new ParseField("index_following_started");
 
     private static final ConstructingObjectParser<PutFollowResponse, Void> PARSER = new ConstructingObjectParser<>(
-        "put_follow_response", args -> new PutFollowResponse((boolean) args[0], (boolean) args[1], (boolean) args[2]));
+        "put_follow_response", true, args -> new PutFollowResponse((boolean) args[0], (boolean) args[1], (boolean) args[2]));
 
     static {
         PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), FOLLOW_INDEX_CREATED);

client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/CcrStatsResponseTests.java

@@ -51,7 +51,7 @@ public void testFromXContent() throws IOException {
             CcrStatsResponseTests::createTestInstance,
             CcrStatsResponseTests::toXContent,
             CcrStatsResponse::fromXContent)
-            .supportsUnknownFields(false)
+            .supportsUnknownFields(true)
             .assertEqualsConsumer(CcrStatsResponseTests::assertEqualInstances)
             .assertToXContentEquivalence(false)
             .test();

client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/FollowStatsResponseTests.java

@@ -48,7 +48,7 @@ public void testFromXContent() throws IOException {
             FollowStatsResponseTests::createTestInstance,
             FollowStatsResponseTests::toXContent,
             FollowStatsResponse::fromXContent)
-            .supportsUnknownFields(false)
+            .supportsUnknownFields(true)
             .assertEqualsConsumer(FollowStatsResponseTests::assertEqualInstances)
             .assertToXContentEquivalence(false)
             .test();

client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponseTests.java

@@ -43,7 +43,7 @@ public void testFromXContent() throws IOException {
             this::createTestInstance,
             GetAutoFollowPatternResponseTests::toXContent,
             GetAutoFollowPatternResponse::fromXContent)
-            .supportsUnknownFields(false)
+            .supportsUnknownFields(true)
             .test();
     }
 

client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/PutFollowRequestTests.java

@@ -31,7 +31,7 @@
 public class PutFollowRequestTests extends AbstractXContentTestCase<PutFollowRequest> {
 
     private static final ConstructingObjectParser<PutFollowRequest, Void> PARSER = new ConstructingObjectParser<>("test_parser",
-        (args) -> new PutFollowRequest((String) args[0], (String) args[1], (String) args[2]));
+        true, (args) -> new PutFollowRequest((String) args[0], (String) args[1], (String) args[2]));
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), PutFollowRequest.REMOTE_CLUSTER_FIELD);

client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/PutFollowResponseTests.java

@@ -33,7 +33,7 @@ public void testFromXContent() throws IOException {
             this::createTestInstance,
             PutFollowResponseTests::toXContent,
             PutFollowResponse::fromXContent)
-            .supportsUnknownFields(false)
+            .supportsUnknownFields(true)
             .test();
     }
 

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/QueryDSLDocumentationTests.java

@@ -23,7 +23,7 @@
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.geo.builders.CoordinatesBuilder;
-import org.elasticsearch.common.geo.builders.ShapeBuilders;
+import org.elasticsearch.common.geo.builders.MultiPointBuilder;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.index.query.GeoShapeQueryBuilder;
 import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
@@ -190,7 +190,7 @@ public void testGeoShape() throws IOException {
             // tag::geo_shape
             GeoShapeQueryBuilder qb = geoShapeQuery(
                     "pin.location",                                      // <1>
-                    ShapeBuilders.newMultiPoint(                         // <2>
+                    new MultiPointBuilder(                         // <2>
                             new CoordinatesBuilder()
                         .coordinate(0, 0)
                         .coordinate(0, 10)

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java

@@ -709,12 +709,12 @@ public void onFailure(Exception e) {
             SearchHit[] searchHits = searchResponse.getHits().getHits();
 
             while (searchHits != null && searchHits.length > 0) { // <2>
-                SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); // <3>
+                // <3>
+                SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); // <4>
                 scrollRequest.scroll(scroll);
                 searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
                 scrollId = searchResponse.getScrollId();
                 searchHits = searchResponse.getHits().getHits();
-                // <4>
             }
 
             ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); // <5>

dev-tools/es_release_notes.pl

@@ -32,7 +32,7 @@
     ">enhancement", ">bug",           ">regression",  ">upgrade"
 );
 my %Ignore = map { $_ => 1 }
-    ( ">non-issue", ">refactoring", ">docs", ">test", ">test-failure", ":Core/Build", "backport" );
+    ( ">non-issue", ">refactoring", ">docs", ">test", ">test-failure", ">test-mute", ":Core/Infra/Build", "backport" );
 
 my %Group_Labels = (
     '>breaking'      => 'Breaking changes',
@@ -48,7 +48,7 @@
 
 my %Area_Overrides = (
     ':ml'            => 'Machine Learning',
-    ':beats'         => 'Beats Plugin',
+    ':Beats'         => 'Beats Plugin',
     ':Docs'          => 'Docs Infrastructure'
 );
 

docs/Versions.asciidoc

@@ -11,7 +11,7 @@
 release-state can be: released | prerelease | unreleased
 //////////
 
-:release-state:   unreleased
+:release-state:   released
 
 :issue:           https://github.com/elastic/elasticsearch/issues/
 :ml-issue:        https://github.com/elastic/ml-cpp/issues/

docs/java-rest/high-level/search/scroll.asciidoc

@@ -128,9 +128,9 @@ include-tagged::{doc-tests}/SearchDocumentationIT.java[search-scroll-example]
 <1> Initialize the search context by sending the initial `SearchRequest`
 <2> Retrieve all the search hits by calling the Search Scroll api in a loop
 until no documents are returned
-<3> Create a new `SearchScrollRequest` holding the last returned scroll
+<3> Process the returned search results
+<4> Create a new `SearchScrollRequest` holding the last returned scroll
 identifier and the scroll interval
-<4> Process the returned search results
 <5> Clear the scroll context once the scroll is completed
 
 [[java-rest-high-clear-scroll]]

docs/plugins/analysis.asciidoc

@@ -58,6 +58,7 @@ A number of analysis plugins have been contributed by our community:
 * https://github.com/ofir123/elasticsearch-network-analysis[Network Addresses Analysis Plugin] (by Ofir123)
 * https://github.com/medcl/elasticsearch-analysis-string2int[String2Integer Analysis Plugin] (by Medcl)
 * https://github.com/ZarHenry96/elasticsearch-dandelion-plugin[Dandelion Analysis Plugin] (by ZarHenry96)
+* https://github.com/medcl/elasticsearch-analysis-stconvert[STConvert Analysis Plugin] (by Medcl)
 
 include::analysis-icu.asciidoc[]
 

docs/plugins/repository-s3.asciidoc

@@ -34,10 +34,10 @@ PUT _snapshot/my_s3_repository
 [[repository-s3-client]]
 ==== Client Settings
 
-The client used to connect to S3 has a number of settings available. Client setting names are of
-the form `s3.client.CLIENT_NAME.SETTING_NAME`. The default client name, which is looked up by
-an `s3` repository, is called `default`. It can be modified using the
-<<repository-s3-repository, repository setting>> `client`. For example:
+The client that you use to connect to S3 has a number of settings available. The
+settings have the form `s3.client.CLIENT_NAME.SETTING_NAME`. The default client
+name that is looked up by an `s3` repository is `default`. It can be modified
+using the <<repository-s3-repository,repository setting>> `client`. For example:
 
 [source,js]
 ----
@@ -53,11 +53,13 @@ PUT _snapshot/my_s3_repository
 // CONSOLE
 // TEST[skip:we don't have s3 setup while testing this]
 
-Most client settings are specified inside `elasticsearch.yml`, but some are
-sensitive and must be stored in the {ref}/secure-settings.html[elasticsearch keystore].
+Most client settings can be added to the `elasticsearch.yml` configuration file
+with the exception of the secure settings, which you add to the {es} keystore.
+For more information about creating and updating the {es} keystore, see
+{ref}/secure-settings.html[Secure settings].
 
-For example, before you start the node, run these commands to add AWS access
-key settings to the keystore:
+For example, before you start the node, run these commands to add AWS access key
+settings to the keystore:
 
 [source,sh]
 ----
@@ -76,16 +78,17 @@ NOTE: In progress snapshot/restore tasks will not be preempted by a *reload*
 of the client's secure settings. The task will complete using the client as it
 was built when the operation started.
 
-The following is the list of all the available client settings.
-Those that must be stored in the keystore are marked as `Secure` and are *reloadable*.
+The following list contains the available client settings. Those that must be
+stored in the keystore are marked as "secure" and are *reloadable*; the other
+settings belong in the `elasticsearch.yml` file.
 
-`access_key`::
+`access_key` ({ref}/secure-settings.html[Secure])::
 
-    An s3 access key. The `secret_key` setting must also be specified. (Secure)
+    An s3 access key. The `secret_key` setting must also be specified. 
 
-`secret_key`::
+`secret_key` ({ref}/secure-settings.html[Secure])::
 
-    An s3 secret key. The `access_key` setting must also be specified. (Secure)
+    An s3 secret key. The `access_key` setting must also be specified. 
 
 `session_token`::
     An s3 session token. The `access_key` and `secret_key` settings must also
@@ -110,13 +113,13 @@ Those that must be stored in the keystore are marked as `Secure` and are *reload
 
     The port of a proxy to connect to s3 through.
 
-`proxy.username`::
+`proxy.username` ({ref}/secure-settings.html[Secure])::
 
-    The username to connect to the `proxy.host` with. (Secure)    
+    The username to connect to the `proxy.host` with.    
 
-`proxy.password`::
+`proxy.password` ({ref}/secure-settings.html[Secure])::
 
-    The password to connect to the `proxy.host` with. (Secure)    
+    The password to connect to the `proxy.host` with.   
 
 `read_timeout`::
 

docs/reference/cat.asciidoc

@@ -9,8 +9,8 @@
 
 JSON is great... for computers.  Even if it's pretty-printed, trying
 to find relationships in the data is tedious.  Human eyes, especially
-when looking at an ssh terminal, need compact and aligned text.  The
-cat API aims to meet this need.
+when looking at a terminal, need compact and aligned text.  The cat API
+aims to meet this need.
 
 All the cat commands accept a query string parameter `help` to see all
 the headers and info they provide, and the `/_cat` command alone lists all