Merge branch 'master' into retention-lease-ccr

* master:
  Address some CCR REST test case flakiness (elastic#38975)
  Edits to text in Completion Suggester doc (elastic#38980)
  SQL: doc polishing
  [DOCS] Fixes broken formatting
  SQL: Polish the rest chapter (elastic#38971)
  Remove `nGram` and  `edgeNGram` token filter names (elastic#38911)
  Add an exception throw if waiting on transport port file fails (elastic#37574)
  Improve testcluster distribution artifact handling (elastic#38933)
  Advance max_seq_no before add operation to Lucene (elastic#38879)
  Reduce global checkpoint sync interval in disruption tests (elastic#38931)
  [test] disable packaging tests for suse boxes
  Relax testStressMaybeFlushOrRollTranslogGeneration (elastic#38918)
  [DOCS] Edits warning in put watch API (elastic#38582)
  Fix serialization bug in ShardFollowTask after cutting this class over to extend from ImmutableFollowParameters.
  [DOCS] Updates methods for upgrading machine learning (elastic#38876)

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -118,11 +118,16 @@ class ClusterConfiguration {
         if (seedNode == node) {
             return null
         }
-        ant.waitfor(maxwait: '40', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond') {
+        ant.waitfor(maxwait: '40', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond',
+                timeoutproperty: "failed.${seedNode.transportPortsFile.path}") {
             resourceexists {
                 file(file: seedNode.transportPortsFile.toString())
             }
         }
+        if (ant.properties.containsKey("failed.${seedNode.transportPortsFile.path}".toString())) {
+            throw new GradleException("Failed to locate seed node transport file [${seedNode.transportPortsFile}]: " +
+                    "timed out waiting for it to be created after ${waitSeconds} seconds")
+        }
         return seedNode.transportUri()
     }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -547,7 +547,11 @@ class VagrantTestPlugin implements Plugin<Project> {
                     project.gradle.removeListener(batsPackagingReproListener)
                 }
                 if (project.extensions.esvagrant.boxes.contains(box)) {
-                    packagingTest.dependsOn(batsPackagingTest)
+                    // these tests are temporarily disabled for suse boxes while we debug an issue
+                    // https://github.com/elastic/elasticsearch/issues/30295
+                    if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) {
+                        packagingTest.dependsOn(batsPackagingTest)
+                    }
                 }
             }
 
@@ -586,7 +590,11 @@ class VagrantTestPlugin implements Plugin<Project> {
                 project.gradle.removeListener(javaPackagingReproListener)
             }
             if (project.extensions.esvagrant.boxes.contains(box)) {
-                packagingTest.dependsOn(javaPackagingTest)
+                // these tests are temporarily disabled for suse boxes while we debug an issue
+                // https://github.com/elastic/elasticsearch/issues/30295
+                if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) {
+                    packagingTest.dependsOn(javaPackagingTest)
+                }
             }
 
             /*

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java

@@ -25,11 +25,14 @@
 import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.component.ComponentArtifactIdentifier;
 import org.gradle.api.execution.TaskActionListener;
 import org.gradle.api.execution.TaskExecutionListener;
+import org.gradle.api.file.FileTree;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
 import org.gradle.api.plugins.ExtraPropertiesExtension;
+import org.gradle.api.tasks.Sync;
 import org.gradle.api.tasks.TaskState;
 
 import java.io.File;
@@ -39,6 +42,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -87,6 +91,20 @@ public void apply(Project project) {
                 "Internal helper configuration used by cluster configuration to download " +
                     "ES distributions and plugins."
             );
+            helperConfiguration.getIncoming().afterResolve(resolvableDependencies -> {
+                Set<ComponentArtifactIdentifier> nonZipComponents = resolvableDependencies.getArtifacts()
+                    .getArtifacts()
+                    .stream()
+                    .filter(artifact -> artifact.getFile().getName().endsWith(".zip") == false)
+                    .map(artifact -> artifact.getId())
+                    .collect(Collectors.toSet());
+
+                if(nonZipComponents.isEmpty() == false) {
+                    throw new IllegalStateException("Dependencies with non-zip artifacts found in configuration '" +
+                        TestClustersPlugin.HELPER_CONFIGURATION_NAME + "': " + nonZipComponents
+                    );
+                }
+            });
 
             // When running in the Daemon it's possible for this to hold references to past
             usedClusters.clear();
@@ -98,7 +116,15 @@ public void apply(Project project) {
             // the clusters will look for artifacts there based on the naming conventions.
             // Tasks that use a cluster will add this as a dependency automatically so it's guaranteed to run early in
             // the build.
-            rootProject.getTasks().create(SYNC_ARTIFACTS_TASK_NAME, SyncTestClustersConfiguration.class);
+            rootProject.getTasks().create(SYNC_ARTIFACTS_TASK_NAME, Sync.class, sync -> {
+                sync.from((Callable<List<FileTree>>) () ->
+                    helperConfiguration.getFiles()
+                        .stream()
+                        .map(project::zipTree)
+                        .collect(Collectors.toList())
+                );
+                sync.into(new File(getTestClustersConfigurationExtractDir(project), "zip"));
+            });
 
             // When we know what tasks will run, we claim the clusters of those task to differentiate between clusters
             // that are defined in the build script and the ones that will actually be used in this invocation of gradle
@@ -129,7 +155,7 @@ private NamedDomainObjectContainer<ElasticsearchNode> createTestClustersContaine
                 project.getPath(),
                 name,
                 GradleServicesAdapter.getInstance(project),
-                SyncTestClustersConfiguration.getTestClustersConfigurationExtractDir(project),
+                getTestClustersConfigurationExtractDir(project),
                 new File(project.getBuildDir(), "testclusters")
             )
         );
@@ -249,8 +275,8 @@ public void beforeExecute(Task task) {}
         );
     }
 
-    static File getTestClustersBuildDir(Project project) {
-        return new File(project.getRootProject().getBuildDir(), "testclusters");
+    static File getTestClustersConfigurationExtractDir(Project project) {
+        return new File(project.getRootProject().getBuildDir(), "testclusters/extract");
     }
 
     /**

docs/reference/analysis/tokenfilters/edgengram-tokenfilter.asciidoc

@@ -1,9 +1,9 @@
 [[analysis-edgengram-tokenfilter]]
 === Edge NGram Token Filter
 
-A token filter of type `edgeNGram`.
+A token filter of type `edge_ngram`.
 
-The following are settings that can be set for a `edgeNGram` token
+The following are settings that can be set for a `edge_ngram` token
 filter type:
 
 [cols="<,<",options="header",]

docs/reference/analysis/tokenfilters/ngram-tokenfilter.asciidoc

@@ -1,9 +1,9 @@
 [[analysis-ngram-tokenfilter]]
 === NGram Token Filter
 
-A token filter of type `nGram`.
+A token filter of type `ngram`.
 
-The following are settings that can be set for a `nGram` token filter
+The following are settings that can be set for a `ngram` token filter
 type:
 
 [cols="<,<",options="header",]

docs/reference/index-modules/allocation/filtering.asciidoc

@@ -49,6 +49,7 @@ settings support three types of filters: `include`, `exclude`, and `require`.
 For example, to tell {es} to allocate shards from the `test` index to either
 `big` or `medium` nodes, use `index.routing.allocation.include`:
 +
+--
 [source,js]
 ------------------------
 PUT test/_settings
@@ -58,11 +59,11 @@ PUT test/_settings
 ------------------------
 // CONSOLE
 // TEST[s/^/PUT test\n/]
-+
+
 If you specify multiple filters, all conditions must be satisfied for shards to
 be relocated. For example, to move the `test` index to `big` nodes in `rack1`,
 you could specify:
-+
+
 [source,js]
 ------------------------
 PUT test/_settings
@@ -73,6 +74,7 @@ PUT test/_settings
 ------------------------
 // CONSOLE
 // TEST[s/^/PUT test\n/]
+--
 
 [float]
 [[index-allocation-settings]]

docs/reference/migration/migrate_8_0.asciidoc

@@ -9,4 +9,8 @@ your application to {es} 8.0.
 
 See also <<release-highlights>> and <<es-release-notes>>.
 
-coming[8.0.0]
+coming[8.0.0]
+
+* <<breaking_80_mappings_changes>>
+
+include::migrate_8_0/mappings.asciidoc[]

docs/reference/migration/migrate_8_0/mappings.asciidoc

@@ -0,0 +1,10 @@
+[float]
+[[breaking_80_mappings_changes]]
+=== Mapping changes
+
+[float]
+==== The `nGram` and `edgeNGram` token filter names have been removed
+
+The `nGram` and `edgeNGram` token filter names that have been deprecated since
+version 6.4 have been removed. Both token filters should be used by their 
+alternative names `ngram` and `edge_ngram` instead.

docs/reference/search/suggesters/completion-suggest.asciidoc

@@ -43,6 +43,7 @@ PUT music
 
 Mapping supports the following parameters:
 
+[horizontal]
 `analyzer`::
     The index analyzer to use, defaults to `simple`.
     In case you are wondering why we did not opt for the `standard`
@@ -70,7 +71,7 @@ Mapping supports the following parameters:
     Limits the length of a single input, defaults to `50` UTF-16 code points.
     This limit is only used at index time to reduce the total number of
     characters per input string in order to prevent massive inputs from
-    bloating the underlying datastructure. Most usecases won't be influenced
+    bloating the underlying datastructure. Most use cases won't be influenced
     by the default value since prefix completions seldom grow beyond prefixes longer
     than a handful of characters.
 
@@ -97,6 +98,7 @@ PUT music/_doc/1?refresh
 
 The following parameters are supported:
 
+[horizontal]
 `input`::
     The input to store, this can be an array of strings or just
     a string. This field is mandatory.
@@ -285,6 +287,7 @@ Which should look like:
 
 The basic completion suggester query supports the following parameters:
 
+[horizontal]
 `field`:: The name of the field on which to run the query (required).
 `size`::  The number of suggestions to return (defaults to `5`).
 `skip_duplicates`:: Whether duplicate suggestions should be filtered out (defaults to `false`).
@@ -326,13 +329,13 @@ POST music/_search?pretty
 --------------------------------------------------
 // CONSOLE
 
-WARNING: when set to true this option can slow down search because more suggestions
+WARNING: When set to true, this option can slow down search because more suggestions
 need to be visited to find the top N.
 
 [[fuzzy]]
 ==== Fuzzy queries
 
-The completion suggester also supports fuzzy queries - this means,
+The completion suggester also supports fuzzy queries -- this means
 you can have a typo in your search and still get results back.
 
 [source,js]