Merge branch 'master' into feature/data_stream_support_routing

* master: (868 commits)
  Query API key - Rest spec and yaml tests (elastic#76238)
  Delay shard reassignment from nodes which are known to be restarting (elastic#75606)
  Reenable bwc tests for elastic#76475 (elastic#76576)
  Set version to 7.15 in BWC code (elastic#76577)
  Don't remove warning headers on all failure (elastic#76434)
  Disable bwc tests for elastic#76475 (elastic#76541)
  Re-enable bwc tests (elastic#76567)
  Keep track of data recovered from snapshots in RecoveryState (elastic#76499)
  [Transform] Align transform checkpoint range with date_histogram interval for better performance (elastic#74004)
  EQL: Remove "wildcard" function (elastic#76099)
  Fix 'accept' and 'content_type' fields for search_mvt API
  Add persistent licensed feature tracking (elastic#76476)
  Add system data streams to feature state snapshots (elastic#75902)
  fix the error message for instance methods that don't exist (elastic#76512)
  ILM: Add validation of the number_of_shards parameter in Shrink Action of ILM (elastic#74219)
  remove dashboard only reserved role (elastic#76507)
  Fix Stack Overflow in UnassignedInfo in Corner Case (elastic#76480)
  Add (Extended)KeyUsage KeyUsage, CipherSuite & Protocol to SSL diagnostics (elastic#65634)
  Add recovery from snapshot to tests (elastic#76535)
  Reenable BwC Tests after elastic#76532 (elastic#76534)
  ...

.backportrc.json

@@ -0,0 +1,16 @@
+{
+  "upstream": "elastic/elasticsearch",
+  "targetBranchChoices": [
+    { "name": "master", "checked": true },
+    { "name": "7.x", "checked": true },
+    "7.14",
+    "7.13",
+    "6.8"
+  ],
+  "targetPRLabels": ["backport"],
+  "branchLabelMapping": {
+    "^v8.0.0$": "master",
+    "^v7.15.0$": "7.x",
+    "^v(\\d+).(\\d+).\\d+$": "$1.$2"
+  }
+}

.ci/bwcVersions

@@ -37,5 +37,8 @@ BWC_VERSION:
   - "7.13.1"
   - "7.13.2"
   - "7.13.3"
+  - "7.13.4"
   - "7.14.0"
+  - "7.14.1"
+  - "7.15.0"
   - "8.0.0"

.ci/dockerOnLinuxExclusions

@@ -11,6 +11,7 @@ sles-12.3 # older version used in Vagrant image
 sles-12.5
 sles-15.1
 sles-15.2
+sles-15.3
 
 # These OSes are deprecated and filtered starting with 8.0.0, but need to be excluded
 # for PR checks

.ci/java-versions.properties

@@ -5,4 +5,4 @@
 # are 'java' or 'openjdk' followed by the major release number.
 
 ES_BUILD_JAVA=openjdk16
-ES_RUNTIME_JAVA=openjdk11
+ES_RUNTIME_JAVA=java11

.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-unix.yml

@@ -17,7 +17,6 @@
             - centos-8-packaging
             - debian-9-packaging
             - debian-10-packaging
-            - fedora-32-packaging
             - opensuse-15-1-packaging
             - oraclelinux-7-packaging
             - oraclelinux-8-packaging

.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-arm.yml

@@ -20,6 +20,7 @@
       - inject:
           properties-file: '.ci/java-versions-aarch64.properties'
           properties-content: |
+            COMPOSE_HTTP_TIMEOUT=120
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
             JAVA15_HOME=$HOME/.java/jdk15

.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml

@@ -17,7 +17,6 @@
             - "centos-8&&immutable"
             - "debian-9&&immutable"
             - "debian-10&&immutable"
-            - "fedora-32&&immutable"
             - "opensuse-15-1&&immutable"
             - "oraclelinux-7&&immutable"
             - "oraclelinux-8&&immutable"

.ci/jobs.t/elastic+elasticsearch+periodic+java-fips-matrix.yml

@@ -4,6 +4,6 @@ vars:
   - job-name: elastic+elasticsearch+%BRANCH%+periodic+java-fips-matrix
   - job-display-name: "elastic / elasticsearch # %BRANCH% - java fips compatibility matrix"
   - job-description: "Testing of the Elasticsearch %BRANCH% branch java FIPS compatibility matrix.\n"
-  - matrix-yaml-file: ".ci/matrix-runtime-javas.yml"
+  - matrix-yaml-file: ".ci/matrix-runtime-javas-fips.yml"
   - matrix-variable: ES_RUNTIME_JAVA
   - gradle-args: "-Dbwc.checkout.align=true -Dtests.fips.enabled=true check"

.ci/jobs.t/elastic+elasticsearch+pull-request+bwc.yml

@@ -1,5 +1,37 @@
 ---
-jjbb-template: pull-request-gradle-unix.yml
-vars:
-  - pr-job: "bwc"
-  - gradle-args: "-Dignore.tests.seed bwcTestSnapshots"
+- job:
+    name: "elastic+elasticsearch+pull-request+bwc"
+    display-name: "elastic / elasticsearch - pull request bwc"
+    description: "Testing of Elasticsearch pull requests - bwc"
+    workspace: "/dev/shm/elastic+elasticsearch+pull-request+bwc"
+    scm:
+      - git:
+          refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
+          branches:
+            - "${ghprbActualCommit}"
+    triggers:
+      - github-pull-request:
+          org-list:
+            - elastic
+          allow-whitelist-orgs-as-admins: true
+          trigger-phrase: '.*run\W+elasticsearch-ci/bwc.*'
+          github-hooks: true
+          status-context: elasticsearch-ci/bwc
+          cancel-builds-on-update: true
+          excluded-regions:
+            - ^docs/.*
+          black-list-labels:
+            - '>test-mute'
+            - 'test-full-bwc'
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA8_HOME=$HOME/.java/java8
+            JAVA11_HOME=$HOME/.java/java11
+            JAVA15_HOME=$HOME/.java/openjdk15
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          $WORKSPACE/.ci/scripts/run-gradle.sh -Dignore.tests.seed bwcTestSnapshots

.ci/jobs.t/elastic+elasticsearch+pull-request+docs-check.yml

@@ -1,9 +1,9 @@
 ---
 - job:
-    name: "elastic+elasticsearch+%BRANCH%+pull-request/elastic+elasticsearch+%BRANCH%+pull-request+docs-check"
-    display-name: "elastic / elasticsearch # %BRANCH% - pull request docs-check"
+    name: "elastic+elasticsearch+pull-request+docs-check"
+    display-name: "elastic / elasticsearch - pull request docs-check"
     description: "Testing of Elasticsearch pull requests - docs-check"
-    workspace: "/dev/shm/elastic+elasticsearch+%BRANCH%+pull-request+docs-check"
+    workspace: "/dev/shm/elastic+elasticsearch+pull-request+docs-check"
     scm:
       - git:
           refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
@@ -18,9 +18,6 @@
           github-hooks: true
           status-context: elasticsearch-ci/docs-check
           cancel-builds-on-update: true
-          white-list-target-branches:
-            - %BRANCH%
-            - ^feat(ure)?/.*
           included-regions:
             - ^docs/.*
           black-list-labels:
@@ -30,6 +27,8 @@
           properties-file: '.ci/java-versions.properties'
           properties-content: |
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            JAVA8_HOME=$HOME/.java/java8
+            JAVA11_HOME=$HOME/.java/java11
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr

.ci/jobs.t/elastic+elasticsearch+pull-request+eql-correctness.yml

@@ -1,9 +1,9 @@
 ---
 - job:
-    name: "elastic+elasticsearch+%BRANCH%+pull-request/elastic+elasticsearch+%BRANCH%+pull-request+eql-correctness"
-    display-name: "elastic / elasticsearch # %BRANCH% - pull request eql-correctness"
+    name: "elastic+elasticsearch+pull-request+eql-correctness"
+    display-name: "elastic / elasticsearch - pull request eql-correctness"
     description: "Testing of Elasticsearch pull requests - eql-correctness"
-    workspace: "/dev/shm/elastic+elasticsearch+%BRANCH%+pull-request+eql-correctness"
+    workspace: "/dev/shm/elastic+elasticsearch+pull-request+eql-correctness"
     scm:
       - git:
           refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
@@ -18,9 +18,8 @@
           github-hooks: true
           status-context: elasticsearch-ci/eql-correctness
           cancel-builds-on-update: true
-          white-list-target-branches:
-            - %BRANCH%
-            - ^feat(ure)?/.*
+          black-list-target-branches:
+            - 6.8
           excluded-regions:
             - ^docs/.*
           black-list-labels:

.ci/jobs.t/elastic+elasticsearch+pull-request+full-bwc.yml

@@ -0,0 +1,50 @@
+---
+- job:
+    name: "elastic+elasticsearch+pull-request+full-bwc"
+    display-name: "elastic / elasticsearch - pull request full-bwc"
+    description: "Testing of Elasticsearch pull requests - full-bwc"
+    project-type: matrix
+    node: master
+    child-workspace: "/dev/shm/elastic+elasticsearch+pull-request+full-bwc"
+    scm:
+      - git:
+          refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
+          branches:
+            - "${ghprbActualCommit}"
+    triggers:
+      - github-pull-request:
+          org-list:
+            - elastic
+          allow-whitelist-orgs-as-admins: true
+          trigger-phrase: '.*run\W+elasticsearch-ci/full-bwc.*'
+          github-hooks: true
+          status-context: elasticsearch-ci/full-bwc
+          cancel-builds-on-update: true
+          excluded-regions:
+            - ^docs/.*
+          white-list-labels:
+            - 'test-full-bwc'
+          black-list-labels:
+            - '>test-mute'
+    axes:
+      - axis:
+          type: slave
+          name: nodes
+          values:
+            - "general-purpose"
+      - axis:
+          type: yaml
+          filename: ".ci/bwcVersions"
+          name: "BWC_VERSION"
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA8_HOME=$HOME/.java/java8
+            JAVA11_HOME=$HOME/.java/java11
+            JAVA15_HOME=$HOME/.java/openjdk15
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          $WORKSPACE/.ci/scripts/run-gradle.sh -Dignore.tests.seed v$BWC_VERSION#bwcTest

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix-sample.yml

@@ -1,7 +1,7 @@
 ---
 - job:
-    name: "elastic+elasticsearch+%BRANCH%+pull-request/elastic+elasticsearch+%BRANCH%+pull-request+packaging-tests-unix-sample"
-    display-name: "elastic / elasticsearch # %BRANCH% - pull request packaging-tests-unix-sample"
+    name: "elastic+elasticsearch+pull-request+packaging-tests-unix-sample"
+    display-name: "elastic / elasticsearch - pull request packaging-tests-unix-sample"
     description: "Testing of Elasticsearch pull requests - packaging-tests-unix-sample"
     project-type: matrix
     node: master
@@ -18,10 +18,9 @@
           trigger-phrase: '.*run\W+elasticsearch-ci/packaging-tests-unix-sample.*'
           github-hooks: true
           status-context: elasticsearch-ci/packaging-tests-unix-sample
-          cancel-builds-on-update:
-          white-list-target-branches:
-            - %BRANCH%
-            - ^feat(ure)?/.*
+          cancel-builds-on-update: true
+          black-list-target-branches:
+            - 6.8
           excluded-regions:
             - ^docs/.*
           black-list-labels:

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix.yml

@@ -1,7 +1,7 @@
 ---
 - job:
-    name: "elastic+elasticsearch+%BRANCH%+pull-request/elastic+elasticsearch+%BRANCH%+pull-request+packaging-tests-unix"
-    display-name: "elastic / elasticsearch # %BRANCH% - pull request packaging-tests-unix"
+    name: "elastic+elasticsearch+pull-request+packaging-tests-unix"
+    display-name: "elastic / elasticsearch - pull request packaging-tests-unix"
     description: "Testing of Elasticsearch pull requests - packaging-tests-unix"
     project-type: matrix
     node: master
@@ -19,9 +19,8 @@
           github-hooks: true
           status-context: elasticsearch-ci/packaging-tests-unix
           cancel-builds-on-update: true
-          white-list-target-branches:
-            - %BRANCH%
-            - ^feat(ure)?/.*
+          black-list-target-branches:
+            - 6.8
           excluded-regions:
             - ^docs/.*
           white-list-labels:
@@ -37,7 +36,6 @@
             - centos-8-packaging
             - debian-9-packaging
             - debian-10-packaging
-            - fedora-32-packaging
             - opensuse-15-1-packaging
             - oraclelinux-7-packaging
             - oraclelinux-8-packaging

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-sample.yml

@@ -1,7 +1,7 @@
 ---
 - job:
-    name: "elastic+elasticsearch+%BRANCH%+pull-request/elastic+elasticsearch+%BRANCH%+pull-request+packaging-tests-windows-sample"
-    display-name: "elastic / elasticsearch # %BRANCH% - pull request packaging-tests-windows-sample"
+    name: "elastic+elasticsearch+pull-request+packaging-tests-windows-sample"
+    display-name: "elastic / elasticsearch - pull request packaging-tests-windows-sample"
     description: "Testing of Elasticsearch pull requests - packaging-tests-windows-sample"
     # We use a hard-coded workspace directory here to avoid hitting windows path length limits
     child-workspace: "C:\\Users\\jenkins\\workspace\\pr-packaging-windows\\${BUILD_NUMBER}"
@@ -21,9 +21,8 @@
           github-hooks: true
           status-context: elasticsearch-ci/packaging-tests-windows-sample
           cancel-builds-on-update: true
-          white-list-target-branches:
-            - %BRANCH%
-            - ^feat(ure)?/.*
+          black-list-target-branches:
+            - 6.8
           excluded-regions:
             - ^docs/.*
           black-list-labels:

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows.yml

@@ -1,7 +1,7 @@
 ---
 - job:
-    name: "elastic+elasticsearch+%BRANCH%+pull-request/elastic+elasticsearch+%BRANCH%+pull-request+packaging-tests-windows"
-    display-name: "elastic / elasticsearch # %BRANCH% - pull request packaging-tests-windows"
+    name: "elastic+elasticsearch+pull-request+packaging-tests-windows"
+    display-name: "elastic / elasticsearch - pull request packaging-tests-windows"
     description: "Testing of Elasticsearch pull requests - packaging-tests-windows"
     # We use a hard-coded workspace directory here to avoid hitting windows path length limits
     child-workspace: "C:\\Users\\jenkins\\workspace\\pr-packaging-windows\\${BUILD_NUMBER}"
@@ -21,9 +21,8 @@
           github-hooks: true
           status-context: elasticsearch-ci/packaging-tests-windows
           cancel-builds-on-update: true
-          white-list-target-branches:
-            - %BRANCH%
-            - ^feat(ure)?/.*
+          black-list-target-branches:
+            - 6.8
           excluded-regions:
             - ^docs/.*
           white-list-labels:

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-upgrade-tests.yml

@@ -0,0 +1,53 @@
+---
+- job:
+    name: "elastic+elasticsearch+pull-request+packaging-upgrade-tests"
+    display-name: "elastic / elasticsearch - pull request packaging-upgrade-tests"
+    description: "Testing of Elasticsearch pull requests - packaging-upgrade-tests"
+    project-type: matrix
+    node: master
+    child-workspace: "/dev/shm/elastic+elasticsearch+pull-request+packaging-upgrade-tests"
+    scm:
+      - git:
+          refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
+          branches:
+            - "${ghprbActualCommit}"
+    triggers:
+      - github-pull-request:
+          org-list:
+            - elastic
+          allow-whitelist-orgs-as-admins: true
+          trigger-phrase: '.*run\W+elasticsearch-ci/packaging-upgrade-tests.*'
+          github-hooks: true
+          status-context: elasticsearch-ci/packaging-upgrade-tests
+          cancel-builds-on-update: true
+          black-list-target-branches:
+            - 6.8
+          excluded-regions:
+            - ^docs/.*
+          white-list-labels:
+            - ':Delivery/Packaging'
+          black-list-labels:
+            - '>test-mute'
+    axes:
+      - axis:
+          type: label-expression
+          name: os
+          values:
+            - centos-8-packaging
+            - ubuntu-20.04-packaging
+      - axis:
+          type: yaml
+          filename: ".ci/bwcVersions"
+          name: "BWC_VERSION"
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA8_HOME=$HOME/.java/java8
+            JAVA11_HOME=$HOME/.java/java11
+            JAVA15_HOME=$HOME/.java/openjdk15
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          ./.ci/os.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ destructiveDistroUpgradeTest.v$BWC_VERSION