Merge branch 'master' into unbound_id_response_timeout

.ci/README.md

@@ -0,0 +1,16 @@
+CI configuration for Elasticsearch
+==================================
+
+Layout and Syntax
+-----------------
+
+CI is run by Jenkins at [elasticsearch-ci](https://elasticsearch-ci.elastic.co/).
+Jobs live in the [jobs.t](jobs.t) directory, these are defined in YML using a syntax 
+simmilar to [JJB](https://elasticsearch-ci.elastic.co/view/Elasticsearch%20master/).
+Macros are not allowed, and each job must be defined in its own file.
+Merging of the default configuration is customized so unlike in standard JJB,
+it recurses into YML objects. 
+Further (internal) documentation  on the setup 
+[is available](https://github.com/elastic/infra/blob/master/flavortown/jjbb/README.md) 
+.
+

.ci/build.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# drop page cache and kernel slab objects on linux
+[[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches
+
+rm -Rfv ~/.gradle/init.d/init.gradle
+mkdir -p ~/.gradle/init.d && cp -v $WORKSPACE/.ci/init.gradle ~/.gradle/init.d
+
+if [ -f /proc/cpuinfo ] ; then
+   MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo  | uniq | sed 's/\s\+//g' |  cut -d':' -f 2`
+else
+   if [[ "$OSTYPE" == "darwin"* ]]; then
+      # Parallel is disabled at  this time (eventually set to 1) due to errors on the Mac workers
+      # We'll have to do more testing to see if this can be re-enabled or what the proper value is.
+      # MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'`
+      MAX_WORKERS=2
+   else
+      echo "Unsupported OS Type: $OSTYPE"
+      exit 1
+   fi
+fi
+
+if pwd | grep -v -q ^/dev/shm ; then
+   echo "Not running on a ramdisk, reducing number of workers"
+   MAX_WORKERS=$(($MAX_WORKERS*2/3))
+fi
+
+set -e
+./gradlew --parallel --scan \
+  -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ \
+  --parallel --max-workers=$MAX_WORKERS \
+   "$@"

.ci/bwcVersions

@@ -10,6 +10,11 @@ BWC_VERSION:
   - "7.3.2"
   - "7.4.0"
   - "7.4.1"
+  - "7.4.2"
   - "7.5.0"
+  - "7.5.1"
+  - "7.5.2"
+  - "7.5.3"
   - "7.6.0"
+  - "7.7.0"
   - "8.0.0"

.ci/dockerOnLinuxExclusions

@@ -0,0 +1,12 @@
+# This file specifies the Linux OS versions on which we can't build and
+# test Docker images for some reason. These values correspond to ID and
+# VERSION_ID from /etc/os-release, and a matching value will cause the
+# Docker tests to be skipped on that OS. If /etc/os-release doesn't exist
+# (as is the case on centos-6, for example) then that OS will again be
+# excluded.
+centos-6
+debian-8
+opensuse-15-1
+ol-6.10
+ol-7.7
+sles-12

.ci/init.gradle

@@ -1,114 +1,114 @@
-import com.bettercloud.vault.VaultConfig;
-import com.bettercloud.vault.Vault;
+import com.bettercloud.vault.VaultConfig
+import com.bettercloud.vault.Vault
 
 initscript {
-    repositories {
-        mavenCentral()
-    }
-    dependencies {
-        classpath 'com.bettercloud:vault-java-driver:4.1.0'
-    }
+  repositories {
+    mavenCentral()
+  }
+  dependencies {
+    classpath 'com.bettercloud:vault-java-driver:4.1.0'
+  }
 }
 
-boolean USE_ARTIFACTORY=false
+boolean USE_ARTIFACTORY = false
 
 if (System.getenv('VAULT_ADDR') == null) {
-    throw new GradleException("You must set the VAULT_ADDR environment variable to use this init script.")
+  throw new GradleException("You must set the VAULT_ADDR environment variable to use this init script.")
 }
 
 if (System.getenv('VAULT_ROLE_ID') == null && System.getenv('VAULT_SECRET_ID') == null && System.getenv('VAULT_TOKEN') == null) {
-    throw new GradleException("You must set either the VAULT_ROLE_ID and VAULT_SECRET_ID environment variables, " +
-            "or the VAULT_TOKEN environment variable to use this init script.")
+  throw new GradleException("You must set either the VAULT_ROLE_ID and VAULT_SECRET_ID environment variables, " +
+    "or the VAULT_TOKEN environment variable to use this init script.")
 }
 
 final String vaultToken = System.getenv('VAULT_TOKEN') ?: new Vault(
-        new VaultConfig()
-            .address(System.env.VAULT_ADDR)
-            .engineVersion(1)
-            .build()
-    )
-    .withRetries(5, 1000)
-    .auth()
-    .loginByAppRole("approle", System.env.VAULT_ROLE_ID, System.env.VAULT_SECRET_ID)
-    .getAuthClientToken();
+  new VaultConfig()
+    .address(System.env.VAULT_ADDR)
+    .engineVersion(1)
+    .build()
+)
+  .withRetries(5, 1000)
+  .auth()
+  .loginByAppRole("approle", System.env.VAULT_ROLE_ID, System.env.VAULT_SECRET_ID)
+  .getAuthClientToken()
 
 final Vault vault = new Vault(
-     new VaultConfig()
-        .address(System.env.VAULT_ADDR)
-        .engineVersion(1)
-        .token(vaultToken)
-        .build()
+  new VaultConfig()
+    .address(System.env.VAULT_ADDR)
+    .engineVersion(1)
+    .token(vaultToken)
+    .build()
 )
-        .withRetries(5, 1000)
+  .withRetries(5, 1000)
 
 
 if (USE_ARTIFACTORY) {
-    final Map<String,String> artifactoryCredentials = vault.logical()
-            .read("secret/elasticsearch-ci/artifactory.elstc.co")
-            .getData();
-    logger.info("Using elastic artifactory repos")
-    Closure configCache = {
-        return {
-            name "artifactory-gradle-release"
-            url "https://artifactory.elstc.co/artifactory/gradle-release"
-            credentials {
-                username artifactoryCredentials.get("username")
-                password artifactoryCredentials.get("token")
-            }
-        }
+  final Map<String, String> artifactoryCredentials = vault.logical()
+    .read("secret/elasticsearch-ci/artifactory.elstc.co")
+    .getData()
+  logger.info("Using elastic artifactory repos")
+  Closure configCache = {
+    return {
+      name "artifactory-gradle-release"
+      url "https://artifactory.elstc.co/artifactory/gradle-release"
+      credentials {
+        username artifactoryCredentials.get("username")
+        password artifactoryCredentials.get("token")
+      }
     }
-    settingsEvaluated { settings ->
-        settings.pluginManagement {
-            repositories {
-                maven configCache()
-            }
-        }
+  }
+  settingsEvaluated { settings ->
+    settings.pluginManagement {
+      repositories {
+        maven configCache()
+      }
     }
-    projectsLoaded {
-        allprojects {
-            buildscript {
-                repositories {
-                    maven configCache()
-                }
-            }
-            repositories {
-                maven configCache()
-            }
+  }
+  projectsLoaded {
+    allprojects {
+      buildscript {
+        repositories {
+          maven configCache()
         }
+      }
+      repositories {
+        maven configCache()
+      }
     }
+  }
 }
 
 projectsLoaded {
-    rootProject {
-        project.pluginManager.withPlugin('com.gradle.build-scan') {
-            buildScan.server = 'https://gradle-enterprise.elastic.co'
-        }
+  rootProject {
+    project.pluginManager.withPlugin('com.gradle.build-scan') {
+      buildScan.server = 'https://gradle-enterprise.elastic.co'
     }
+  }
 }
 
 
 final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url')
 final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false'))
 
 if (buildCacheUrl) {
-    final Map<String,String> buildCacheCredentials = vault.logical()
-            .read("secret/elasticsearch-ci/gradle-build-cache")
-            .getData();
-    gradle.settingsEvaluated { settings ->
-        settings.buildCache {
-            local {
-                // Disable the local build cache in CI since we use ephemeral workers and it incurs an IO penalty
-                enabled = false
-            }
-            remote(HttpBuildCache) {
-                url = buildCacheUrl
-                push = buildCachePush
-                credentials {
-                        username = buildCacheCredentials.get("username")
-                        password = buildCacheCredentials.get("password")
-                }
-            }
+  final Map<String, String> buildCacheCredentials = vault.logical()
+    .read("secret/elasticsearch-ci/gradle-build-cache")
+    .getData()
+  gradle.settingsEvaluated { settings ->
+    settings.buildCache {
+      local {
+        // Disable the local build cache in CI since we use ephemeral workers and it incurs an IO penalty
+        enabled = false
+      }
+      remote(HttpBuildCache) {
+        url = buildCacheUrl
+        push = buildCachePush
+        credentials {
+          username = buildCacheCredentials.get("username")
+          password = buildCacheCredentials.get("password")
         }
+      }
     }
+  }
 }
 

.ci/java-versions.properties

@@ -4,8 +4,6 @@
 # build and test Elasticsearch for this branch. Valid Java versions
 # are 'java' or 'openjdk' followed by the major release number.
 
-ES_BUILD_JAVA=openjdk12
+ES_BUILD_JAVA=openjdk13
 ES_RUNTIME_JAVA=openjdk11
 GRADLE_TASK=build
-
-

.ci/jobs.t/defaults.yml

@@ -0,0 +1,93 @@
+---
+
+##### GLOBAL METADATA
+
+- meta:
+    cluster: elasticsearch-ci
+
+##### JOB DEFAULTS
+
+- job:
+    vault:
+      url: https://secrets.elastic.co:8200
+      role_id: 1ba1ac3e-aee4-d040-d9a3-6ae23bd2b3db
+    node: "general-purpose"
+    concurrent: true
+    logrotate:
+      daysToKeep: 30
+      numToKeep: 90
+      artifactDaysToKeep: 7
+    scm:
+      - git:
+          name: origin
+          # master node jenkins user ~/.ssh
+          credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+          reference-repo: "/var/lib/jenkins/.git-references/elasticsearch.git"
+          branches:
+            - "%BRANCH%"
+          url: "https://github.com/elastic/elasticsearch.git"
+          basedir: ""
+          wipe-workspace: "yes"
+    wrappers:
+      - timeout:
+          type: absolute
+          timeout: "120"
+          fail: true
+      - ansicolor
+      - timestamps
+      - gradle-build-scan
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            WORKSPACE=/dev/shm/$JOB_NAME
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA7_HOME=$HOME/.java/java7
+            JAVA8_HOME=$HOME/.java/java8
+            JAVA9_HOME=$HOME/.java/java9
+            JAVA10_HOME=$HOME/.java/java10
+            JAVA11_HOME=$HOME/.java/java11
+            JAVA12_HOME=$HOME/.java/openjdk12
+            JAVA13_HOME=$HOME/.java/openjdk13
+            GRADLE_OPTS=-XX:+HeapDumpOnOutOfMemoryError -Xmx128m -Xms128m
+    properties:
+        - github:
+            url: https://github.com/elastic/elasticsearch/
+        - inject:
+            properties-content: |
+              HOME=$JENKINS_HOME
+    builders:
+       - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr --last-good-commit elastic+elasticsearch+%BRANCH%+git+push
+          .ci/build.sh -Dbwc.checkout.align=true check
+    publishers:
+      - email:
+          recipients: infra-root+build@elastic.co
+      # Upload additional logs
+      - google-cloud-storage:
+          credentials-id: 'elasticsearch-ci-gcs-plugin'
+          uploads:
+            - classic:
+                file-pattern: 'build/*.tar.bz2'
+                storage-location: 'gs://elasticsearch-ci-artifacts/jobs/$JOB_NAME'
+                share-publicly: false
+                upload-for-failed-jobs: true
+                show-inline: true
+      # Notify homer
+      - postbuildscript:
+          builders:
+            - role: SLAVE
+              build-on:
+                  - SUCCESS
+                  - FAILURE
+                  - UNSTABLE
+              build-steps:
+                - shell: |
+                    curl -sS -X POST \
+                      -H "Content-Type: text/plain" \
+                      --data "$BUILD_URL" \
+                      "https://homer.app.elstc.co/webhook/jenkins/build-finished" || true
+
+
+
+

.ci/jobs.t/elastic+elasticsearch+{branch}+periodic-next.yml

@@ -0,0 +1,6 @@
+- job:
+    name: elastic+elasticsearch+%BRANCH%+periodic-next
+    workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic
+    display-name: "elastic / elasticsearch # %BRANCH% - periodic (experimental)"
+    description: "Periodic testing of the Elasticsearch %BRANCH% branch.\n"
+    triggers: []

.ci/make-branch-config.sh

@@ -0,0 +1,11 @@
+#!/bin/bash 
+
+if [ -z "$BRANCH" ] ; then 
+    echo "BRANCH is unset"
+    exit 1
+fi 
+
+rm -Rf .ci/jobs
+cp -r .ci/jobs.t .ci/jobs
+
+sed -i "s/%BRANCH%/${BRANCH}/g" .ci/jobs/*.yml

.ci/matrix-build-javas.yml

@@ -6,4 +6,4 @@
 # or 'openjdk' followed by the major release number.
 
 ES_BUILD_JAVA:
-  - openjdk12
+  - openjdk13