Move periodic job to ES repo (#48570)

* Move periodic job to ES repo

This change kickstarts the process of moving CI job definitions to this
repo.

* Added a minimal readme to provide pointers to the documentation

* Update .ci/README.md

Co-Authored-By: Rory Hunter <pugnascotia@users.noreply.github.com>

* Update .ci/README.md

Co-Authored-By: Rory Hunter <pugnascotia@users.noreply.github.com>

* point to main repo

* PR review

* Add link to JJBB

.ci/README.md

@@ -0,0 +1,16 @@
+CI configuration for Elasticsearch
+==================================
+
+Layout and Syntax
+-----------------
+
+CI is run by Jenkins at [elasticsearch-ci](https://elasticsearch-ci.elastic.co/).
+Jobs live in the [jobs.t](jobs.t) directory, these are defined in YML using a syntax 
+simmilar to [JJB](https://elasticsearch-ci.elastic.co/view/Elasticsearch%20master/).
+Macros are not allowed, and each job must be defined in its own file.
+Merging of the default configuration is customized so unlike in standard JJB,
+it recurses into YML objects. 
+Further (internal) documentation  on the setup 
+[is available](https://github.com/elastic/infra/blob/master/flavortown/jjbb/README.md) 
+.
+

.ci/build.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# drop page cache and kernel slab objects on linux
+[[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches
+
+rm -Rfv ~/.gradle/init.d/init.gradle
+mkdir -p ~/.gradle/init.d && cp -v $WORKSPACE/.ci/init.gradle ~/.gradle/init.d
+
+if [ -f /proc/cpuinfo ] ; then
+   MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo  | uniq | sed 's/\s\+//g' |  cut -d':' -f 2`
+else
+   if [[ "$OSTYPE" == "darwin"* ]]; then
+      MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'`
+      # Looks like it's too much for our workers so reduce it further
+      MAX_WORKERS=$(($MAX_WORKERS/2))
+   else
+      echo "Unsupported OS Type: $OSTYPE"
+      exit 1
+   fi
+fi
+
+if pwd | grep -v -q ^/dev/shm ; then
+   echo "Not running on a ramdisk, reducing number of workers"
+   MAX_WORKERS=$(($MAX_WORKERS*2/3))
+fi
+
+export GRADLE_OPTS="-XX:+HeapDumpOnOutOfMemoryError -Xmx128m -Xms128m"
+set -e
+./gradlew --parallel --scan \
+  -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ \
+  --parallel --max-workers=$MAX_WORKERS \
+   "$@"

.ci/jobs.t/defaults.yml

@@ -0,0 +1,92 @@
+---
+
+##### GLOBAL METADATA
+
+- meta:
+    cluster: elasticsearch-ci
+
+##### JOB DEFAULTS
+
+- job:
+    vault:
+      url: https://secrets.elastic.co:8200
+      role_id: 1ba1ac3e-aee4-d040-d9a3-6ae23bd2b3db
+    node: "general-purpose"
+    concurrent: true
+    logrotate:
+      daysToKeep: 30
+      numToKeep: 90
+      artifactDaysToKeep: 7
+    scm:
+      - git:
+          name: origin
+          # master node jenkins user ~/.ssh
+          credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+          reference-repo: "/var/lib/jenkins/.git-references/elasticsearch.git"
+          branches:
+            - "%BRANCH%"
+          url: "https://github.com/elastic/elasticsearch.git"
+          basedir: ""
+          wipe-workspace: "yes"
+    wrappers:
+      - timeout:
+          type: absolute
+          timeout: "120"
+          fail: true
+      - ansicolor
+      - timestamps
+      # TODO: No support un JJBB ?
+      # - gradle-build-scan
+    properties:
+        - github:
+            url: https://github.com/elastic/elasticsearch/
+        - inject:
+            properties-content: |
+              HOME=$JENKINS_HOME
+    builders:
+       - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA7_HOME=$HOME/.java/java7
+            JAVA8_HOME=$HOME/.java/java8
+            JAVA9_HOME=$HOME/.java/java9
+            JAVA10_HOME=$HOME/.java/java10
+            JAVA11_HOME=$HOME/.java/java11
+            JAVA12_HOME=$HOME/.java/openjdk12
+            JAVA13_HOME=$HOME/.java/openjdk13
+       - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr --last-good-commit elastic+elasticsearch+%BRANCH%+git+push
+          .ci/build.sh check
+    publishers:
+      - email:
+          recipients: infra-root+build@elastic.co
+      # Upload additional logs
+      - google-cloud-storage:
+          credentials-id: 'elasticsearch-ci-gcs-plugin'
+          uploads:
+            - classic:
+                file-pattern: 'build/*.tar.bz2'
+                storage-location: 'gs://elasticsearch-ci-artifacts/jobs/$JOB_NAME'
+                share-publicly: false
+                upload-for-failed-jobs: true
+                show-inline: true
+      # Notify homer
+      - postbuildscript:
+          builders:
+            - role: SLAVE
+              build-on:
+                  - SUCCESS
+                  - FAILURE
+                  - UNSTABLE
+              build-steps:
+                - shell: |
+                    curl -sS -X POST \
+                      -H "Content-Type: text/plain" \
+                      --data "$BUILD_URL" \
+                      "https://homer.app.elstc.co/webhook/jenkins/build-finished" || true
+
+
+
+            

.ci/jobs.t/elastic+elasticsearch+{branch}+periodic-next.yml

@@ -0,0 +1,11 @@
+- job:
+    name: elastic+elasticsearch+%BRANCH%+periodic-next
+    workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic
+    display-name: "elastic / elasticsearch # %BRANCH% - periodic (experimental)"
+    description: "Periodic testing of the Elasticsearch %BRANCH% branch.\n"
+    triggers:
+      - timed: "H H/1 * * *"
+    builders:
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr --last-good-commit elastic+elasticsearch+%BRANCH%+git+push
+          .ci/build.sh -Dbwc.checkout.align=true check

.ci/make-branch-config.sh

@@ -0,0 +1,11 @@
+#!/bin/bash 
+
+if [ -z "$BRANCH" ] ; then 
+    echo "BRANCH is unset"
+    exit 1
+fi 
+
+rm -Rf .ci/jobs
+cp -r .ci/jobs.t .ci/jobs
+
+sed -i "s/%BRANCH%/${BRANCH}/g" .ci/jobs/*.yml

.gitignore

@@ -41,3 +41,9 @@ html_docs
 # random old stuff that we should look at the necessity of...
 /tmp/
 eclipse-build
+
+# projects using testfixtures 
+testfixtures_shared/
+
+# These are generated from .ci/jobs.t
+.ci/jobs/