Merge branch 'master' into ingest/input-vars-templates

.ci/teamcity/bootstrap.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/util.sh"
+
+tc_start_block "Bootstrap"
+
+tc_start_block "yarn install and kbn bootstrap"
+verify_no_git_changes yarn kbn bootstrap --prefer-offline
+tc_end_block "yarn install and kbn bootstrap"
+
+tc_start_block "build kbn-pm"
+verify_no_git_changes yarn kbn run build -i @kbn/pm
+tc_end_block "build kbn-pm"
+
+tc_start_block "build plugin list docs"
+verify_no_git_changes node scripts/build_plugin_list_docs
+tc_end_block "build plugin list docs"
+
+tc_end_block "Bootstrap"

.ci/teamcity/checks/bundle_limits.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+node scripts/build_kibana_platform_plugins --validate-limits

.ci/teamcity/checks/doc_api_changes.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:checkDocApiChanges

.ci/teamcity/checks/file_casing.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:checkFileCasing

.ci/teamcity/checks/i18n.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:i18nCheck

.ci/teamcity/checks/licenses.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:licenses

.ci/teamcity/checks/telemetry.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:telemetryCheck

.ci/teamcity/checks/test_hardening.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:test_hardening

.ci/teamcity/checks/ts_projects.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:checkTsProjects

.ci/teamcity/checks/type_check.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:typeCheck

.ci/teamcity/checks/verify_dependency_versions.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:verifyDependencyVersions

.ci/teamcity/checks/verify_notice.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:verifyNotice

.ci/teamcity/ci_stats.js

@@ -0,0 +1,59 @@
+const https = require('https');
+const token = process.env.CI_STATS_TOKEN;
+const host = process.env.CI_STATS_HOST;
+
+const request = (url, options, data = null) => {
+  const httpOptions = {
+    ...options,
+    headers: {
+      ...(options.headers || {}),
+      Authorization: `token ${token}`,
+    },
+  };
+
+  return new Promise((resolve, reject) => {
+    console.log(`Calling https://${host}${url}`);
+
+    const req = https.request(`https://${host}${url}`, httpOptions, (res) => {
+      if (res.statusCode < 200 || res.statusCode >= 300) {
+        return reject(new Error(`Status Code: ${res.statusCode}`));
+      }
+
+      const data = [];
+      res.on('data', (d) => {
+        data.push(d);
+      })
+
+      res.on('end', () => {
+        try {
+          let resp = Buffer.concat(data).toString();
+
+          try {
+            if (resp.trim()) {
+              resp = JSON.parse(resp);
+            }
+          } catch (ex) {
+            console.error(ex);
+          }
+
+          resolve(resp);
+        } catch (ex) {
+          reject(ex);
+        }
+      });
+    })
+
+    req.on('error', reject);
+
+    if (data) {
+      req.write(JSON.stringify(data));
+    }
+
+    req.end();
+  });
+}
+
+module.exports = {
+  get: (url) => request(url, { method: 'GET' }),
+  post: (url, data) => request(url, { method: 'POST' }, data),
+}

.ci/teamcity/ci_stats_complete.js

@@ -0,0 +1,18 @@
+const ciStats = require('./ci_stats');
+
+// This might be better as an API call in the future.
+// Instead, it relies on a separate step setting the BUILD_STATUS env var. BUILD_STATUS is not something provided by TeamCity.
+const BUILD_STATUS = process.env.BUILD_STATUS === 'SUCCESS' ? 'SUCCESS' : 'FAILURE';
+
+(async () => {
+  try {
+    if (process.env.CI_STATS_BUILD_ID) {
+      await ciStats.post(`/v1/build/_complete?id=${process.env.CI_STATS_BUILD_ID}`, {
+        result: BUILD_STATUS,
+      });
+    }
+  } catch (ex) {
+    console.error(ex);
+    process.exit(1);
+  }
+})();

.ci/teamcity/default/accessibility.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+export JOB=kibana-default-accessibility
+export KIBANA_INSTALL_DIR="$PARENT_DIR/build/kibana-build-default"
+
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "X-Pack accessibility tests" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+    --config test/accessibility/config.ts

.ci/teamcity/default/build.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+tc_start_block "Build Platform Plugins"
+node scripts/build_kibana_platform_plugins \
+  --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
+  --scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \
+  --scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \
+  --scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_api_perf/plugins" \
+  --scan-dir "$XPACK_DIR/test/licensing_plugin/plugins" \
+  --verbose
+tc_end_block "Build Platform Plugins"
+
+export KBN_NP_PLUGINS_BUILT=true
+
+tc_start_block "Build Default Distribution"
+
+cd "$KIBANA_DIR"
+node scripts/build --debug --no-oss
+linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
+installDir="$KIBANA_DIR/install/kibana"
+mkdir -p "$installDir"
+tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+
+tc_end_block "Build Default Distribution"

.ci/teamcity/default/build_plugins.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+tc_start_block "Build Platform Plugins"
+node scripts/build_kibana_platform_plugins \
+  --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
+  --scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \
+  --scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \
+  --scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_api_perf/plugins" \
+  --scan-dir "$XPACK_DIR/test/licensing_plugin/plugins" \
+  --verbose
+tc_end_block "Build Platform Plugins"
+
+tc_set_env KBN_NP_PLUGINS_BUILT true

.ci/teamcity/default/ci_group.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+export CI_GROUP="$1"
+export JOB=kibana-default-ciGroup${CI_GROUP}
+export KIBANA_INSTALL_DIR="$PARENT_DIR/build/kibana-build-default"
+
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "Default Distro Chrome Functional tests / Group ${CI_GROUP}" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+    --include-tag "ciGroup$CI_GROUP"

.ci/teamcity/default/firefox.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+export JOB=kibana-default-firefoxSmoke
+export KIBANA_INSTALL_DIR="$PARENT_DIR/build/kibana-build-default"
+
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "X-Pack firefox smoke test" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+    --include-tag "includeFirefox" \
+    --config test/functional/config.firefox.js \
+    --config test/functional_embedded/config.firefox.ts

.ci/teamcity/default/saved_object_field_metrics.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+export JOB=kibana-default-savedObjectFieldMetrics
+export KIBANA_INSTALL_DIR="$PARENT_DIR/build/kibana-build-default"
+
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "Capture Kibana Saved Objects field count metrics" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+    --config test/saved_objects_field_count/config.ts

.ci/teamcity/default/security_solution.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+export JOB=kibana-default-securitySolution
+export KIBANA_INSTALL_DIR="$PARENT_DIR/build/kibana-build-default"
+
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "Security Solution Cypress Tests" \
+ node scripts/functional_tests \
+   --debug --bail \
+   --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+   --config test/security_solution_cypress/cli_config.ts

.ci/teamcity/es_snapshots/build.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+cd ..
+destination="$(pwd)/es-build"
+mkdir -p "$destination"
+
+cd elasticsearch
+
+# These turn off automation in the Elasticsearch repo
+export BUILD_NUMBER=""
+export JENKINS_URL=""
+export BUILD_URL=""
+export JOB_NAME=""
+export NODE_NAME=""
+
+# Reads the ES_BUILD_JAVA env var out of .ci/java-versions.properties and exports it
+export "$(grep '^ES_BUILD_JAVA' .ci/java-versions.properties | xargs)"
+
+export PATH="$HOME/.java/$ES_BUILD_JAVA/bin:$PATH"
+export JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA"
+
+tc_start_block "Build Elasticsearch"
+./gradlew -Dbuild.docker=true assemble --parallel
+tc_end_block "Build Elasticsearch"
+
+tc_start_block "Create distribution archives"
+find distribution -type f \( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \) -not -path '*no-jdk*' -not -path '*build-context*' -exec cp {} "$destination" \;
+tc_end_block "Create distribution archives"
+
+ls -alh "$destination"
+
+tc_start_block "Create docker image archives"
+docker images "docker.elastic.co/elasticsearch/elasticsearch"
+docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 echo 'docker save docker.elastic.co/elasticsearch/elasticsearch:${0} | gzip > ../es-build/elasticsearch-${0}-docker-image.tar.gz'
+docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 bash -c 'docker save docker.elastic.co/elasticsearch/elasticsearch:${0} | gzip > ../es-build/elasticsearch-${0}-docker-image.tar.gz'
+tc_end_block "Create docker image archives"
+
+cd "$destination"
+
+find ./* -exec bash -c "shasum -a 512 {} > {}.sha512" \;
+ls -alh "$destination"