[CI] Migrates CI to Buildkite

elastic · Jul 17, 2023 · 7487d26 · 7487d26
1 parent 5bc1a82
commit 7487d26
Show file tree

Hide file tree

Showing 7 changed files with 59 additions and 4 deletions.
diff --git a/.ci/Dockerfile → .buildkite/Dockerfile b/.ci/Dockerfile → .buildkite/Dockerfile
diff --git a/.buildkite/build-documentation.sh b/.buildkite/build-documentation.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+sudo apt-get update
+sudo apt-get install -y pandoc python3 python3-pip
+python3 -m pip install nox
+/opt/buildkite-agent/.local/bin/nox -s docs
diff --git a/.buildkite/lint-code.sh b/.buildkite/lint-code.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+docker build --file .buildkite/Dockerfile --tag elastic/eland --build-arg PYTHON_VERSION=${PYTHON_VERSION} .
+docker run \
+       --name linter \
+       --rm \
+       elastic/eland \
+       nox -s lint
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -0,0 +1,38 @@
+steps:
+  - label: ":terminal: Lint code"
+    env:
+      PYTHON_VERSION: 3
+    agents:
+      provider: "gcp"
+    commands:
+      - ./.buildkite/lint-code.sh
+  - label: ":books: Build documentation"
+    env:
+      PYTHON_VERSION: 3.9-bookworm
+    agents:
+      provider: "gcp"
+    commands:
+      - ./.buildkite/build-documentation.sh
+  - label: "Eland :python: {{ matrix.python }} :elasticsearch: {{ matrix.stack }}"
+    agents:
+      provider: "gcp"
+    env:
+      PYTHON_VERSION: "{{ matrix.python }}"
+      PANDAS_VERSION: "{{ matrix.pandas }}"
+      TEST_SUITE: "xpack"
+      ELASTICSEARCH_VERSION: "{{ matrix.stack }}"
+    matrix:
+      setup:
+        python:
+          - '3.10'
+          - '3.9'
+          - '3.8'
+        pandas:
+          - '1.2.0'
+          - '1.3.0'
+        stack:
+          - '7.x-SNAPSHOT'
+          - '7.11-SNAPSHOT'
+          - '7.14-SNAPSHOT'
+          - '8.0-SNAPSHOT'
+    command: ./.buildkite/run-tests
diff --git a/.ci/run-elasticsearch.sh → .buildkite/run-elasticsearch.sh b/.ci/run-elasticsearch.sh → .buildkite/run-elasticsearch.sh
@@ -37,6 +37,11 @@ NETWORK_NAME=${NETWORK_NAME-"$network_default"}
 
 set +x
 
+# Set vm.max_map_count kernel setting to 262144 if we're in CI
+if [[ "$BUILDKITE" == "true" ]]; then
+  sudo sysctl -w vm.max_map_count=262144
+fi
+
 function cleanup_volume {
   if [[ "$(docker volume ls -q -f name=$1)" ]]; then
     echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m"

diff --git a/.ci/run-repository.sh → .buildkite/run-repository.sh b/.ci/run-repository.sh → .buildkite/run-repository.sh
@@ -25,7 +25,7 @@ echo -e "\033[34;1mINFO:\033[0m PANDAS_VERSION ${PANDAS_VERSION}\033[0m"
 
 echo -e "\033[1m>>>>> Build [elastic/eland container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
 
-docker build --file .ci/Dockerfile --tag elastic/eland --build-arg PYTHON_VERSION=${PYTHON_VERSION} .
+docker build --file .buildkite/Dockerfile --tag elastic/eland --build-arg PYTHON_VERSION=${PYTHON_VERSION} .
 
 echo -e "\033[1m>>>>> Run [elastic/eland container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
 

diff --git a/.ci/run-tests → .buildkite/run-tests b/.ci/run-tests → .buildkite/run-tests
@@ -29,7 +29,7 @@ function cleanup {
     NODE_NAME=${NODE_NAME} \
     NETWORK_NAME=elasticsearch \
     CLEANUP=true \
-    bash ./.ci/run-elasticsearch.sh
+    bash ./.buildkite/run-elasticsearch.sh
   # Report status and exit
   if [[ "$status" == "0" ]]; then
     echo -e "\n\033[32;1mSUCCESS run-tests\033[0m"
@@ -47,7 +47,7 @@ ELASTICSEARCH_VERSION=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \
   NODE_NAME=${NODE_NAME} \
   NETWORK_NAME=host \
   DETACH=true \
-  bash .ci/run-elasticsearch.sh
+  bash .buildkite/run-elasticsearch.sh
 
 echo -e "\033[1m>>>>> Repository specific tests >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
 
@@ -57,5 +57,5 @@ ELASTICSEARCH_CONTAINER=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \
   ELASTICSEARCH_URL=${elasticsearch_url} \
   TEST_SUITE=${TEST_SUITE} \
   PANDAS_VERSION=${PANDAS_VERSION} \
-  bash .ci/run-repository.sh
+  bash .buildkite/run-repository.sh