[8.x] Backport of #190, dynamic CI pr/build/e2e jobs for target branch.

.buildkite/build-pipeline.yml

@@ -1 +1,30 @@
-# Intentional empty definition because it is in 8.x
+- label: "Build pipeline"
+  command: |
+    #!/usr/bin/env bash
+    set -eo pipefail
+
+    echo "--- Downloading prerequisites"
+    python3 -m pip install ruamel.yaml
+    python3 -m pip install requests
+    curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+    chmod a+x /usr/bin/yq
+
+    export TARGET_BRANCH="$BUILDKITE_BRANCH"
+
+    echo "--- Generating steps dynamically"
+    set +e
+    python3 .buildkite/scripts/build-pipeline/generate-steps.py > pipeline_steps.yml
+    if [[ $$? -ne 0 ]]; then
+      echo "^^^ +++"
+      echo "There was a problem with generating pipeline steps."
+      cat pipeline_steps.yml
+      echo "Exiting now."
+      exit 1
+    else
+      set -eo pipefail
+      cat pipeline_steps.yml | yq .
+    fi
+
+    set -eo pipefail
+    echo "--- Uploading steps to buildkite"
+    cat pipeline_steps.yml | buildkite-agent pipeline upload

.buildkite/e2e-pipeline.yml

@@ -1 +1,30 @@
-# Intentional empty definition because it is in 8.x
+- label: "E2E pipeline"
+  command: |
+    #!/usr/bin/env bash
+    set -eo pipefail
+
+    echo "--- Downloading prerequisites"
+    python3 -m pip install ruamel.yaml
+    python3 -m pip install requests
+    curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+    chmod a+x /usr/bin/yq
+
+    export TARGET_BRANCH="$BUILDKITE_BRANCH"
+
+    echo "--- Generating steps dynamically"
+    set +e
+    python3 .buildkite/scripts/e2e-pipeline/generate-steps.py > pipeline_steps.yml
+    if [[ $$? -ne 0 ]]; then
+      echo "^^^ +++"
+      echo "There was a problem with generating pipeline steps."
+      cat pipeline_steps.yml
+      echo "Exiting now."
+      exit 1
+    else
+      set -eo pipefail
+      cat pipeline_steps.yml | yq .
+    fi
+
+    set -eo pipefail
+    echo "--- Uploading steps to buildkite"
+    cat pipeline_steps.yml | buildkite-agent pipeline upload

.buildkite/pull-request-pipeline.yml

@@ -1,53 +1,29 @@
-# define a GCP VM agent to support container management (by default agent doesn't support)
-agents:
-  provider: "gcp"
-  machineType: "n1-standard-4"
-  image: family/core-ubuntu-2204
-
-steps:
-  # ------------- Unit tests ---------------------
-  - label: ":hammer: Unit tests with LS & ES 8.current :docker:"
-    # Builds the plugin (with current changes) with the version defined in gradle.properties
-    #   and against LS and ES 8.current
-    # Runs unit tests on LS & ES 8.current docker
-    command:
-      - .buildkite/scripts/run_tests.sh
-    env:
-      ELASTIC_STACK_VERSION: "8.current"
-      INTEGRATION: false
-
-  - label: ":hammer: Unit tests with LS & ES 8.current-SNAPSHOT :docker:"
-    # Builds the plugin (with current changes) with the version defined in gradle.properties
-    #   and against LS and ES 8.current-SNAPSHOT
-    # Runs unit tests on LS & ES 8.current-SNAPSHOT docker
-    command:
-      - .buildkite/scripts/run_tests.sh
-    env:
-      ELASTIC_STACK_VERSION: "8.current"
-      SNAPSHOT: true
-      INTEGRATION: false
-
-  # ------------- Integration tests ---------------------
-  - label: ":hammer: Integration tests with LS & ES 8.current :docker:"
-    # Builds the plugin (with current changes) with the version defined in gradle.properties
-    #   and against LS and ES 8.current
-    # Runs integration tests on LS & ES 8.current docker
-    command:
-      - .buildkite/scripts/run_tests.sh
-    env:
-      ELASTIC_STACK_VERSION: "8.current"
-      INTEGRATION: true
-      SECURE_INTEGRATION: true
-
-  - label: ":hammer: Integration tests with LS & ES 8.current-SNAPSHOT :docker:"
-    # Builds the plugin (with current changes) with the version defined in gradle.properties
-    #   and against LS and ES 8.current-SNAPSHOT
-    # Runs integration tests on LS & ES 8.current-SNAPSHOT docker
-    command:
-      - .buildkite/scripts/run_tests.sh
-    env:
-      ELASTIC_STACK_VERSION: "8.current"
-      SNAPSHOT: true
-      INTEGRATION: true
-      SECURE_INTEGRATION: true
-      LOG_LEVEL: "info"
+- label: "Pull request pipeline"
+  command: |
+    #!/usr/bin/env bash
+    set -eo pipefail
+    echo "--- Downloading prerequisites"
+    python3 -m pip install ruamel.yaml
+    python3 -m pip install requests
+    curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+    chmod a+x /usr/bin/yq
+
+    export TARGET_BRANCH="$GITHUB_PR_TARGET_BRANCH"
+
+    echo "--- Generating steps dynamically"
+    set +e
+    python3 .buildkite/scripts/pull-request-pipeline/generate-steps.py > pipeline_steps.yml
+    if [[ $$? -ne 0 ]]; then
+      echo "^^^ +++"
+      echo "There was a problem with generating pipeline steps."
+      cat pipeline_steps.yml
+      echo "Exiting now."
+      exit 1
+    else
+      set -eo pipefail
+      cat pipeline_steps.yml | yq .
+    fi
+
+    set -eo pipefail
+    echo "--- Uploading steps to buildkite"
+    cat pipeline_steps.yml | buildkite-agent pipeline upload

.buildkite/scripts/build-pipeline/generate-steps.py

@@ -0,0 +1,102 @@
+import os
+import requests
+import sys
+import typing
+from requests.adapters import HTTPAdapter, Retry
+
+from ruamel.yaml import YAML
+
+RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
+TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh"
+
+
+def generate_unit_and_integration_test_steps(stack_version, snapshot) -> list[typing.Any]:
+    test_steps = []
+
+    # step-1, unit tests
+    label_unit_test: typing.final = f"Unit test for {stack_version}, snapshot: {snapshot}"
+    test_steps.append({
+        "label": label_unit_test,
+        "command": TEST_COMMAND,
+        "env": {
+            "SNAPSHOT": snapshot,
+            "ELASTIC_STACK_VERSION": stack_version,
+            "INTEGRATION": "false"
+        }
+    })
+
+    # step-2, integration tests
+    label_integration_test: typing.final = f"Integration test for {stack_version}, snapshot: {snapshot}"
+    test_steps.append({
+        "label": label_integration_test,
+        "command": TEST_COMMAND,
+        "env": {
+            "SNAPSHOT": snapshot,
+            "ELASTIC_STACK_VERSION": stack_version,
+            "INTEGRATION": "true",
+            "SECURE_INTEGRATION": "true",
+            "LOG_LEVEL": "info"
+        }
+    })
+    return test_steps
+
+
+def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> requests.Response:
+    schema = "https://" if "https://" in url else "http://"
+    session = requests.Session()
+    # retry on most common failures such as connection timeout(408), etc...
+    retries = Retry(total=max_retries, backoff_factor=delay, status_forcelist=[408, 502, 503, 504])
+    session.mount(schema, HTTPAdapter(max_retries=retries))
+    return session.get(url)
+
+
+if __name__ == "__main__":
+    structure = {
+        "agents": {
+            "provider": "gcp",
+            "machineType": "n1-standard-4",
+            "image": "family/core-ubuntu-2204"
+        },
+        "steps": []}
+
+    steps = []
+    response = call_url_with_retry(RELEASES_URL)
+    versions_json = response.json()
+
+    # there are situations to manually run CIs with PR change,
+    # set MANUAL_TARGET_BRANCH with upstream target branch and run
+    manually_set_target_branch: typing.final = os.getenv("MANUAL_TARGET_BRANCH")
+    target_branch: typing.final = manually_set_target_branch if manually_set_target_branch else os.getenv("TARGET_BRANCH")
+    print(f"Running with target_branch: {target_branch}")
+    if target_branch == '8.x':
+        full_stack_version: typing.final = versions_json["snapshots"]["8.future"]
+        steps += generate_unit_and_integration_test_steps(full_stack_version, "true")
+    elif target_branch == 'main':
+        full_stack_version: typing.final = versions_json["snapshots"][target_branch]
+        steps += generate_unit_and_integration_test_steps(full_stack_version, "true")
+    else:
+        # generate steps for the version if released
+        releases = versions_json["releases"]
+        for release_version in releases:
+            if releases[release_version].startswith(target_branch):
+                steps += generate_unit_and_integration_test_steps(releases[release_version], "false")
+                break
+
+        # steps for snapshot version
+        snapshots = versions_json["snapshots"]
+        for snapshot_version in snapshots:
+            if snapshots[snapshot_version].startswith(target_branch):
+                steps += generate_unit_and_integration_test_steps(snapshots[snapshot_version], "false")
+                break
+
+    group_desc = f"{target_branch} branch steps"
+    key_desc = "pr-and-build-steps"
+    structure["steps"].append({
+        "group": group_desc,
+        "key": key_desc,
+        "steps": steps
+    })
+
+    print(
+        '# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json')
+    YAML().dump(structure, sys.stdout)

.buildkite/scripts/e2e-pipeline/README.md

@@ -0,0 +1,56 @@
+
+## Getting started
+E2E tests can be also run on your local machine.
+
+### Prerequisites
+
+#### Building the plugin
+E2E tries to install the plugin on Logstash container from the gem file. To generate a gem:
+1. Building plugin requires Logstash
+- If you have `LOGSTASH_PATH` already defined, skip this step
+- You can also download Logstash and export its path to `LOGSTASH_PATH`
+- OR build from source
+    ```shell
+    git clone --single-branch https://github.com/elastic/logstash.git
+    cd logstash && ./gradlew clean bootstrap assemble installDefaultGems && cd ..
+    LOGSTASH_PATH=$(pwd)/logstash
+    export LOGSTASH_PATH
+   ```
+2. Run the following command:
+```shell
+./gradlew clean vendor localGem
+```
+
+#### Defining a project type
+If you want to run tests with serverless, this will be for you.
+Defaults to `on_prems` where local stack containers will be spun up and tested.
+```bash
+export E2E_PROJECT_TYPE="serverless"
+```
+
+In order to run tests with serverless, you also need to export `EC_API_KEY` which is an organization API key to create a project.
+In the pipelines, this will be automatically retrieved from Vault services.
+
+#### Stack version
+E2E also requires `ELASTIC_STACK_VERSION` (ex: "8.12.0") environment variable in order to test against.
+Make sure to export it before running. In the Buildkite pipeline, this var will be resolved and exported. 
+
+#### Installing dependencies
+Make sure you have python installed on you local
+```bash
+pip install -r .buildkite/scripts/e2e-pipeline/requirements.txt
+```
+
+### Run
+Run the following command from the repo dir:
+```bash
+python3 .buildkite/scripts/e2e-pipeline/main.py
+```
+
+## Troubleshooting
+- The project retries on some operations to overcome timeout issues, uses [`retry` tool](https://formulae.brew.sh/formula/retry). If you get `retry` undefined error, make sure to install it.
+```
+brew install retry
+```
+
+- If you run multiple times, currently cloning `integrations` repo will fail, so make sure to remove the repo folder or comment out the `bootstrap.py#__clone_integrations_repo()` method