diff --git a/.buildkite/branches.sh b/.buildkite/branches.sh
new file mode 100644
index 000000000..66d7508ba
--- /dev/null
+++ b/.buildkite/branches.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# This determines which branches will have pipelines triggered periodically, for tests and dra workflows.
+BRANCHES=(main 8.9 7.17)
diff --git a/.buildkite/dra-workflow.trigger.sh b/.buildkite/dra-workflow.trigger.sh
new file mode 100755
index 000000000..0ee8b51a9
--- /dev/null
+++ b/.buildkite/dra-workflow.trigger.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -euo pipefail
+
+echo "steps:"
+
+source .buildkite/branches.sh
+
+BRANCHES=(main 8.9 7.17)
+for BRANCH in "${BRANCHES[@]}"; do
+  cat <<EOF
+  - trigger: elasticsearch-hadoop-dra-workflow
+    label: Trigger DRA snapshot workflow for $BRANCH
+    async: true
+    build:
+      branch: $BRANCH
+      env:
+        DRA_WORKFLOW: snapshot
+  - trigger: elasticsearch-hadoop-dra-workflow
+    label: Trigger DRA staging workflow for $BRANCH
+    async: true
+    build:
+      branch: $BRANCH
+      env:
+        DRA_WORKFLOW: staging
+EOF
+done
diff --git a/.buildkite/dra-workflow.yml b/.buildkite/dra-workflow.yml
new file mode 100644
index 000000000..481b36821
--- /dev/null
+++ b/.buildkite/dra-workflow.yml
@@ -0,0 +1,6 @@
+steps:
+  - label: DRA Workflow
+command: .buildkite/dra.sh
+timeout_in_minutes: 60
+env:
+  USE_DRA_CREDENTIALS: true
diff --git a/.buildkite/dra.sh b/.buildkite/dra.sh
new file mode 100644
index 000000000..99794644d
--- /dev/null
+++ b/.buildkite/dra.sh
@@ -0,0 +1,64 @@
+#!/bin/bash
+
+set -euo pipefail
+
+DRA_WORKFLOW=${DRA_WORKFLOW:-snapshot}
+
+if [[ "$BUILDKITE_BRANCH" == "main" && "$DRA_WORKFLOW" == "staging" ]]; then
+  exit 0
+fi
+
+echo --- Creating distribution
+
+rm -Rfv ~/.gradle/init.d
+HADOOP_VERSION=$(grep eshadoop buildSrc/esh-version.properties | sed "s/eshadoop *= *//g")
+
+VERSION_SUFFIX=""
+BUILD_ARGS="-Dbuild.snapshot=false"
+if [[ "$DRA_WORKFLOW" == "snapshot" ]]; then
+  VERSION_SUFFIX="-SNAPSHOT"
+  BUILD_ARGS="-Dbuild.snapshot=true"
+fi
+
+RM_BRANCH="$BUILDKITE_BRANCH"
+if [[ "$BUILDKITE_BRANCH" == "main" ]]; then
+  RM_BRANCH=master
+fi
+
+echo "DRA_WORKFLOW=$DRA_WORKFLOW"
+echo "HADOOP_VERSION=$HADOOP_VERSION"
+echo "RM_BRANCH=$RM_BRANCH"
+echo "VERSION_SUFFIX=$VERSION_SUFFIX"
+echo "BUILD_ARGS=$BUILD_ARGS"
+
+ES_BUILD_ID=$(curl -sS "https://artifacts-$DRA_WORKFLOW.elastic.co/elasticsearch/latest/${RM_BRANCH}.json" | jq -r '.build_id')
+echo "ES_BUILD_ID=$ES_BUILD_ID"
+
+mkdir localRepo
+wget --quiet "https://artifacts-$DRA_WORKFLOW.elastic.co/elasticsearch/${ES_BUILD_ID}/maven/org/elasticsearch/gradle/build-tools/${HADOOP_VERSION}${VERSION_SUFFIX}/build-tools-${HADOOP_VERSION}${VERSION_SUFFIX}.jar" \
+  -O "localRepo/build-tools-${HADOOP_VERSION}${VERSION_SUFFIX}.jar"
+
+./gradlew -S -PlocalRepo=true "${BUILD_ARGS}" -Dorg.gradle.warning.mode=summary -Dcsv="$WORKSPACE/build/distributions/dependencies-${HADOOP_VERSION}${VERSION_SUFFIX}.csv" :dist:generateDependenciesReport distribution
+
+# Allow other users access to read the artifacts so they are readable in the container
+find "$WORKSPACE" -type f -path "*/build/distributions/*" -exec chmod a+r {} \;
+
+# Allow other users write access to create checksum files
+find "$WORKSPACE" -type d -path "*/build/distributions" -exec chmod a+w {} \;
+
+echo --- Running release-manager
+
+docker run --rm \
+  --name release-manager \
+  -e VAULT_ADDR="$DRA_VAULT_ADDR" \
+  -e VAULT_ROLE_ID="$DRA_VAULT_ROLE_ID_SECRET" \
+  -e VAULT_SECRET_ID="$DRA_VAULT_SECRET_ID_SECRET" \
+  --mount type=bind,readonly=false,src="$PWD",target=/artifacts \
+  docker.elastic.co/infra/release-manager:latest \
+  cli collect \
+  --project elasticsearch-hadoop \
+  --branch "$RM_BRANCH" \
+  --commit "$BUILDKITE_COMMIT" \
+  --workflow "$DRA_WORKFLOW" \
+  --version "$HADOOP_VERSION" \
+  --artifact-set main
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
new file mode 100644
index 000000000..47f9795ae
--- /dev/null
+++ b/.buildkite/hooks/pre-command
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+WORKSPACE="$(pwd)"
+export WORKSPACE
+
+# i.e. source and export the variables
+export $(cat .ci/java-versions.properties | grep '=' | xargs)
+
+JAVA_HOME="$HOME/.java/$ESH_BUILD_JAVA"
+export JAVA_HOME
+
+RUNTIME_JAVA_HOME="$HOME/.java/$ESH_RUNTIME_JAVA"
+export RUNTIME_JAVA_HOME
+
+JAVA7_HOME="$HOME/.java/java7"
+export JAVA7_HOME
+
+JAVA8_HOME="$HOME/.java/java8"
+export JAVA8_HOME
+
+JAVA9_HOME="$HOME/.java/java9"
+export JAVA9_HOME
+
+JAVA10_HOME="$HOME/.java/java10"
+export JAVA10_HOME
+
+JAVA11_HOME="$HOME/.java/java11"
+export JAVA11_HOME
+
+JAVA12_HOME="$HOME/.java/openjdk12"
+export JAVA12_HOME
+
+JAVA13_HOME="$HOME/.java/openjdk13"
+export JAVA13_HOME
+
+JAVA14_HOME="$HOME/.java/openjdk14"
+export JAVA14_HOME
+
+if [[ "$USE_DRA_CREDENTIALS" == "true" ]]; then
+  DRA_VAULT_ROLE_ID_SECRET=$(vault read -field=role-id secret/ci/elastic-elasticsearch-hadoop/legacy-vault-credentials)
+  export DRA_VAULT_ROLE_ID_SECRET
+
+  DRA_VAULT_SECRET_ID_SECRET=$(vault read -field=secret-id secret/ci/elastic-elasticsearch-hadoop/legacy-vault-credentials)
+  export DRA_VAULT_SECRET_ID_SECRET
+
+  DRA_VAULT_ADDR=https://secrets.elastic.co:8200
+  export DRA_VAULT_ADDR
+fi
diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py
new file mode 100755
index 000000000..ddd6c10e9
--- /dev/null
+++ b/.buildkite/pipeline.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+
+import json
+import os
+import re
+from typing import Dict
+
+# Note: If you'd like to add any debug info here, make sure to do it on stderr
+# stdout will be fed into `buildkite-agent pipeline upload`
+
+coreFile = open("spark/core/build.gradle", "r")
+core = coreFile.read()
+coreFile.close()
+
+# `Variant "spark20scala212"` => ["20", "212"]
+groupings = re.findall(r'Variant +"spark([0-9]+)scala([0-9]+)"', core)
+
+groupingsBySparkVersion: Dict[str, list[str]] = {}
+for grouping in groupings:
+    if grouping[0] not in groupingsBySparkVersion:
+        groupingsBySparkVersion[grouping[0]] = []
+    groupingsBySparkVersion[grouping[0]].append(grouping[1])
+
+gradlePropertiesFile = open("gradle.properties", "r")
+gradleProperties = gradlePropertiesFile.read()
+gradlePropertiesFile.close()
+# `scala210Version = 2.10.7` => ["210", "2.10.7"]
+matches = re.findall(
+    r"scala([0-9]+)Version *= *([0-9]+\.[0-9]+\.[0-9]+)", gradleProperties
+)
+
+scalaVersions = {}
+for match in matches:
+    scalaVersions[match[0]] = match[1]
+
+
+pipeline = {
+    "agents": {
+        "provider": "gcp",
+        "image": "family/elasticsearch-ubuntu-2004",
+        "machineType": "n2-standard-8",
+        "diskType": "pd-ssd",
+        "diskSizeGb": "100",
+        "useVault": "false",
+    },
+    "steps": [],
+}
+
+intakeTasks = map(
+    lambda sparkVersion: f"-x :elasticsearch-spark-{sparkVersion}:integrationTest",
+    groupingsBySparkVersion.keys(),
+)
+
+
+pipeline["steps"].append(
+    {
+        "label": "intake",
+        "timeout_in_minutes": 240,
+        "command": "./gradlew check " + " ".join(intakeTasks),
+    }
+)
+
+for sparkVersion in groupingsBySparkVersion.keys():
+    for scalaVersion in groupingsBySparkVersion[sparkVersion]:
+        scalaFullVersion = scalaVersions[scalaVersion]
+        pipeline["steps"].append(
+            {
+                "label": f"spark-{sparkVersion} / scala-{scalaFullVersion}",
+                "timeout_in_minutes": 180,
+                "command": f"./gradlew :elasticsearch-spark-{sparkVersion}:integrationTest -Pscala.variant={scalaFullVersion}",
+            }
+        )
+
+if os.environ.get("ENABLE_DRA_WORKFLOW") == "true":
+    pipeline["steps"].append(
+        {
+            "wait": None,
+        }
+    )
+
+    pipeline["steps"].append(
+        {
+            "label": "DRA Snapshot Workflow",
+            "command": ".buildkite/dra.sh",
+            "timeout_in_minutes": 60,
+            "agents": {"useVault": "true"},
+            "env": {
+                "USE_DRA_CREDENTIALS": "true",
+            },
+        },
+    )
+
+print(json.dumps(pipeline, indent=2))
diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json
new file mode 100644
index 000000000..c4300e010
--- /dev/null
+++ b/.buildkite/pull-requests.json
@@ -0,0 +1,27 @@
+{
+  "jobs": [
+    {
+      "enabled": true,
+      "pipeline_slug": "elasticsearch-hadoop-tests",
+      "allow_org_users": true,
+      "allowed_repo_permissions": [
+        "admin",
+        "write"
+      ],
+      "set_commit_status": false,
+      "build_on_commit": true,
+      "build_on_comment": true,
+      "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
+      "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
+      "skip_ci_labels": [
+        "skip-ci"
+      ],
+      "skip_ci_on_only_changed": [
+        "^docs/",
+        "\\.md$",
+        "\\.mdx$",
+        "^\\.buildkite/pull_requests\\.json$"
+      ]
+    }
+  ]
+}
diff --git a/.buildkite/tests.trigger.sh b/.buildkite/tests.trigger.sh
new file mode 100755
index 000000000..4d7c5ead8
--- /dev/null
+++ b/.buildkite/tests.trigger.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -euo pipefail
+
+echo "steps:"
+
+source .buildkite/branches.sh
+
+for BRANCH in "${BRANCHES[@]}"; do
+  cat <<EOF
+  - trigger: elasticsearch-hadoop-tests
+    label: Trigger tests pipeline for $BRANCH
+    async: true
+    build:
+      branch: $BRANCH
+EOF
+done
+
+
+
diff --git a/.gitignore b/.gitignore
index 7fdfa68da..e026a68eb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,4 @@ metastore_db
 /spark/keyvaluerdd.parquet
 out/
 localRepo/
+.vscode
diff --git a/catalog-info.yaml b/catalog-info.yaml
new file mode 100644
index 000000000..1e1e0f921
--- /dev/null
+++ b/catalog-info.yaml
@@ -0,0 +1,189 @@
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-elasticsearch-hadoop
+  description: Runs full suite of elasticsearch-hadoop tests
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/elasticsearch-hadoop-tests
+spec:
+  type: buildkite-pipeline
+  system: buildkite
+  owner: group:elasticsearch-team
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      description: 'Runs full suite of elasticsearch-hadoop tests'
+      name: elasticsearch-hadoop / tests
+    spec:
+      repository: elastic/elasticsearch-hadoop
+      pipeline_file: .buildkite/pipeline.py
+      default_branch: main
+      branch_configuration: 'main 8.* 7.17'
+      teams:
+        elasticsearch-team: {}
+        everyone:
+          access_level: BUILD_AND_READ
+      provider_settings:
+        trigger_mode: none
+        build_pull_requests: true
+        build_pull_request_forks: false
+        publish_commit_status: true
+        build_branches: true
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-elasticsearch-hadoop-on-merge
+  description: Runs full suite of elasticsearch-hadoop tests and publishes DRA snapshot
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/elasticsearch-hadoop-on-merge
+spec:
+  type: buildkite-pipeline
+  system: buildkite
+  owner: group:elasticsearch-team
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      description: 'Runs full suite of elasticsearch-hadoop tests and publishes DRA snapshot'
+      name: elasticsearch-hadoop / on-merge
+    spec:
+      repository: elastic/elasticsearch-hadoop
+      pipeline_file: .buildkite/pipeline.py
+      env:
+        ENABLE_DRA_WORKFLOW: 'true'
+      default_branch: main
+      branch_configuration: 'main 8.* 7.17'
+      teams:
+        elasticsearch-team: {}
+        everyone:
+          access_level: BUILD_AND_READ
+      provider_settings:
+        trigger_mode: code
+        build_pull_requests: false
+        build_pull_request_forks: false
+        publish_commit_status: true
+        build_branches: true
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-elasticsearch-hadoop-dra-workflow
+  description: Publishes DRA snapshot or staging candidate
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/elasticsearch-hadoop-dra-workflow
+spec:
+  type: buildkite-pipeline
+  system: buildkite
+  owner: group:elasticsearch-team
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      description: 'Publishes DRA snapshot or staging candidate'
+      name: elasticsearch-hadoop / dra-workflow
+    spec:
+      repository: elastic/elasticsearch-hadoop
+      pipeline_file: .buildkite/dra-workflow.yml
+      env:
+        ENABLE_DRA_WORKFLOW: 'true'
+      default_branch: main
+      branch_configuration: 'main 8.* 7.17'
+      teams:
+        elasticsearch-team: {}
+        everyone:
+          access_level: BUILD_AND_READ
+      provider_settings:
+        trigger_mode: none
+        build_pull_requests: false
+        build_pull_request_forks: false
+        publish_commit_status: false
+        build_branches: false
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-elasticsearch-hadoop-dra-workflow-trigger
+  description: Publishes DRA snapshot or staging candidate
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/elasticsearch-hadoop-dra-workflow-trigger
+spec:
+  type: buildkite-pipeline
+  system: buildkite
+  owner: group:elasticsearch-team
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      description: 'Publishes DRA snapshot or staging candidate'
+      name: elasticsearch-hadoop / dra-workflow-trigger
+    spec:
+      repository: elastic/elasticsearch-hadoop
+      pipeline_file: .buildkite/dra-workflow.trigger.sh
+      default_branch: main
+      branch_configuration: main
+      teams:
+        elasticsearch-team: {}
+        everyone:
+          access_level: BUILD_AND_READ
+      provider_settings:
+        trigger_mode: none
+        build_pull_requests: false
+        build_pull_request_forks: false
+        publish_commit_status: false
+        build_branches: false
+      schedules:
+        Periodically on main:
+          branch: main
+          cronline: '0 0,12 * * * America/New_York'
+          message: 'Triggers DRA snapshot workflows 2x per day'
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-elasticsearch-hadoop-tests-trigger
+  description: Triggers test pipelines once per day
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/elasticsearch-hadoop-tests-trigger
+spec:
+  type: buildkite-pipeline
+  system: buildkite
+  owner: group:elasticsearch-team
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      description: 'Triggers test pipelines once per day'
+      name: elasticsearch-hadoop / tests-trigger
+    spec:
+      repository: elastic/elasticsearch-hadoop
+      pipeline_file: .buildkite/tests.trigger.sh
+      default_branch: main
+      branch_configuration: main
+      teams:
+        elasticsearch-team: {}
+        everyone:
+          access_level: BUILD_AND_READ
+      provider_settings:
+        trigger_mode: none
+        build_pull_requests: false
+        build_pull_request_forks: false
+        publish_commit_status: false
+        build_branches: false
+      schedules:
+        Daily on main:
+          branch: main
+          cronline: '0 4 * * * America/New_York'
+          message: 'Triggers test pipelines once per day'
diff --git a/spark/core/build.gradle b/spark/core/build.gradle
index b68800af9..cb4ec6670 100644
--- a/spark/core/build.gradle
+++ b/spark/core/build.gradle
@@ -9,6 +9,9 @@ apply plugin: 'spark.variants'
 
 sparkVariants {
     capabilityGroup 'org.elasticsearch.spark.variant'
+
+    // Changing the formatting of these lines could break .buildkite/pipeline.py, it uses regex to parse the `spark20scala212` part
+    // We should maybe move these to a separate config file that can be read from both this file and the pipeline script in the future if it creates issues
     setCoreDefaultVariant "spark20scala212", spark24Version, scala212Version
     addCoreFeatureVariant "spark30scala213", spark30Version, scala213Version
     addCoreFeatureVariant "spark30scala212", spark30Version, scala212Version