Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[8.13](backport #39298) Branch specific concurrency gates #39304

Closed
wants to merge 1 commit into from
Closed

[8.13](backport #39298) Branch specific concurrency gates #39304

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
276 changes: 276 additions & 0 deletions .buildkite/packaging.pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,276 @@
# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
# TODO: Pre-cache beats-dev/golang-crossbuild container image

env:
ASDF_MAGE_VERSION: 1.15.0
AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"

PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
PLATFORMS_ARM: "linux/arm64"

steps:
# we use concurrency gates (https://buildkite.com/blog/concurrency-gates)
# to implement two FIFO queues for DRA-snapshot and DRA-staging
# this prevents parallel builds and possibility of publishing out of order DRA artifacts if the first job takes longer than the second

- name: Start of concurrency group for DRA Snapshot
if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
command: echo "--> Start of concurrency gate dra-snapshot"
concurrency_group: "dra-gate-snapshot-$BUILDKITE_BRANCH"
concurrency: 1
key: start-gate-snapshot

- name: Start of concurrency group for DRA Staging
if: build.branch =~ /^\d+\.\d+$$/
command: echo "--> Start of concurrency gate dra-staging"
concurrency_group: "dra-gate-staging-$BUILDKITE_BRANCH"
concurrency: 1
key: start-gate-staging

- wait

- group: Beats dashboards
key: dashboards
steps:
- label: Snapshot dashboards
if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
depends_on: start-gate-snapshot
key: dashboards-snapshot
# TODO: container with go and make
agents:
provider: gcp
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
commands:
- make build/distributions/dependencies.csv
- make beats-dashboards
env:
SNAPSHOT: true
DEV: true
artifact_paths:
- build/distributions/**/*

- label: Staging dashboards
if: build.branch =~ /^\d+\.\d+$$/
depends_on: start-gate-staging
key: dashboards-staging
# TODO: container with go and make
agents:
provider: gcp
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
commands:
- make build/distributions/dependencies.csv
- make beats-dashboards
env:
SNAPSHOT: false
DEV: false
artifact_paths:
- build/distributions/**/*

- group: Packaging snapshot
if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
key: packaging-snapshot
depends_on: start-gate-snapshot
steps:
- label: "SNAPSHOT: {{matrix}}"
env:
PLATFORMS: "${PLATFORMS}"
SNAPSHOT: true
DEV: true
command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
agents:
provider: gcp
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
artifact_paths:
- build/distributions/**/*
matrix:
- auditbeat
- filebeat
- heartbeat
- metricbeat
- packetbeat
- winlogbeat
- x-pack/auditbeat
- x-pack/dockerlogbeat
- x-pack/filebeat
- x-pack/functionbeat
- x-pack/heartbeat
- x-pack/metricbeat
- x-pack/osquerybeat
- x-pack/packetbeat
- x-pack/winlogbeat

- label: "SNAPSHOT: {{matrix}} docker Linux/arm64"
env:
PLATFORMS: "${PLATFORMS_ARM}"
PACKAGES: "docker"
SNAPSHOT: true
DEV: true
command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
agents:
provider: "aws"
imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
instanceType: "${AWS_ARM_INSTANCE_TYPE}"
artifact_paths:
- build/distributions/**/*
matrix:
- auditbeat
- filebeat
- heartbeat
- metricbeat
- packetbeat
- x-pack/auditbeat
- x-pack/dockerlogbeat
- x-pack/filebeat
- x-pack/heartbeat
- x-pack/metricbeat
- x-pack/packetbeat

## Agentbeat needs more CPUs because it builds many other beats
- label: "SNAPSHOT: x-pack/agentbeat"
env:
PLATFORMS: "${PLATFORMS}"
SNAPSHOT: true
DEV: true
command: ".buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat"
agents:
provider: gcp
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "c2-standard-16"
artifact_paths:
- build/distributions/**/*

- group: Packaging Staging
key: packaging-staging
depends_on: start-gate-staging
## Only for release
if: build.branch =~ /^\d+\.\d+$$/
steps:
- label: "STAGING: {{matrix}}"
env:
PLATFORMS: "${PLATFORMS}"
SNAPSHOT: false
DEV: false
command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
agents:
provider: gcp
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
artifact_paths:
- build/distributions/**/*
matrix:
- auditbeat
- filebeat
- heartbeat
- metricbeat
- packetbeat
- winlogbeat
- x-pack/auditbeat
- x-pack/dockerlogbeat
- x-pack/filebeat
- x-pack/functionbeat
- x-pack/heartbeat
- x-pack/metricbeat
- x-pack/osquerybeat
- x-pack/packetbeat
- x-pack/winlogbeat

- label: "STAGING: {{matrix}} docker Linux/arm64"
env:
PLATFORMS: "${PLATFORMS_ARM}"
PACKAGES: "docker"
SNAPSHOT: false
DEV: false
command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
agents:
provider: "aws"
imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
instanceType: "${AWS_ARM_INSTANCE_TYPE}"
artifact_paths:
- build/distributions/**/*
matrix:
- auditbeat
- filebeat
- heartbeat
- metricbeat
- packetbeat
- x-pack/auditbeat
- x-pack/dockerlogbeat
- x-pack/filebeat
- x-pack/heartbeat
- x-pack/metricbeat
- x-pack/packetbeat

## Agentbeat needs more CPUs because it builds many other beats
- label: "STAGING: x-pack/agentbeat"
env:
PLATFORMS: "${PLATFORMS}"
SNAPSHOT: false
DEV: false
command: ".buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat"
agents:
provider: gcp
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "c2-standard-16"
artifact_paths:
- build/distributions/**/*

- group: DRA publish
key: dra
steps:
- label: DRA Snapshot
## Only for release branches and main
if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
key: dra-snapshot
env:
DRA_WORKFLOW: snapshot
depends_on:
- start-gate-snapshot
- packaging-snapshot
- dashboards-snapshot
command: |
buildkite-agent artifact download "build/**/*" .
.buildkite/scripts/packaging/prepare-release-manager.sh snapshot
.buildkite/scripts/dra.sh
agents:
provider: gcp
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"

- label: DRA Staging
## Only for release branches
if: build.branch =~ /^\d+\.\d+$$/
key: dra-staging
env:
DRA_WORKFLOW: staging
depends_on:
- start-gate-staging
- packaging-staging
- dashboards-staging
command: |
buildkite-agent artifact download "build/**" .
.buildkite/scripts/packaging/prepare-release-manager.sh staging
.buildkite/scripts/dra.sh
agents:
provider: gcp
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"

- wait

- command: echo "End of concurrency gate dra-snapshot <--"
if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
concurrency_group: "dra-gate-snapshot-$BUILDKITE_BRANCH"
concurrency: 1
key: end-gate-snapshot

- command: echo "End of concurrency gate dra-staging <--"
if: build.branch =~ /^\d+\.\d+$$/
concurrency_group: "dra-gate-staging-$BUILDKITE_BRANCH"
concurrency: 1
key: end-gate-staging