diff --git a/.buildkite/configs/cleanup.aws.yml b/.buildkite/configs/cleanup.aws.yml new file mode 100644 index 0000000000..911d6bb620 --- /dev/null +++ b/.buildkite/configs/cleanup.aws.yml @@ -0,0 +1,75 @@ +--- +version: "1.0" + +accounts: + - name: "${ACCOUNT_PROJECT}" + driver: "aws" + options: + key: '${ACCOUNT_KEY}' + secret: '${ACCOUNT_SECRET}' + +scanners: + - account_name: "${ACCOUNT_PROJECT}" + resources: + - type: 'node' + regions: + - us-east-1 + filters: + - type: "<" + pointer: "/created_at" + param: "${CREATION_DATE}" + converters: + param: "date" + - type: "regex" + pointer: "/extra/tags/repo" + param: "^(elastic-package|integrations)" + - type: "=" + pointer: "/extra/tags/environment" + param: "ci" + - type: "regex" + pointer: "/name" + param: "^elastic-package-(.*)" + - type: "!=" + pointer: "/state" + param: "unknown" + - type: "!=" + pointer: "/state" + param: "terminated" + - type: 'object_storage_bucket' + regions: + - us-east-1 + filters: + - type: "<" + pointer: "/created_at" + param: "${CREATION_DATE}" + converters: + param: "date" + value: "date" + - type: "regex" + pointer: "/extra/tags/repo" + param: "^(elastic-package|integrations)" + - type: "=" + pointer: "/extra/tags/environment" + param: "ci" + - type: "regex" + pointer: "/name" + param: "^elastic-package-(.*)" + - type: 'queue' + regions: + - us-east-1 + filters: + - type: "<" + pointer: "/extra/tags/created_at" + param: "${CREATION_DATE}" + converters: + param: "date" + value: "date_epoch_ms" + - type: "regex" + pointer: "/extra/tags/repo" + param: "^(elastic-package|integrations)" + - type: "=" + pointer: "/extra/tags/environment" + param: "ci" + - type: "regex" + pointer: "/id" + param: "^https://(.*)/elastic-package-(.*)" diff --git a/.buildkite/configs/cleanup.gcp.yml b/.buildkite/configs/cleanup.gcp.yml new file mode 100644 index 0000000000..1dd01f24be --- /dev/null +++ b/.buildkite/configs/cleanup.gcp.yml @@ -0,0 +1,40 @@ +--- +version: "1.0" + +accounts: + - name: "${ACCOUNT_PROJECT}" + driver: "gce" + options: + key: "${ACCOUNT_KEY}" + secret: "${ACCOUNT_SECRET}" + project: "${ACCOUNT_PROJECT}" + +scanners: + - account_name: "${ACCOUNT_PROJECT}" + resources: + - type: "node" + regions: + - "us-east1" + filters: + - type: "<" + pointer: "/extra/creationTimestamp" + param: "${CREATION_DATE}" + converters: + param: "date" + value: "date" + - type: "=" + pointer: "/extra/labels/repo" + param: "elastic-package" + - type: "=" + pointer: "/extra/labels/environment" + param: "ci" + - type: "regex" + pointer: "/name" + param: "^elastic-package-(.*)" + - type: "!=" + pointer: "/state" + param: "unknown" + - type: "!=" + pointer: "/state" + param: "terminated" + diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index c4dc9ed9ed..e6128c1c75 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -47,7 +47,7 @@ export CREATED_DATE # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && ("$BUILDKITE_STEP_KEY" =~ ^integration-parallel || "$BUILDKITE_STEP_KEY" =~ ^integration-false_positives) ]]; then - PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) + PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH} | jq -c) export PRIVATE_CI_GCS_CREDENTIALS_SECRET export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci" fi @@ -55,7 +55,7 @@ fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "integration-parallel-gcp" ]]; then ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectId ${GCP_SERVICE_ACCOUNT_SECRET_PATH}) export ELASTIC_PACKAGE_GCP_PROJECT_SECRET - ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH}) + ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH} | jq -c) export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET # Environment variables required by the service deployer @@ -87,3 +87,22 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-test-with-integrations" && GITHUB_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH}) export GITHUB_TOKEN fi + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-cloud-cleanup" && "$BUILDKITE_STEP_KEY" == "cloud-cleanup" ]]; then + ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) + export ELASTIC_PACKAGE_AWS_SECRET_KEY + ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) + export ELASTIC_PACKAGE_AWS_ACCESS_KEY + ELASTIC_PACKAGE_AWS_USER_SECRET=$(retry 5 vault kv get -field user ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) + export ELASTIC_PACKAGE_AWS_USER_SECRET + + ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET=$(retry 5 vault read -field credentials ${GCP_SERVICE_ACCOUNT_SECRET_PATH} | jq -c) + export ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET + ELASTIC_PACKAGE_GCP_KEY_SECRET=$(echo "${ELASTIC_PACKAGE_GCP_CREDENTIALS_SECRET}" | jq -r '.private_key' | tr -d '\n') + export ELASTIC_PACKAGE_GCP_KEY_SECRET + ELASTIC_PACKAGE_GCP_PROJECT_SECRET=$(retry 5 vault read -field projectId ${GCP_SERVICE_ACCOUNT_SECRET_PATH}) + export ELASTIC_PACKAGE_GCP_PROJECT_SECRET + ELASTIC_PACKAGE_GCP_EMAIL_SECRET=$(retry 5 vault read -field username ${GCP_SERVICE_ACCOUNT_SECRET_PATH}) + export ELASTIC_PACKAGE_GCP_EMAIL_SECRET +fi + diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit index 02346d1f0d..591b7acfe2 100755 --- a/.buildkite/hooks/pre-exit +++ b/.buildkite/hooks/pre-exit @@ -15,3 +15,4 @@ unset ELASTIC_PACKAGE_AWS_ACCESS_KEY unset ELASTIC_PACKAGE_AWS_SECRET_KEY unset AWS_ACCESS_KEY_ID unset AWS_SECRET_ACCESS_KEY + diff --git a/.buildkite/pipeline.cloud-cleanup.yml b/.buildkite/pipeline.cloud-cleanup.yml new file mode 100644 index 0000000000..cb36298db1 --- /dev/null +++ b/.buildkite/pipeline.cloud-cleanup.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json + +# Removes stale Cloud resources (AWS and GCP) having matching labels, name prefixes and older than 24 hours +name: elastic-package-cloud-cleanup + +env: + DOCKER_REGISTRY: docker.elastic.co + NOTIFY_TO: "ecosystem-team@elastic.co" + +steps: + - label: "Cloud Cleanup" + key: "cloud-cleanup" + command: ".buildkite/scripts/cloud-cleanup.sh" + env: + RESOURCE_RETENTION_PERIOD: "24 hours" + DRY_RUN: "true" + agents: + provider: "gcp" + +notify: + - email: "$NOTIFY_TO" + if: "build.state == 'failed' && build.env('BUILDKITE_PULL_REQUEST') == 'false'" diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh new file mode 100644 index 0000000000..ab73877323 --- /dev/null +++ b/.buildkite/scripts/cloud-cleanup.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/install_deps.sh + +set -euo pipefail + +AWS_RESOURCES_FILE="aws.resources.txt" +GCP_RESOURCES_FILE="gcp.resources.txt" + +RESOURCE_RETENTION_PERIOD="${RESOURCE_RETENTION_PERIOD:-"24 hours"}" +export DELETE_RESOURCES_BEFORE_DATE=$(date -Is -d "${RESOURCE_RETENTION_PERIOD} ago") + +CLOUD_REAPER_IMAGE="${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0" + +resources_to_delete=0 + +COMMAND="validate" +if [[ "${DRY_RUN}" != "true" ]]; then + COMMAND="plan" # TODO: to be changed to "destroy --confirm" +else + COMMAND="plan" +fi + +any_resources_to_delete() { + local file=$1 + local number=0 + # First three lines are like: + # ⇒ Loading configuration... + # ✓ Succeeded to load configuration + # Scanning resources... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% 0:00:00 + number=$(tail -n +4 "${file}" | wc -l) + if [ "${number}" -eq 0 ]; then + return 1 + fi + return 0 +} + +cloud_reaper_aws() { + echo "Validating configuration" + docker run --rm -v $(pwd)/.buildkite/configs/cleanup.aws.yml:/etc/cloud-reaper/config.yml \ + -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_AWS_SECRET_KEY}" \ + -e ACCOUNT_KEY="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}" \ + -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_AWS_USER_SECRET}" \ + -e CREATION_DATE="${DELETE_RESOURCES_BEFORE_DATE}" \ + "${CLOUD_REAPER_IMAGE}" \ + cloud-reaper \ + --config /etc/cloud-reaper/config.yml \ + validate + + echo "Scanning resources" + docker run --rm -v $(pwd)/.buildkite/configs/cleanup.aws.yml:/etc/cloud-reaper/config.yml \ + -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_AWS_SECRET_KEY}" \ + -e ACCOUNT_KEY="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}" \ + -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_AWS_USER_SECRET}" \ + -e CREATION_DATE="${DELETE_RESOURCES_BEFORE_DATE}" \ + "${CLOUD_REAPER_IMAGE}" \ + cloud-reaper \ + --config /etc/cloud-reaper/config.yml \ + ${COMMAND} | tee "${AWS_RESOURCES_FILE}" +} + +cloud_reaper_gcp() { + echo "Validating configuration" + docker run --rm -v $(pwd)/.buildkite/configs/cleanup.gcp.yml:/etc/cloud-reaper/config.yml \ + -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_GCP_KEY_SECRET}" \ + -e ACCOUNT_KEY="${ELASTIC_PACKAGE_GCP_EMAIL_SECRET}" \ + -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}" \ + -e CREATION_DATE="${DELETE_RESOURCES_BEFORE_DATE}" \ + "${CLOUD_REAPER_IMAGE}" \ + cloud-reaper \ + --config /etc/cloud-reaper/config.yml \ + validate + + echo "Scanning resources" + docker run --rm -v $(pwd)/.buildkite/configs/cleanup.gcp.yml:/etc/cloud-reaper/config.yml \ + -e ACCOUNT_SECRET="${ELASTIC_PACKAGE_GCP_KEY_SECRET}" \ + -e ACCOUNT_KEY="${ELASTIC_PACKAGE_GCP_EMAIL_SECRET}" \ + -e ACCOUNT_PROJECT="${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}" \ + -e CREATION_DATE="${DELETE_RESOURCES_BEFORE_DATE}" \ + "${CLOUD_REAPER_IMAGE}" \ + cloud-reaper \ + --config /etc/cloud-reaper/config.yml \ + ${COMMAND} | tee "${GCP_RESOURCES_FILE}" +} + +echo "--- Cleaning up GCP resources older than ${DELETE_RESOURCES_BEFORE_DATE}..." +cloud_reaper_gcp + +if any_resources_to_delete "${GCP_RESOURCES_FILE}"; then + echo "Pending GCP resources" + resources_to_delete=1 +fi + +echo "--- Cleaning up AWS resources older than ${DELETE_RESOURCES_BEFORE_DATE}..." +cloud_reaper_aws + +if any_resources_to_delete "${AWS_RESOURCES_FILE}" ; then + echo "Pending AWS resources" + resources_to_delete=1 +fi + +if [ "${resources_to_delete}" -eq 1 ]; then + message="There are resources to be deleted" + echo "${message}" + if running_on_buildkite ; then + buildkite-agent annotate \ + "${message}" \ + --context "ctx-cloud-reaper-error" \ + --style "error" + fi + exit 1 +fi + +# TODO: List and delete the required resources using aws cli +echo "--- Cleaning up other AWS resources older than ${DELETE_RESOURCES_BEFORE_DATE}" +echo "--- Installing awscli" +with_aws_cli + +export AWS_ACCESS_KEY_ID="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}" +export AWS_SECRET_ACCESS_KEY="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}" +export AWS_DEFAULT_REGION=us-east-1 + +echo "--- TODO: Cleaning up Redshift clusters" +echo "--- TODO: Cleaning up IAM roles" +echo "--- TODO: Cleaning up IAM policies" +echo "--- TODO: Cleaning up Schedulers" diff --git a/.buildkite/scripts/install_deps.sh b/.buildkite/scripts/install_deps.sh index e68c86a10c..c4ed488735 100755 --- a/.buildkite/scripts/install_deps.sh +++ b/.buildkite/scripts/install_deps.sh @@ -147,3 +147,17 @@ with_jq() { chmod +x "${WORKSPACE}/bin/jq" jq --version } + +with_aws_cli() { + check_platform_architecture + + if ! which aws; then + curl -s "https://awscli.amazonaws.com/awscli-exe-${platform_type_lowercase}-${hw_type}.zip" -o "awscliv2.zip" + unzip -q awscliv2.zip + sudo ./aws/install + rm -rf awscliv2.zip aws + aws --version + return + fi + echo "\"aws\" already installed" +} diff --git a/.buildkite/scripts/tooling.sh b/.buildkite/scripts/tooling.sh index 1eb38337cd..0c18d9502f 100755 --- a/.buildkite/scripts/tooling.sh +++ b/.buildkite/scripts/tooling.sh @@ -67,3 +67,10 @@ google_cloud_logout_active_account() { unset GOOGLE_APPLICATION_CREDENTIALS fi } + +running_on_buildkite() { + if [[ "${BUILDKITE:-"false"}" == "true" ]]; then + return 0 + fi + return 1 +} diff --git a/catalog-info.yaml b/catalog-info.yaml index 664f8a7fa7..201c992691 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -108,3 +108,50 @@ spec: access_level: MANAGE_BUILD_AND_READ everyone: access_level: READ_ONLY + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-elastic-package-cloud-cleanup + description: Clean up stale cloud resources + links: + - title: Pipeline + url: https://buildkite.com/elastic/elastic-package-cloud-cleanup + +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: elastic-package-cloud-cleanup + description: Buildkite pipeline for cleaning stale resource in cloud providers + spec: + pipeline_file: ".buildkite/pipeline.cloud-cleanup.yml" + provider_settings: + build_pull_request_forks: false + build_pull_requests: false # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + publish_commit_status: false # do not update status of commits for this pipeline + build_tags: false + build_branches: false + filter_enabled: true + filter_condition: >- + build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) + cancel_intermediate_builds: false # do not cancel any build to avoid inconsistent states + skip_intermediate_builds: true # just need to run the latest commit + repository: elastic/elastic-package + schedules: + Daily main: + branch: main + cronline: "00 1 * * *" + message: Daily Cloud cleanup + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: BUILD_AND_READ +