From 62ac3946a6e01b4b572ffa0a63a6ba9da84a839e Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 08:33:35 +0200 Subject: [PATCH 01/49] Build docker images if there is a label --- .github/workflows/run-end-to-end.yml | 1 + utils/scripts/get_pr_merged_labels.sh | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index 2b22ee5aee0..d206c35466c 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -69,6 +69,7 @@ env: jobs: end-to-end: + if: ${{ false }} # disable for now runs-on: ubuntu-latest strategy: matrix: diff --git a/utils/scripts/get_pr_merged_labels.sh b/utils/scripts/get_pr_merged_labels.sh index 96d036957bb..137a14a4d12 100755 --- a/utils/scripts/get_pr_merged_labels.sh +++ b/utils/scripts/get_pr_merged_labels.sh @@ -20,13 +20,20 @@ if [[ $CI_COMMIT_MESSAGE =~ ($PR_PATTERN) ]]; then echo "We found PR labels: $PR_DATA" is_build_buddies=$(echo "$PR_DATA" | jq -c '.[] | select(.name | contains("build-buddies-images"))'); + is_build_python_base_images=$(echo "$PR_DATA" | jq -c '.[] | select(.name | contains("build-python-base-images"))'); + is_build_lib_injection_app_images=$(echo "$PR_DATA" | jq -c '.[] | select(.name | contains("build-lib-injection-app-images"))'); + + if [ -z "$is_build_buddies" ] && [ -z "$is_build_python_base_images" ] && [ -z "$is_build_lib_injection_app_images" ] + echo "The PR $PR_NUMBER doesn't contain any docker build label " + exit 0 + fi + echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin if [ -z "$is_build_buddies" ] then echo "The PR $PR_NUMBER doesn't contain the 'build-buddies-images' label " else echo "The PR $PR_NUMBER contains the 'build-buddies-images' label. Launching the images generation process " - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin ./utils/build/build_tracer_buddies.sh --push fi else From ea5c16597469cf1effb080a7726539dbd9be6a8c Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 08:37:17 +0200 Subject: [PATCH 02/49] set Pr number manually --- utils/scripts/get_pr_merged_labels.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/utils/scripts/get_pr_merged_labels.sh b/utils/scripts/get_pr_merged_labels.sh index 137a14a4d12..506dc030f6b 100755 --- a/utils/scripts/get_pr_merged_labels.sh +++ b/utils/scripts/get_pr_merged_labels.sh @@ -7,8 +7,10 @@ PR_PATTERN='#[0-9]+' -if [[ $CI_COMMIT_MESSAGE =~ ($PR_PATTERN) ]]; then - PR_NUMBER=${BASH_REMATCH[1]:1} +#if [[ $CI_COMMIT_MESSAGE =~ ($PR_PATTERN) ]]; then +if [[ "1"== "1" ) ]]; then + #PR_NUMBER=${BASH_REMATCH[1]:1} + PR_NUMBER=2321 echo "Merged the PR number: [$PR_NUMBER]"; #search for labels PR_DATA=$(curl -L \ From 0e059ae9883ad164fdcc5e82b73982b518f4f415 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 08:39:01 +0200 Subject: [PATCH 03/49] merge labels --- utils/scripts/get_pr_merged_labels.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/scripts/get_pr_merged_labels.sh b/utils/scripts/get_pr_merged_labels.sh index 506dc030f6b..c4d826f6ab7 100755 --- a/utils/scripts/get_pr_merged_labels.sh +++ b/utils/scripts/get_pr_merged_labels.sh @@ -8,7 +8,7 @@ PR_PATTERN='#[0-9]+' #if [[ $CI_COMMIT_MESSAGE =~ ($PR_PATTERN) ]]; then -if [[ "1"== "1" ) ]]; then +if [[ "1" == "1" ) ]]; then #PR_NUMBER=${BASH_REMATCH[1]:1} PR_NUMBER=2321 echo "Merged the PR number: [$PR_NUMBER]"; From e26c13172380757c45e65dbc81518956823bcdce Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 08:41:26 +0200 Subject: [PATCH 04/49] merge labels --- utils/scripts/get_pr_merged_labels.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/scripts/get_pr_merged_labels.sh b/utils/scripts/get_pr_merged_labels.sh index c4d826f6ab7..31b171a2d75 100755 --- a/utils/scripts/get_pr_merged_labels.sh +++ b/utils/scripts/get_pr_merged_labels.sh @@ -8,7 +8,7 @@ PR_PATTERN='#[0-9]+' #if [[ $CI_COMMIT_MESSAGE =~ ($PR_PATTERN) ]]; then -if [[ "1" == "1" ) ]]; then +if [[ "1" == "1" ]]; then #PR_NUMBER=${BASH_REMATCH[1]:1} PR_NUMBER=2321 echo "Merged the PR number: [$PR_NUMBER]"; From 973a4f3d4da6158804dada2ed4064a2e32a852a7 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 08:43:28 +0200 Subject: [PATCH 05/49] launch gitlab job --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b3061ee5c65..14198e9c807 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -203,4 +203,4 @@ check_merge_labels: script: - ./utils/scripts/get_pr_merged_labels.sh rules: - - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" \ No newline at end of file + - if: 1 == 1 || $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" \ No newline at end of file From 66cf58ab450253f9a8dc48c33d2a13a7ca23c152 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 08:47:13 +0200 Subject: [PATCH 06/49] pr labels --- utils/scripts/get_pr_merged_labels.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/scripts/get_pr_merged_labels.sh b/utils/scripts/get_pr_merged_labels.sh index 31b171a2d75..9ad2b83213b 100755 --- a/utils/scripts/get_pr_merged_labels.sh +++ b/utils/scripts/get_pr_merged_labels.sh @@ -26,6 +26,7 @@ if [[ "1" == "1" ]]; then is_build_lib_injection_app_images=$(echo "$PR_DATA" | jq -c '.[] | select(.name | contains("build-lib-injection-app-images"))'); if [ -z "$is_build_buddies" ] && [ -z "$is_build_python_base_images" ] && [ -z "$is_build_lib_injection_app_images" ] + then echo "The PR $PR_NUMBER doesn't contain any docker build label " exit 0 fi From e95c26be0324b1ca486870675d30e9f3089a2a1d Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 08:50:01 +0200 Subject: [PATCH 07/49] trigger gitlab --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 14198e9c807..7fa503ed647 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -203,4 +203,4 @@ check_merge_labels: script: - ./utils/scripts/get_pr_merged_labels.sh rules: - - if: 1 == 1 || $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" \ No newline at end of file + - if: 1 == 1 #$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" \ No newline at end of file From 26162aff6b1de9f0db2441651d3e11f27be15d8e Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 09:08:21 +0200 Subject: [PATCH 08/49] pr labels --- utils/scripts/get_pr_merged_labels.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/scripts/get_pr_merged_labels.sh b/utils/scripts/get_pr_merged_labels.sh index 9ad2b83213b..12aacf1af03 100755 --- a/utils/scripts/get_pr_merged_labels.sh +++ b/utils/scripts/get_pr_merged_labels.sh @@ -1,11 +1,11 @@ #!/bin/bash -# shellcheck disable=SC2164 +# shellcheck disable=SC2164,SC2050 #Check the commit message to figure out if we are merging a PR. #We extract the PR number and using GitHub API we check the PR labels. #If the PR contains the label "build-buddies-images" we launch the build and push process -PR_PATTERN='#[0-9]+' +#PR_PATTERN='#[0-9]+' #if [[ $CI_COMMIT_MESSAGE =~ ($PR_PATTERN) ]]; then if [[ "1" == "1" ]]; then From 9c6def02a275138b1d46a7fb94ec88da3dd03a5c Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 09:10:13 +0200 Subject: [PATCH 09/49] pr labels always --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7fa503ed647..bfa029418c6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -202,5 +202,5 @@ check_merge_labels: - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) script: - ./utils/scripts/get_pr_merged_labels.sh - rules: - - if: 1 == 1 #$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" \ No newline at end of file + #rules: + # - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" \ No newline at end of file From c0b3516abd0b55a4bf6fb2d2b5e6a9a2a1d921c4 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 09:33:02 +0200 Subject: [PATCH 10/49] build lib injection images --- utils/scripts/get_pr_merged_labels.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/utils/scripts/get_pr_merged_labels.sh b/utils/scripts/get_pr_merged_labels.sh index 12aacf1af03..d6ccab6b41e 100755 --- a/utils/scripts/get_pr_merged_labels.sh +++ b/utils/scripts/get_pr_merged_labels.sh @@ -32,6 +32,7 @@ if [[ "1" == "1" ]]; then fi echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin + #BUILD BUDDIES IMAGES if [ -z "$is_build_buddies" ] then echo "The PR $PR_NUMBER doesn't contain the 'build-buddies-images' label " @@ -39,6 +40,16 @@ if [[ "1" == "1" ]]; then echo "The PR $PR_NUMBER contains the 'build-buddies-images' label. Launching the images generation process " ./utils/build/build_tracer_buddies.sh --push fi + + #BUILD LIB INJECTION IMAGES + if [ -z "$is_build_lib_injection_app_images" ] + then + echo "The PR $PR_NUMBER doesn't contain the 'build-lib-injection-app-images' label " + else + echo "The PR $PR_NUMBER contains the 'build-lib-injection-app-images' label. Launching the images generation process " + ./lib-injection/build/build_lib_injection_images.sh + fi + else echo "The commit message $CI_COMMIT_MESSAGE doesn't contain the PR number." fi \ No newline at end of file From ca0a089c4444220f8a842a221498a303f85d0af3 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 09:40:16 +0200 Subject: [PATCH 11/49] lib injection images build script --- lib-injection/build/build_lib_injection_images.sh | 5 +++++ 1 file changed, 5 insertions(+) create mode 100755 lib-injection/build/build_lib_injection_images.sh diff --git a/lib-injection/build/build_lib_injection_images.sh b/lib-injection/build/build_lib_injection_images.sh new file mode 100755 index 00000000000..d1b1756a8fc --- /dev/null +++ b/lib-injection/build/build_lib_injection_images.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +export DOCKER_IMAGE_WEBLOG_TAG=latest + +APP_DOCKER_IMAGE_REPO=ghcr.io/datadog/system-tests/dd-lib-dotnet-init-test-app LIBRARY_INJECTION_TEST_APP_IMAGE=ghcr.io/datadog/system-tests/dd-lib-dotnet-init-test-app:$DOCKER_IMAGE_WEBLOG_TAG ./lib-injection/build/docker/dotnet/dd-lib-dotnet-init-test-app/build.sh From 3f7d84799a07276aee6488835ef0cde21e8cea11 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 10:28:00 +0200 Subject: [PATCH 12/49] Build lib injeciton images --- lib-injection/build/build_lib_injection_images.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib-injection/build/build_lib_injection_images.sh b/lib-injection/build/build_lib_injection_images.sh index d1b1756a8fc..3de8c902ef8 100755 --- a/lib-injection/build/build_lib_injection_images.sh +++ b/lib-injection/build/build_lib_injection_images.sh @@ -1,5 +1,14 @@ #!/usr/bin/env bash +set -e export DOCKER_IMAGE_WEBLOG_TAG=latest -APP_DOCKER_IMAGE_REPO=ghcr.io/datadog/system-tests/dd-lib-dotnet-init-test-app LIBRARY_INJECTION_TEST_APP_IMAGE=ghcr.io/datadog/system-tests/dd-lib-dotnet-init-test-app:$DOCKER_IMAGE_WEBLOG_TAG ./lib-injection/build/docker/dotnet/dd-lib-dotnet-init-test-app/build.sh +declare -A variants +variants=(["dd-lib-dotnet-init-test-app"]="dotnet" ["dd-lib-java-init-test-app"]="java") + +for variant in "${!variants[@]}"; do + language="${variants[$variant]}" + echo "Building $variant - $language"; + echo "$(pwd)" + cd ./lib-injection/build/docker/$language/$variant/ && APP_DOCKER_IMAGE_REPO=ghcr.io/datadog/system-tests/$variant LIBRARY_INJECTION_TEST_APP_IMAGE=ghcr.io/datadog/system-tests/$variant:$DOCKER_IMAGE_WEBLOG_TAG ./build.sh && cd ../../../../../ +done \ No newline at end of file From 60512229380409d61bd4d33bb0e5495e7c55b9e9 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 10:48:37 +0200 Subject: [PATCH 13/49] build nodejs --- lib-injection/build/build_lib_injection_images.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib-injection/build/build_lib_injection_images.sh b/lib-injection/build/build_lib_injection_images.sh index 3de8c902ef8..a0210e67310 100755 --- a/lib-injection/build/build_lib_injection_images.sh +++ b/lib-injection/build/build_lib_injection_images.sh @@ -4,7 +4,8 @@ set -e export DOCKER_IMAGE_WEBLOG_TAG=latest declare -A variants -variants=(["dd-lib-dotnet-init-test-app"]="dotnet" ["dd-lib-java-init-test-app"]="java") +#variants=(["dd-lib-dotnet-init-test-app"]="dotnet" ["dd-lib-java-init-test-app"]="java") +variants=(["dd-lib-dotnet-init-test-app"]="dotnet" ["sample-app"]="nodejs") for variant in "${!variants[@]}"; do language="${variants[$variant]}" From ca139db91fb4ee45e002efa1ab904356746daa4d Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 10:56:47 +0200 Subject: [PATCH 14/49] Buildx config --- lib-injection/build/build_lib_injection_images.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/lib-injection/build/build_lib_injection_images.sh b/lib-injection/build/build_lib_injection_images.sh index a0210e67310..127143205f6 100755 --- a/lib-injection/build/build_lib_injection_images.sh +++ b/lib-injection/build/build_lib_injection_images.sh @@ -6,6 +6,7 @@ export DOCKER_IMAGE_WEBLOG_TAG=latest declare -A variants #variants=(["dd-lib-dotnet-init-test-app"]="dotnet" ["dd-lib-java-init-test-app"]="java") variants=(["dd-lib-dotnet-init-test-app"]="dotnet" ["sample-app"]="nodejs") +docker buildx create --name multiarch --driver docker-container --use for variant in "${!variants[@]}"; do language="${variants[$variant]}" From ce7304057a1619aa7355df1279df22e559c53b91 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 10:59:41 +0200 Subject: [PATCH 15/49] build platforms sample-app --- lib-injection/build/docker/nodejs/sample-app/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/lib-injection/build/docker/nodejs/sample-app/build.sh b/lib-injection/build/docker/nodejs/sample-app/build.sh index a44f6aefff1..985dd040619 100755 --- a/lib-injection/build/docker/nodejs/sample-app/build.sh +++ b/lib-injection/build/docker/nodejs/sample-app/build.sh @@ -3,4 +3,5 @@ if [ -z "${BUILDX_PLATFORMS}" ] ; then BUILDX_PLATFORMS=`docker buildx imagetools inspect --raw python:3.9 | jq -r 'reduce (.manifests[] | [ .platform.os, .platform.architecture, .platform.variant ] | join("/") | sub("\\/$"; "")) as $item (""; . + "," + $item)' | sed 's/,//'` fi +echo "Build for platforms: ${BUILDX_PLATFORMS}" docker buildx build --platform ${BUILDX_PLATFORMS} --tag ${LIBRARY_INJECTION_TEST_APP_IMAGE} --push . \ No newline at end of file From cd64e06cad022c6de3c3e64ab8900962a3ec32ec Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 11:10:43 +0200 Subject: [PATCH 16/49] gitlab image build and push --- .gitlab-ci.yml | 25 ++++++++++++++++++++++++- .gitlab/Dockerfile.gitlab | 23 +++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 .gitlab/Dockerfile.gitlab diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bfa029418c6..502fca030e3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,7 @@ stages: - dotnet_tracer - parse_results - before_tests + - manual_images .base_job_onboarding: only: @@ -46,6 +47,9 @@ variables: # Do not modify this - must be the repository name for Kubernetes gitlab runners to run KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: system-tests #helm-charts TEST: 1 + DOCKER_REGISTRY: registry.ddbuild.io + CI_DOCKER_IMAGE_BASE: $DOCKER_REGISTRY/ci/system-tests + PROMOTION_TAG: current onboarding_nodejs: extends: .base_job_onboarding @@ -203,4 +207,23 @@ check_merge_labels: script: - ./utils/scripts/get_pr_merged_labels.sh #rules: - # - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" \ No newline at end of file + # - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" + +build-image: + stage: manual-images + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "push" + changes: + - gitlab/Dockerfile.* + - when: manual + allow_failure: true + tags: [ "arch:amd64" ] + image: $DOCKER_REGISTRY/docker:20.10.13 + script: + - > + docker buildx build + --tag $CI_DOCKER_IMAGE_BASE/gitlab:current + --platform linux/amd64 + --push + --file gitlab/Dockerfile.gitlab + . \ No newline at end of file diff --git a/.gitlab/Dockerfile.gitlab b/.gitlab/Dockerfile.gitlab new file mode 100644 index 00000000000..9ce845dc5fa --- /dev/null +++ b/.gitlab/Dockerfile.gitlab @@ -0,0 +1,23 @@ +FROM registry.ddbuild.io/base:focal + +USER root + +RUN clean-apt install \ + binutils \ + rpm \ + jq \ + rubygems \ + default-jdk \ + awscli \ + ruby-full \ + build-essential \ + ca-certificates \ + python3-pip \ + unzip \ + git + +# Newer versions of dotenv don't work with this version of ruby +RUN gem install dotenv -v 2.8.1 +RUN gem install fpm + +RUN curl -fsSL https://deb.nodesource.com/setup_21.x | bash - && apt-get install -y nodejs From 487f709ca4b3b96d9d02e68157a7d7a6a0b9630c Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 11:29:48 +0200 Subject: [PATCH 17/49] gitlab manual images --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 502fca030e3..11fd589af0c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -210,7 +210,7 @@ check_merge_labels: # - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" build-image: - stage: manual-images + stage: manual_images rules: - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "push" changes: From 18a95073e9e881c59a2ee0afa4aa783c5b41adbf Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 11:34:14 +0200 Subject: [PATCH 18/49] gitlab ci --- .gitlab-ci.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 11fd589af0c..07345a81e00 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,10 +1,10 @@ stages: - - ruby_tracer - - nodejs_tracer - - java_tracer - - python_tracer - - dotnet_tracer - - parse_results + #- ruby_tracer + #- nodejs_tracer + #- java_tracer + #- python_tracer + #- dotnet_tracer + #- parse_results - before_tests - manual_images @@ -51,7 +51,7 @@ variables: CI_DOCKER_IMAGE_BASE: $DOCKER_REGISTRY/ci/system-tests PROMOTION_TAG: current -onboarding_nodejs: +.onboarding_nodejs: extends: .base_job_onboarding stage: nodejs_tracer allow_failure: true @@ -76,7 +76,7 @@ onboarding_nodejs: - ./build.sh -i runner - timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws -onboarding_java: +.onboarding_java: extends: .base_job_onboarding stage: java_tracer allow_failure: true @@ -100,7 +100,7 @@ onboarding_java: - ./build.sh -i runner - timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws -onboarding_python: +.onboarding_python: extends: .base_job_onboarding stage: python_tracer allow_failure: true @@ -124,7 +124,7 @@ onboarding_python: - ./build.sh -i runner - timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws -onboarding_dotnet: +.onboarding_dotnet: extends: .base_job_onboarding stage: dotnet_tracer allow_failure: true @@ -148,7 +148,7 @@ onboarding_dotnet: - ./build.sh -i runner - timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws -onboarding_ruby: +.onboarding_ruby: extends: .base_job_onboarding stage: ruby_tracer allow_failure: true @@ -172,7 +172,7 @@ onboarding_ruby: - ./build.sh -i runner - timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws -onboarding_parse_results: +.onboarding_parse_results: image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner:a58cc31c tags: ["arch:amd64"] stage: parse_results From b3c940d8b098399bb82e409b4154e986152a70ca Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 11:40:18 +0200 Subject: [PATCH 19/49] gitlab ci --- .gitlab-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 07345a81e00..ba2e36aef8b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -200,6 +200,7 @@ check_merge_labels: image: registry.ddbuild.io/images/ci_docker_base tags: ["runner:docker"] stage: before_tests + allow_failure: true before_script: - export GH_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.gh-token --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) @@ -212,9 +213,9 @@ check_merge_labels: build-image: stage: manual_images rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "push" - changes: - - gitlab/Dockerfile.* + #- if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "push" + # changes: + # - gitlab/Dockerfile.* - when: manual allow_failure: true tags: [ "arch:amd64" ] From 618af6a6b4540bfb6626753bcb7b679386a3f0b0 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 11:42:33 +0200 Subject: [PATCH 20/49] gitlab ci --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ba2e36aef8b..658617b61f4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -226,5 +226,5 @@ build-image: --tag $CI_DOCKER_IMAGE_BASE/gitlab:current --platform linux/amd64 --push - --file gitlab/Dockerfile.gitlab + --file .gitlab/Dockerfile.gitlab . \ No newline at end of file From 181f496b9fcd35971bbd9c79af4f1091a64974e8 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 11:48:53 +0200 Subject: [PATCH 21/49] set platforms --- lib-injection/build/build_lib_injection_images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-injection/build/build_lib_injection_images.sh b/lib-injection/build/build_lib_injection_images.sh index 127143205f6..671888594d9 100755 --- a/lib-injection/build/build_lib_injection_images.sh +++ b/lib-injection/build/build_lib_injection_images.sh @@ -2,7 +2,7 @@ set -e export DOCKER_IMAGE_WEBLOG_TAG=latest - +export BUILDX_PLATFORMS=linux/arm64/v8,linux/amd64 declare -A variants #variants=(["dd-lib-dotnet-init-test-app"]="dotnet" ["dd-lib-java-init-test-app"]="java") variants=(["dd-lib-dotnet-init-test-app"]="dotnet" ["sample-app"]="nodejs") From 9f55ce933592b8dabf2285e321d7a477086015cf Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 14:37:31 +0200 Subject: [PATCH 22/49] ghcr login --- utils/scripts/get_pr_merged_labels.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/utils/scripts/get_pr_merged_labels.sh b/utils/scripts/get_pr_merged_labels.sh index d6ccab6b41e..ea213a639b9 100755 --- a/utils/scripts/get_pr_merged_labels.sh +++ b/utils/scripts/get_pr_merged_labels.sh @@ -30,13 +30,13 @@ if [[ "1" == "1" ]]; then echo "The PR $PR_NUMBER doesn't contain any docker build label " exit 0 fi - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin - + #BUILD BUDDIES IMAGES if [ -z "$is_build_buddies" ] then echo "The PR $PR_NUMBER doesn't contain the 'build-buddies-images' label " else + echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin echo "The PR $PR_NUMBER contains the 'build-buddies-images' label. Launching the images generation process " ./utils/build/build_tracer_buddies.sh --push fi @@ -47,6 +47,7 @@ if [[ "1" == "1" ]]; then echo "The PR $PR_NUMBER doesn't contain the 'build-lib-injection-app-images' label " else echo "The PR $PR_NUMBER contains the 'build-lib-injection-app-images' label. Launching the images generation process " + echo "$GITHUB_TOKEN" | docker login ghcr.io --username "publisher" --password-stdin ./lib-injection/build/build_lib_injection_images.sh fi From b4835ef4d7bc1bbc547afe669b41e8deafcd000c Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 14:40:35 +0200 Subject: [PATCH 23/49] login different registries --- utils/scripts/get_pr_merged_labels.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/scripts/get_pr_merged_labels.sh b/utils/scripts/get_pr_merged_labels.sh index ea213a639b9..795f1f34c9b 100755 --- a/utils/scripts/get_pr_merged_labels.sh +++ b/utils/scripts/get_pr_merged_labels.sh @@ -30,13 +30,12 @@ if [[ "1" == "1" ]]; then echo "The PR $PR_NUMBER doesn't contain any docker build label " exit 0 fi - + echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin #BUILD BUDDIES IMAGES if [ -z "$is_build_buddies" ] then echo "The PR $PR_NUMBER doesn't contain the 'build-buddies-images' label " else - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin echo "The PR $PR_NUMBER contains the 'build-buddies-images' label. Launching the images generation process " ./utils/build/build_tracer_buddies.sh --push fi From 660bcb2724e48434b95a7e77e154f8f16202f9e1 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 14:43:46 +0200 Subject: [PATCH 24/49] fix gh token --- utils/scripts/get_pr_merged_labels.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/scripts/get_pr_merged_labels.sh b/utils/scripts/get_pr_merged_labels.sh index 795f1f34c9b..4263e79cd1c 100755 --- a/utils/scripts/get_pr_merged_labels.sh +++ b/utils/scripts/get_pr_merged_labels.sh @@ -46,7 +46,7 @@ if [[ "1" == "1" ]]; then echo "The PR $PR_NUMBER doesn't contain the 'build-lib-injection-app-images' label " else echo "The PR $PR_NUMBER contains the 'build-lib-injection-app-images' label. Launching the images generation process " - echo "$GITHUB_TOKEN" | docker login ghcr.io --username "publisher" --password-stdin + echo "$GH_TOKEN" | docker login ghcr.io --username "publisher" --password-stdin ./lib-injection/build/build_lib_injection_images.sh fi From ffc65e80f53e0e181438251eff2e3d5e3f1853d6 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 14:50:29 +0200 Subject: [PATCH 25/49] add python variants --- lib-injection/build/build_lib_injection_images.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib-injection/build/build_lib_injection_images.sh b/lib-injection/build/build_lib_injection_images.sh index 671888594d9..eefff8dcf2b 100755 --- a/lib-injection/build/build_lib_injection_images.sh +++ b/lib-injection/build/build_lib_injection_images.sh @@ -5,7 +5,13 @@ export DOCKER_IMAGE_WEBLOG_TAG=latest export BUILDX_PLATFORMS=linux/arm64/v8,linux/amd64 declare -A variants #variants=(["dd-lib-dotnet-init-test-app"]="dotnet" ["dd-lib-java-init-test-app"]="java") -variants=(["dd-lib-dotnet-init-test-app"]="dotnet" ["sample-app"]="nodejs") +variants=(["dd-lib-dotnet-init-test-app"]="dotnet" + ["sample-app"]="nodejs" + ["dd-lib-python-init-test-django"]="python" + ["dd-lib-python-init-test-django-gunicorn"]="python" + ["dd-lib-python-init-test-django-uvicorn"]="python" + ["dd-lib-python-init-test-django-uvicorn"]="python" + ) docker buildx create --name multiarch --driver docker-container --use for variant in "${!variants[@]}"; do From a58cadfa771b6ab73073b096348abe750c253a76 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 14:59:53 +0200 Subject: [PATCH 26/49] build ruby images --- lib-injection/build/build_lib_injection_images.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib-injection/build/build_lib_injection_images.sh b/lib-injection/build/build_lib_injection_images.sh index eefff8dcf2b..d3b30467b46 100755 --- a/lib-injection/build/build_lib_injection_images.sh +++ b/lib-injection/build/build_lib_injection_images.sh @@ -10,7 +10,12 @@ variants=(["dd-lib-dotnet-init-test-app"]="dotnet" ["dd-lib-python-init-test-django"]="python" ["dd-lib-python-init-test-django-gunicorn"]="python" ["dd-lib-python-init-test-django-uvicorn"]="python" - ["dd-lib-python-init-test-django-uvicorn"]="python" + ["dd-lib-ruby-init-test-rails"]="ruby" + ["dd-lib-ruby-init-test-rails-bundle-deploy"]="ruby" + ["dd-lib-ruby-init-test-rails-conflict"]="ruby" + ["dd-lib-ruby-init-test-rails-explicit"]="ruby" + ["dd-lib-ruby-init-test-rails-gemsrb"]="ruby" + ["lib_injection_rails_app"]="ruby" ) docker buildx create --name multiarch --driver docker-container --use From 2f2e596f454539adee1b8b52f06cbdbd32377217 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 15:06:44 +0200 Subject: [PATCH 27/49] remove ruby app --- lib-injection/build/build_lib_injection_images.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/lib-injection/build/build_lib_injection_images.sh b/lib-injection/build/build_lib_injection_images.sh index d3b30467b46..712234f5fcd 100755 --- a/lib-injection/build/build_lib_injection_images.sh +++ b/lib-injection/build/build_lib_injection_images.sh @@ -15,7 +15,6 @@ variants=(["dd-lib-dotnet-init-test-app"]="dotnet" ["dd-lib-ruby-init-test-rails-conflict"]="ruby" ["dd-lib-ruby-init-test-rails-explicit"]="ruby" ["dd-lib-ruby-init-test-rails-gemsrb"]="ruby" - ["lib_injection_rails_app"]="ruby" ) docker buildx create --name multiarch --driver docker-container --use From f5e97eb11d7b1302e9dd973b12a8cf641016d04c Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 15:42:43 +0200 Subject: [PATCH 28/49] build python base images --- utils/build/build_python_base_images.sh | 19 +++++++++++++++++++ utils/scripts/get_pr_merged_labels.sh | 12 ++++++++++++ 2 files changed, 31 insertions(+) create mode 100755 utils/build/build_python_base_images.sh diff --git a/utils/build/build_python_base_images.sh b/utils/build/build_python_base_images.sh new file mode 100755 index 00000000000..d694a4b5bd7 --- /dev/null +++ b/utils/build/build_python_base_images.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# build and push python base images + + +docker buildx build --load --progress=plain -f utils/build/docker/python/fastapi.base.Dockerfile -t datadog/system-tests:fastapi.base-v0 . +docker buildx build --load --progress=plain -f utils/build/docker/python/python3.12.base.Dockerfile -t datadog/system-tests:python3.12.base-v1 . +docker buildx build --load --progress=plain -f utils/build/docker/python/django-poc.base.Dockerfile -t datadog/system-tests:django-poc.base-v0 . +docker buildx build --load --progress=plain -f utils/build/docker/python/flask-poc.base.Dockerfile -t datadog/system-tests:flask-poc.base-v2 . +docker buildx build --load --progress=plain -f utils/build/docker/python/uwsgi-poc.base.Dockerfile -t datadog/system-tests:uwsgi-poc.base-v1 . + +if [ "$1" = "--push" ]; then + docker push datadog/system-tests:fastapi.base-v0 + docker push datadog/system-tests:python3.12.base-v1 + docker push datadog/system-tests:django-poc.base-v0 + docker push datadog/system-tests:flask-poc.base-v2 + docker push datadog/system-tests:uwsgi-poc.base-v1 +fi + diff --git a/utils/scripts/get_pr_merged_labels.sh b/utils/scripts/get_pr_merged_labels.sh index 4263e79cd1c..e6a5bb1bf28 100755 --- a/utils/scripts/get_pr_merged_labels.sh +++ b/utils/scripts/get_pr_merged_labels.sh @@ -38,6 +38,17 @@ if [[ "1" == "1" ]]; then else echo "The PR $PR_NUMBER contains the 'build-buddies-images' label. Launching the images generation process " ./utils/build/build_tracer_buddies.sh --push + echo "------------- The buddies images have been built and pushed ------------- " + fi + + #BUILD PYTHON BASE IMAGES + if [ -z "$is_build_python_base_images" ] + then + echo "The PR $PR_NUMBER doesn't contain the 'build-python-base-images' label " + else + echo "The PR $PR_NUMBER contains the 'build-python-base-images' label. Launching the images generation process " + ./utils/build/build_python_base_images.sh --push + echo "------------- The python base images have been built and pushed ------------- " fi #BUILD LIB INJECTION IMAGES @@ -48,6 +59,7 @@ if [[ "1" == "1" ]]; then echo "The PR $PR_NUMBER contains the 'build-lib-injection-app-images' label. Launching the images generation process " echo "$GH_TOKEN" | docker login ghcr.io --username "publisher" --password-stdin ./lib-injection/build/build_lib_injection_images.sh + echo "------------- The lib injection weblog images have been built and pushed ------------- " fi else From 5efecd2badfaa600deb636f2512754c1a6188145 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 15:47:52 +0200 Subject: [PATCH 29/49] try custom image runner gitlab --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 658617b61f4..b0e3c3555e3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -197,8 +197,10 @@ variables: done check_merge_labels: - image: registry.ddbuild.io/images/ci_docker_base - tags: ["runner:docker"] + #image: registry.ddbuild.io/images/ci_docker_base + #tags: ["runner:docker"] + tags: [ "arch:amd64" ] + image: $CI_DOCKER_IMAGE_BASE/gitlab:current stage: before_tests allow_failure: true before_script: From e1e242dfde46ef870aab7820cba30f06ad9d5cca Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 16:38:40 +0200 Subject: [PATCH 30/49] gitlab --- .gitlab/Dockerfile.gitlab | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.gitlab/Dockerfile.gitlab b/.gitlab/Dockerfile.gitlab index 9ce845dc5fa..ec2ca609a55 100644 --- a/.gitlab/Dockerfile.gitlab +++ b/.gitlab/Dockerfile.gitlab @@ -21,3 +21,28 @@ RUN gem install dotenv -v 2.8.1 RUN gem install fpm RUN curl -fsSL https://deb.nodesource.com/setup_21.x | bash - && apt-get install -y nodejs + + +#https://docs.docker.com/engine/install/ubuntu/ + buildx +RUN apt-get -y install ca-certificates curl gnupg +RUN install -m 0755 -d /etc/apt/keyrings +RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg +RUN chmod a+r /etc/apt/keyrings/docker.gpg +RUN echo \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null +RUN apt-get update +RUN apt-get -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +RUN curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/bin/docker-compose && chmod +x /usr/bin/docker-compose && docker-compose --version +RUN echo "DOCKER INSTALLED!" +#To run Docker without root privileges +#RUN groupadd docker +#RUN usermod -aG docker "ubuntu" +#RUN echo "DOCKER USER ADDED" +RUN apt install -y docker-buildx-plugin +RUN docker buildx install +RUN usermod -aG docker root +#FIX ulimit +RUN sed -i 's\ulimit -Hn 524288\ulimit -n 524288\g' /etc/init.d/docker +RUN service docker restart \ No newline at end of file From 13c39dd4e89d5e663a5511e8ac22c40e945e9167 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 17:00:59 +0200 Subject: [PATCH 31/49] docker runner --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b0e3c3555e3..d119a5757f4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -198,8 +198,8 @@ variables: check_merge_labels: #image: registry.ddbuild.io/images/ci_docker_base - #tags: ["runner:docker"] - tags: [ "arch:amd64" ] + tags: ["runner:docker"] + #tags: [ "arch:amd64" ] image: $CI_DOCKER_IMAGE_BASE/gitlab:current stage: before_tests allow_failure: true From e0b4c0786b969a7134842ebd22c2dac83f5566a4 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 17:06:57 +0200 Subject: [PATCH 32/49] test java --- lib-injection/build/build_lib_injection_images.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib-injection/build/build_lib_injection_images.sh b/lib-injection/build/build_lib_injection_images.sh index 712234f5fcd..8e02bed5508 100755 --- a/lib-injection/build/build_lib_injection_images.sh +++ b/lib-injection/build/build_lib_injection_images.sh @@ -4,8 +4,8 @@ set -e export DOCKER_IMAGE_WEBLOG_TAG=latest export BUILDX_PLATFORMS=linux/arm64/v8,linux/amd64 declare -A variants -#variants=(["dd-lib-dotnet-init-test-app"]="dotnet" ["dd-lib-java-init-test-app"]="java") -variants=(["dd-lib-dotnet-init-test-app"]="dotnet" +variants=(["dd-lib-java-init-test-app"]="java") +variants2=(["dd-lib-dotnet-init-test-app"]="dotnet" ["sample-app"]="nodejs" ["dd-lib-python-init-test-django"]="python" ["dd-lib-python-init-test-django-gunicorn"]="python" @@ -15,6 +15,7 @@ variants=(["dd-lib-dotnet-init-test-app"]="dotnet" ["dd-lib-ruby-init-test-rails-conflict"]="ruby" ["dd-lib-ruby-init-test-rails-explicit"]="ruby" ["dd-lib-ruby-init-test-rails-gemsrb"]="ruby" + ["dd-lib-java-init-test-app"]="java" ) docker buildx create --name multiarch --driver docker-container --use From a05488898693807a71bb638236a71f0dc33bc94c Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 20:58:30 +0200 Subject: [PATCH 33/49] java docker build --- lib-injection/build/build_lib_injection_images.sh | 2 +- .../docker/java/dd-lib-java-init-test-app/Dockerfile | 9 +++++++++ .../build/docker/java/dd-lib-java-init-test-app/build.sh | 7 +++++-- 3 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 lib-injection/build/docker/java/dd-lib-java-init-test-app/Dockerfile diff --git a/lib-injection/build/build_lib_injection_images.sh b/lib-injection/build/build_lib_injection_images.sh index 8e02bed5508..eb2c79485ba 100755 --- a/lib-injection/build/build_lib_injection_images.sh +++ b/lib-injection/build/build_lib_injection_images.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -export DOCKER_IMAGE_WEBLOG_TAG=latest +export DOCKER_IMAGE_WEBLOG_TAG=latest_snapshot export BUILDX_PLATFORMS=linux/arm64/v8,linux/amd64 declare -A variants variants=(["dd-lib-java-init-test-app"]="java") diff --git a/lib-injection/build/docker/java/dd-lib-java-init-test-app/Dockerfile b/lib-injection/build/docker/java/dd-lib-java-init-test-app/Dockerfile new file mode 100644 index 00000000000..9ede3500b13 --- /dev/null +++ b/lib-injection/build/docker/java/dd-lib-java-init-test-app/Dockerfile @@ -0,0 +1,9 @@ +FROM alpine:latest +RUN apk --no-cache add openjdk11 --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community +RUN apk add --no-cache bash +COPY . . +RUN ./gradlew build +ENTRYPOINT ["java", "-jar", "k8s-lib-injection-app-0.0.1-SNAPSHOT.jar"] + + + diff --git a/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh b/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh index c88b61a7585..985dd040619 100755 --- a/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh +++ b/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh @@ -1,4 +1,7 @@ #!/bin/bash -./gradlew -PdockerImageRepo=${APP_DOCKER_IMAGE_REPO} -PdockerImageTag=${DOCKER_IMAGE_WEBLOG_TAG} clean bootBuildImage - docker push ${APP_DOCKER_IMAGE_REPO}:${DOCKER_IMAGE_WEBLOG_TAG} \ No newline at end of file +if [ -z "${BUILDX_PLATFORMS}" ] ; then + BUILDX_PLATFORMS=`docker buildx imagetools inspect --raw python:3.9 | jq -r 'reduce (.manifests[] | [ .platform.os, .platform.architecture, .platform.variant ] | join("/") | sub("\\/$"; "")) as $item (""; . + "," + $item)' | sed 's/,//'` +fi +echo "Build for platforms: ${BUILDX_PLATFORMS}" +docker buildx build --platform ${BUILDX_PLATFORMS} --tag ${LIBRARY_INJECTION_TEST_APP_IMAGE} --push . \ No newline at end of file From 31dea4a26d9bf26cc01157ea8db4dc07540b72c2 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Wed, 10 Apr 2024 21:08:53 +0200 Subject: [PATCH 34/49] gitlab ci --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d119a5757f4..c16b857b3cd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -197,10 +197,10 @@ variables: done check_merge_labels: - #image: registry.ddbuild.io/images/ci_docker_base + image: registry.ddbuild.io/images/ci_docker_base tags: ["runner:docker"] #tags: [ "arch:amd64" ] - image: $CI_DOCKER_IMAGE_BASE/gitlab:current + #image: $CI_DOCKER_IMAGE_BASE/gitlab:current stage: before_tests allow_failure: true before_script: From b358138fdafbbab1beed44d41f8daffd134d8d66 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Thu, 11 Apr 2024 08:36:24 +0200 Subject: [PATCH 35/49] java build --- .github/workflows/run-lib-injection.yml | 13 ------------- .gitlab-ci.yml | 1 + lib-injection/build/build_lib_injection_images.sh | 2 +- .../java/dd-lib-java-init-test-app/Dockerfile | 2 +- 4 files changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/run-lib-injection.yml b/.github/workflows/run-lib-injection.yml index e6ad104b333..7eff79fdc31 100644 --- a/.github/workflows/run-lib-injection.yml +++ b/.github/workflows/run-lib-injection.yml @@ -97,19 +97,6 @@ jobs: LIBRARY_INJECTION_TEST_APP_IMAGE=$APP_DOCKER_IMAGE_REPO:${{ github.sha }} ./build.sh cd .. - - name: Build weblog latest base images - #If we execute on system-tests-dashboard, we can't push the images because we don't have the permissions. - #To asign the permissions, we need to configure the image on ghcr, but due to a issue we can't do it - #The case is opened with github support - if: github.ref == 'refs/heads/main' && github.event_name != 'schedule' - env: - DOCKER_IMAGE_WEBLOG_TAG: latest - APP_DOCKER_IMAGE_REPO: ghcr.io/datadog/system-tests/${{ matrix.variant.weblog-variant }} - run: | - cd lib-injection/build/docker/$TEST_LIBRARY/$WEBLOG_VARIANT - LIBRARY_INJECTION_TEST_APP_IMAGE=$APP_DOCKER_IMAGE_REPO:latest ./build.sh - cd .. - - name: Install runner uses: ./.github/actions/install_runner diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c16b857b3cd..2586cd49db8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -197,6 +197,7 @@ variables: done check_merge_labels: + #Build docker images if it's needed. Check if the PR has the labels associated with the image build image: registry.ddbuild.io/images/ci_docker_base tags: ["runner:docker"] #tags: [ "arch:amd64" ] diff --git a/lib-injection/build/build_lib_injection_images.sh b/lib-injection/build/build_lib_injection_images.sh index eb2c79485ba..4b2050f4dd6 100755 --- a/lib-injection/build/build_lib_injection_images.sh +++ b/lib-injection/build/build_lib_injection_images.sh @@ -17,7 +17,7 @@ variants2=(["dd-lib-dotnet-init-test-app"]="dotnet" ["dd-lib-ruby-init-test-rails-gemsrb"]="ruby" ["dd-lib-java-init-test-app"]="java" ) -docker buildx create --name multiarch --driver docker-container --use +#docker buildx create --name multiarch --driver docker-container --use for variant in "${!variants[@]}"; do language="${variants[$variant]}" diff --git a/lib-injection/build/docker/java/dd-lib-java-init-test-app/Dockerfile b/lib-injection/build/docker/java/dd-lib-java-init-test-app/Dockerfile index 9ede3500b13..3ad15c55612 100644 --- a/lib-injection/build/docker/java/dd-lib-java-init-test-app/Dockerfile +++ b/lib-injection/build/docker/java/dd-lib-java-init-test-app/Dockerfile @@ -3,7 +3,7 @@ RUN apk --no-cache add openjdk11 --repository=http://dl-cdn.alpinelinux.org/alpi RUN apk add --no-cache bash COPY . . RUN ./gradlew build -ENTRYPOINT ["java", "-jar", "k8s-lib-injection-app-0.0.1-SNAPSHOT.jar"] +ENTRYPOINT ["java", "-jar", "build/libs/k8s-lib-injection-app-0.0.1-SNAPSHOT.jar"] From f0425d6f20bf5ead67d34c6fcf6f8f2233bedb1a Mon Sep 17 00:00:00 2001 From: roberto montero Date: Thu, 11 Apr 2024 09:02:03 +0200 Subject: [PATCH 36/49] last try --- lib-injection/build/build_lib_injection_images.sh | 7 +++---- utils/scripts/get_pr_merged_labels.sh | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib-injection/build/build_lib_injection_images.sh b/lib-injection/build/build_lib_injection_images.sh index 4b2050f4dd6..13f44329afa 100755 --- a/lib-injection/build/build_lib_injection_images.sh +++ b/lib-injection/build/build_lib_injection_images.sh @@ -1,11 +1,10 @@ #!/usr/bin/env bash set -e -export DOCKER_IMAGE_WEBLOG_TAG=latest_snapshot +export DOCKER_IMAGE_WEBLOG_TAG=latest export BUILDX_PLATFORMS=linux/arm64/v8,linux/amd64 declare -A variants -variants=(["dd-lib-java-init-test-app"]="java") -variants2=(["dd-lib-dotnet-init-test-app"]="dotnet" +variants=(["dd-lib-dotnet-init-test-app"]="dotnet" ["sample-app"]="nodejs" ["dd-lib-python-init-test-django"]="python" ["dd-lib-python-init-test-django-gunicorn"]="python" @@ -17,7 +16,7 @@ variants2=(["dd-lib-dotnet-init-test-app"]="dotnet" ["dd-lib-ruby-init-test-rails-gemsrb"]="ruby" ["dd-lib-java-init-test-app"]="java" ) -#docker buildx create --name multiarch --driver docker-container --use +docker buildx create --name multiarch --driver docker-container --use for variant in "${!variants[@]}"; do language="${variants[$variant]}" diff --git a/utils/scripts/get_pr_merged_labels.sh b/utils/scripts/get_pr_merged_labels.sh index e6a5bb1bf28..f10feb6780e 100755 --- a/utils/scripts/get_pr_merged_labels.sh +++ b/utils/scripts/get_pr_merged_labels.sh @@ -31,6 +31,7 @@ if [[ "1" == "1" ]]; then exit 0 fi echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin + #BUILD BUDDIES IMAGES if [ -z "$is_build_buddies" ] then From be9bd70e7a780a4c1e89b8cf8db9459afb78073c Mon Sep 17 00:00:00 2001 From: roberto montero Date: Thu, 11 Apr 2024 09:31:43 +0200 Subject: [PATCH 37/49] ready --- .github/workflows/run-end-to-end.yml | 1 - .gitlab-ci.yml | 51 ++++++++------------------- .gitlab/Dockerfile.gitlab | 48 ------------------------- utils/scripts/get_pr_merged_labels.sh | 12 +++---- 4 files changed, 19 insertions(+), 93 deletions(-) delete mode 100644 .gitlab/Dockerfile.gitlab diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index d206c35466c..2b22ee5aee0 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -69,7 +69,6 @@ env: jobs: end-to-end: - if: ${{ false }} # disable for now runs-on: ubuntu-latest strategy: matrix: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2586cd49db8..2f7f8818d35 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,10 +1,10 @@ stages: - #- ruby_tracer - #- nodejs_tracer - #- java_tracer - #- python_tracer - #- dotnet_tracer - #- parse_results + - ruby_tracer + - nodejs_tracer + - java_tracer + - python_tracer + - dotnet_tracer + - parse_results - before_tests - manual_images @@ -47,11 +47,8 @@ variables: # Do not modify this - must be the repository name for Kubernetes gitlab runners to run KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: system-tests #helm-charts TEST: 1 - DOCKER_REGISTRY: registry.ddbuild.io - CI_DOCKER_IMAGE_BASE: $DOCKER_REGISTRY/ci/system-tests - PROMOTION_TAG: current -.onboarding_nodejs: +onboarding_nodejs: extends: .base_job_onboarding stage: nodejs_tracer allow_failure: true @@ -76,7 +73,7 @@ variables: - ./build.sh -i runner - timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws -.onboarding_java: +onboarding_java: extends: .base_job_onboarding stage: java_tracer allow_failure: true @@ -100,7 +97,7 @@ variables: - ./build.sh -i runner - timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws -.onboarding_python: +onboarding_python: extends: .base_job_onboarding stage: python_tracer allow_failure: true @@ -124,7 +121,7 @@ variables: - ./build.sh -i runner - timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws -.onboarding_dotnet: +onboarding_dotnet: extends: .base_job_onboarding stage: dotnet_tracer allow_failure: true @@ -148,7 +145,7 @@ variables: - ./build.sh -i runner - timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws -.onboarding_ruby: +onboarding_ruby: extends: .base_job_onboarding stage: ruby_tracer allow_failure: true @@ -172,7 +169,7 @@ variables: - ./build.sh -i runner - timeout 2700s ./run.sh $SCENARIO --vm-weblog ${ONBOARDING_FILTER_WEBLOG} --vm-env ${ONBOARDING_FILTER_ENV} --vm-library ${TEST_LIBRARY} --vm-provider aws -.onboarding_parse_results: +onboarding_parse_results: image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner:a58cc31c tags: ["arch:amd64"] stage: parse_results @@ -200,8 +197,7 @@ check_merge_labels: #Build docker images if it's needed. Check if the PR has the labels associated with the image build image: registry.ddbuild.io/images/ci_docker_base tags: ["runner:docker"] - #tags: [ "arch:amd64" ] - #image: $CI_DOCKER_IMAGE_BASE/gitlab:current + stage: before_tests allow_failure: true before_script: @@ -210,24 +206,5 @@ check_merge_labels: - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) script: - ./utils/scripts/get_pr_merged_labels.sh - #rules: - # - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" - -build-image: - stage: manual_images rules: - #- if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "push" - # changes: - # - gitlab/Dockerfile.* - - when: manual - allow_failure: true - tags: [ "arch:amd64" ] - image: $DOCKER_REGISTRY/docker:20.10.13 - script: - - > - docker buildx build - --tag $CI_DOCKER_IMAGE_BASE/gitlab:current - --platform linux/amd64 - --push - --file .gitlab/Dockerfile.gitlab - . \ No newline at end of file + - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" \ No newline at end of file diff --git a/.gitlab/Dockerfile.gitlab b/.gitlab/Dockerfile.gitlab deleted file mode 100644 index ec2ca609a55..00000000000 --- a/.gitlab/Dockerfile.gitlab +++ /dev/null @@ -1,48 +0,0 @@ -FROM registry.ddbuild.io/base:focal - -USER root - -RUN clean-apt install \ - binutils \ - rpm \ - jq \ - rubygems \ - default-jdk \ - awscli \ - ruby-full \ - build-essential \ - ca-certificates \ - python3-pip \ - unzip \ - git - -# Newer versions of dotenv don't work with this version of ruby -RUN gem install dotenv -v 2.8.1 -RUN gem install fpm - -RUN curl -fsSL https://deb.nodesource.com/setup_21.x | bash - && apt-get install -y nodejs - - -#https://docs.docker.com/engine/install/ubuntu/ + buildx -RUN apt-get -y install ca-certificates curl gnupg -RUN install -m 0755 -d /etc/apt/keyrings -RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg -RUN chmod a+r /etc/apt/keyrings/docker.gpg -RUN echo \ - "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ - "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ - tee /etc/apt/sources.list.d/docker.list > /dev/null -RUN apt-get update -RUN apt-get -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -RUN curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/bin/docker-compose && chmod +x /usr/bin/docker-compose && docker-compose --version -RUN echo "DOCKER INSTALLED!" -#To run Docker without root privileges -#RUN groupadd docker -#RUN usermod -aG docker "ubuntu" -#RUN echo "DOCKER USER ADDED" -RUN apt install -y docker-buildx-plugin -RUN docker buildx install -RUN usermod -aG docker root -#FIX ulimit -RUN sed -i 's\ulimit -Hn 524288\ulimit -n 524288\g' /etc/init.d/docker -RUN service docker restart \ No newline at end of file diff --git a/utils/scripts/get_pr_merged_labels.sh b/utils/scripts/get_pr_merged_labels.sh index f10feb6780e..a539765ed79 100755 --- a/utils/scripts/get_pr_merged_labels.sh +++ b/utils/scripts/get_pr_merged_labels.sh @@ -1,16 +1,14 @@ #!/bin/bash -# shellcheck disable=SC2164,SC2050 +# shellcheck disable=SC2164 #Check the commit message to figure out if we are merging a PR. #We extract the PR number and using GitHub API we check the PR labels. #If the PR contains the label "build-buddies-images" we launch the build and push process -#PR_PATTERN='#[0-9]+' +PR_PATTERN='#[0-9]+' -#if [[ $CI_COMMIT_MESSAGE =~ ($PR_PATTERN) ]]; then -if [[ "1" == "1" ]]; then - #PR_NUMBER=${BASH_REMATCH[1]:1} - PR_NUMBER=2321 +if [[ $CI_COMMIT_MESSAGE =~ ($PR_PATTERN) ]]; then + PR_NUMBER=${BASH_REMATCH[1]:1} echo "Merged the PR number: [$PR_NUMBER]"; #search for labels PR_DATA=$(curl -L \ @@ -31,7 +29,7 @@ if [[ "1" == "1" ]]; then exit 0 fi echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin - + #BUILD BUDDIES IMAGES if [ -z "$is_build_buddies" ] then From d8dd70a350307bc205f101a330a67e6b98a68ca9 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Thu, 11 Apr 2024 10:11:59 +0200 Subject: [PATCH 38/49] unused code --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2f7f8818d35..756bdee6a6e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,6 @@ stages: - dotnet_tracer - parse_results - before_tests - - manual_images .base_job_onboarding: only: From 23a106bd8629ed15d7345f56a0a052d14e85c5b8 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Mon, 15 Apr 2024 13:04:14 +0200 Subject: [PATCH 39/49] launch k8s tests --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 756bdee6a6e..4fb77c7259a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -193,7 +193,7 @@ onboarding_parse_results: done check_merge_labels: - #Build docker images if it's needed. Check if the PR has the labels associated with the image build + #Build docker images if it's needed. Check if the PR has the labels associated with the image build. image: registry.ddbuild.io/images/ci_docker_base tags: ["runner:docker"] From d068b5d28511a2aef03666978c0c54c8109e7ba5 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Mon, 15 Apr 2024 17:04:59 +0200 Subject: [PATCH 40/49] downgrade buildx. There is a bug --- .github/workflows/run-lib-injection.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-lib-injection.yml b/.github/workflows/run-lib-injection.yml index 7eff79fdc31..ed8b24f547c 100644 --- a/.github/workflows/run-lib-injection.yml +++ b/.github/workflows/run-lib-injection.yml @@ -75,11 +75,12 @@ jobs: id: buildx uses: docker/setup-buildx-action@v2 with: + version: v0.12.1 # 0.13.0 is causing the builds to fail install: true config-inline: | [worker.oci] max-parallelism = 1 - + - name: Log in to the Container registry uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # 3.0.0 with: From 043d43c5e523ebc06e5f735866af520b6d3db791 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Mon, 15 Apr 2024 19:31:37 +0200 Subject: [PATCH 41/49] echo docker buildx version --- .../build/docker/java/dd-lib-java-init-test-app/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh b/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh index 985dd040619..0afe1b606eb 100755 --- a/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh +++ b/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh @@ -4,4 +4,5 @@ if [ -z "${BUILDX_PLATFORMS}" ] ; then BUILDX_PLATFORMS=`docker buildx imagetools inspect --raw python:3.9 | jq -r 'reduce (.manifests[] | [ .platform.os, .platform.architecture, .platform.variant ] | join("/") | sub("\\/$"; "")) as $item (""; . + "," + $item)' | sed 's/,//'` fi echo "Build for platforms: ${BUILDX_PLATFORMS}" +echo "Docker buildx version: $(docker buildx version)" docker buildx build --platform ${BUILDX_PLATFORMS} --tag ${LIBRARY_INJECTION_TEST_APP_IMAGE} --push . \ No newline at end of file From 8915e887a26d604445c792cd389e8b2f317b760f Mon Sep 17 00:00:00 2001 From: roberto montero Date: Mon, 15 Apr 2024 19:59:00 +0200 Subject: [PATCH 42/49] test without platform parameter --- .../build/docker/java/dd-lib-java-init-test-app/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh b/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh index 0afe1b606eb..80bc48faa48 100755 --- a/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh +++ b/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh @@ -5,4 +5,4 @@ if [ -z "${BUILDX_PLATFORMS}" ] ; then fi echo "Build for platforms: ${BUILDX_PLATFORMS}" echo "Docker buildx version: $(docker buildx version)" -docker buildx build --platform ${BUILDX_PLATFORMS} --tag ${LIBRARY_INJECTION_TEST_APP_IMAGE} --push . \ No newline at end of file +docker buildx build --tag ${LIBRARY_INJECTION_TEST_APP_IMAGE} --push . \ No newline at end of file From 2244c168b377cc5f267f7336df93c2eb3c63b9fe Mon Sep 17 00:00:00 2001 From: roberto montero Date: Mon, 15 Apr 2024 20:47:12 +0200 Subject: [PATCH 43/49] test arch --- .github/workflows/run-lib-injection.yml | 1 + .../build/docker/java/dd-lib-java-init-test-app/build.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-lib-injection.yml b/.github/workflows/run-lib-injection.yml index ed8b24f547c..5a44a8aff51 100644 --- a/.github/workflows/run-lib-injection.yml +++ b/.github/workflows/run-lib-injection.yml @@ -77,6 +77,7 @@ jobs: with: version: v0.12.1 # 0.13.0 is causing the builds to fail install: true + provenance: false config-inline: | [worker.oci] max-parallelism = 1 diff --git a/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh b/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh index 80bc48faa48..0afe1b606eb 100755 --- a/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh +++ b/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh @@ -5,4 +5,4 @@ if [ -z "${BUILDX_PLATFORMS}" ] ; then fi echo "Build for platforms: ${BUILDX_PLATFORMS}" echo "Docker buildx version: $(docker buildx version)" -docker buildx build --tag ${LIBRARY_INJECTION_TEST_APP_IMAGE} --push . \ No newline at end of file +docker buildx build --platform ${BUILDX_PLATFORMS} --tag ${LIBRARY_INJECTION_TEST_APP_IMAGE} --push . \ No newline at end of file From 708df2c8a284dcab0f9cfa5dab2cad4166e3ba32 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Mon, 15 Apr 2024 21:14:36 +0200 Subject: [PATCH 44/49] remove buildx multiarch --- .../build/docker/java/dd-lib-java-init-test-app/build.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh b/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh index 0afe1b606eb..efc662d72ec 100755 --- a/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh +++ b/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh @@ -1,8 +1,4 @@ #!/bin/bash -if [ -z "${BUILDX_PLATFORMS}" ] ; then - BUILDX_PLATFORMS=`docker buildx imagetools inspect --raw python:3.9 | jq -r 'reduce (.manifests[] | [ .platform.os, .platform.architecture, .platform.variant ] | join("/") | sub("\\/$"; "")) as $item (""; . + "," + $item)' | sed 's/,//'` -fi -echo "Build for platforms: ${BUILDX_PLATFORMS}" -echo "Docker buildx version: $(docker buildx version)" -docker buildx build --platform ${BUILDX_PLATFORMS} --tag ${LIBRARY_INJECTION_TEST_APP_IMAGE} --push . \ No newline at end of file +docker build --tag ${LIBRARY_INJECTION_TEST_APP_IMAGE} . +docker push ${LIBRARY_INJECTION_TEST_APP_IMAGE} \ No newline at end of file From d90d9920970546d9d082d662ae171ee16c24d045 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Tue, 16 Apr 2024 07:42:15 +0200 Subject: [PATCH 45/49] disable buildkit java --- .../build/docker/java/dd-lib-java-init-test-app/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh b/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh index efc662d72ec..6ae7d681a87 100755 --- a/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh +++ b/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh @@ -1,4 +1,4 @@ #!/bin/bash - +DOCKER_BUILDKIT=0 docker build --tag ${LIBRARY_INJECTION_TEST_APP_IMAGE} . docker push ${LIBRARY_INJECTION_TEST_APP_IMAGE} \ No newline at end of file From e43ac5ab90177e6faf8156c9fd4681e13c37210a Mon Sep 17 00:00:00 2001 From: roberto montero Date: Tue, 16 Apr 2024 07:51:59 +0200 Subject: [PATCH 46/49] disable buildkit java --- .../build/docker/java/dd-lib-java-init-test-app/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh b/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh index 6ae7d681a87..769cad76f03 100755 --- a/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh +++ b/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh @@ -1,4 +1,4 @@ #!/bin/bash -DOCKER_BUILDKIT=0 -docker build --tag ${LIBRARY_INJECTION_TEST_APP_IMAGE} . -docker push ${LIBRARY_INJECTION_TEST_APP_IMAGE} \ No newline at end of file + +DOCKER_BUILDKIT=0 docker build --tag ${LIBRARY_INJECTION_TEST_APP_IMAGE} . +DOCKER_BUILDKIT=0 docker push ${LIBRARY_INJECTION_TEST_APP_IMAGE} \ No newline at end of file From 2e937e73b288b07cb16a15a387fde55b690eb4c1 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Tue, 16 Apr 2024 09:07:50 +0200 Subject: [PATCH 47/49] debug logs k8s --- utils/k8s_lib_injection/k8s_weblog.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/utils/k8s_lib_injection/k8s_weblog.py b/utils/k8s_lib_injection/k8s_weblog.py index 2ee7dc4b347..95ca459607c 100644 --- a/utils/k8s_lib_injection/k8s_weblog.py +++ b/utils/k8s_lib_injection/k8s_weblog.py @@ -355,10 +355,13 @@ def export_debug_info(self): app_name = f"{self.library}-app" try: pods = self.k8s_wrapper.list_namespaced_pod("default", label_selector=f"app={app_name}") - if len(pods.items) > 0: - api_response = self.k8s_wrapper.read_namespaced_pod(pods.items[0].metadata.name) + for index in range(len(pods.items)): + k8s_logger(self.output_folder, self.test_name, "deployment.logs").info( + "-----------------------------------------------" + ) + api_response = self.k8s_wrapper.read_namespaced_pod(pods.items[index].metadata.name) k8s_logger(self.output_folder, self.test_name, "deployment.logs").info(api_response) - api_response = self.k8s_wrapper.read_namespaced_pod_log(name=pods.items[0].metadata.name) + api_response = self.k8s_wrapper.read_namespaced_pod_log(name=pods.items[index].metadata.name) k8s_logger(self.output_folder, self.test_name, "deployment.logs").info(api_response) except Exception as e: k8s_logger(self.output_folder, self.test_name, "deployment.logs").info( From 6c2f6fccbf8cc4bab97f64ea76edab51451ad439 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Thu, 18 Apr 2024 08:13:15 +0200 Subject: [PATCH 48/49] Change entry point docker --- .github/workflows/run-lib-injection.yml | 32 +++++++++---------- .../java/dd-lib-java-init-test-app/Dockerfile | 4 ++- .../java/dd-lib-java-init-test-app/app.sh | 4 +++ 3 files changed, 23 insertions(+), 17 deletions(-) create mode 100755 lib-injection/build/docker/java/dd-lib-java-init-test-app/app.sh diff --git a/.github/workflows/run-lib-injection.yml b/.github/workflows/run-lib-injection.yml index 5a44a8aff51..2dccc323c44 100644 --- a/.github/workflows/run-lib-injection.yml +++ b/.github/workflows/run-lib-injection.yml @@ -24,29 +24,29 @@ jobs: - library: java weblog-variant: dd-lib-java-init-test-app - - library: nodejs - weblog-variant: sample-app + # - library: nodejs + # weblog-variant: sample-app - - library: python - weblog-variant: dd-lib-python-init-test-django + # - library: python + # weblog-variant: dd-lib-python-init-test-django - - library: python - weblog-variant: dd-lib-python-init-test-django-gunicorn + # - library: python + # weblog-variant: dd-lib-python-init-test-django-gunicorn - - library: python - weblog-variant: dd-lib-python-init-test-django-uvicorn + # - library: python + # weblog-variant: dd-lib-python-init-test-django-uvicorn - - library: ruby - weblog-variant: dd-lib-ruby-init-test-rails + # - library: ruby + # weblog-variant: dd-lib-ruby-init-test-rails - - library: ruby - weblog-variant: dd-lib-ruby-init-test-rails-explicit + # - library: ruby + # weblog-variant: dd-lib-ruby-init-test-rails-explicit - - library: ruby - weblog-variant: dd-lib-ruby-init-test-rails-gemsrb + # - library: ruby + # weblog-variant: dd-lib-ruby-init-test-rails-gemsrb - - library: dotnet - weblog-variant: dd-lib-dotnet-init-test-app + # - library: dotnet + # weblog-variant: dd-lib-dotnet-init-test-app version: - latest diff --git a/lib-injection/build/docker/java/dd-lib-java-init-test-app/Dockerfile b/lib-injection/build/docker/java/dd-lib-java-init-test-app/Dockerfile index 3ad15c55612..203a294d197 100644 --- a/lib-injection/build/docker/java/dd-lib-java-init-test-app/Dockerfile +++ b/lib-injection/build/docker/java/dd-lib-java-init-test-app/Dockerfile @@ -3,7 +3,9 @@ RUN apk --no-cache add openjdk11 --repository=http://dl-cdn.alpinelinux.org/alpi RUN apk add --no-cache bash COPY . . RUN ./gradlew build -ENTRYPOINT ["java", "-jar", "build/libs/k8s-lib-injection-app-0.0.1-SNAPSHOT.jar"] +RUN echo +#ENTRYPOINT ["java", "-jar", "build/libs/k8s-lib-injection-app-0.0.1-SNAPSHOT.jar"] +CMD [ "app.sh" ] diff --git a/lib-injection/build/docker/java/dd-lib-java-init-test-app/app.sh b/lib-injection/build/docker/java/dd-lib-java-init-test-app/app.sh new file mode 100755 index 00000000000..2385b289a86 --- /dev/null +++ b/lib-injection/build/docker/java/dd-lib-java-init-test-app/app.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +# shellcheck disable=SC2086 +exec java -jar build/libs/k8s-lib-injection-app-0.0.1-SNAPSHOT.jar \ No newline at end of file From d2d57621881cc95316b3b7139948703381a442a7 Mon Sep 17 00:00:00 2001 From: roberto montero Date: Thu, 18 Apr 2024 08:58:37 +0200 Subject: [PATCH 49/49] no build lib injeciton images --- .github/workflows/run-lib-injection.yml | 45 ++++++++++++------- .../java/dd-lib-java-init-test-app/Dockerfile | 4 +- .../java/dd-lib-java-init-test-app/app.sh | 4 -- .../java/dd-lib-java-init-test-app/build.sh | 4 +- utils/scripts/get_pr_merged_labels.sh | 3 +- 5 files changed, 35 insertions(+), 25 deletions(-) delete mode 100755 lib-injection/build/docker/java/dd-lib-java-init-test-app/app.sh diff --git a/.github/workflows/run-lib-injection.yml b/.github/workflows/run-lib-injection.yml index 2dccc323c44..ff8733f2a14 100644 --- a/.github/workflows/run-lib-injection.yml +++ b/.github/workflows/run-lib-injection.yml @@ -24,29 +24,29 @@ jobs: - library: java weblog-variant: dd-lib-java-init-test-app - # - library: nodejs - # weblog-variant: sample-app + - library: nodejs + weblog-variant: sample-app - # - library: python - # weblog-variant: dd-lib-python-init-test-django + - library: python + weblog-variant: dd-lib-python-init-test-django - # - library: python - # weblog-variant: dd-lib-python-init-test-django-gunicorn + - library: python + weblog-variant: dd-lib-python-init-test-django-gunicorn - # - library: python - # weblog-variant: dd-lib-python-init-test-django-uvicorn + - library: python + weblog-variant: dd-lib-python-init-test-django-uvicorn - # - library: ruby - # weblog-variant: dd-lib-ruby-init-test-rails + - library: ruby + weblog-variant: dd-lib-ruby-init-test-rails - # - library: ruby - # weblog-variant: dd-lib-ruby-init-test-rails-explicit + - library: ruby + weblog-variant: dd-lib-ruby-init-test-rails-explicit - # - library: ruby - # weblog-variant: dd-lib-ruby-init-test-rails-gemsrb + - library: ruby + weblog-variant: dd-lib-ruby-init-test-rails-gemsrb - # - library: dotnet - # weblog-variant: dd-lib-dotnet-init-test-app + - library: dotnet + weblog-variant: dd-lib-dotnet-init-test-app version: - latest @@ -98,6 +98,19 @@ jobs: cd lib-injection/build/docker/$TEST_LIBRARY/$WEBLOG_VARIANT LIBRARY_INJECTION_TEST_APP_IMAGE=$APP_DOCKER_IMAGE_REPO:${{ github.sha }} ./build.sh cd .. + + - name: Build weblog latest base images + #If we execute on system-tests-dashboard, we can't push the images because we don't have the permissions. + #To asign the permissions, we need to configure the image on ghcr, but due to a issue we can't do it + #The case is opened with github support + if: github.ref == 'refs/heads/main' && github.event_name != 'schedule' + env: + DOCKER_IMAGE_WEBLOG_TAG: latest + APP_DOCKER_IMAGE_REPO: ghcr.io/datadog/system-tests/${{ matrix.variant.weblog-variant }} + run: | + cd lib-injection/build/docker/$TEST_LIBRARY/$WEBLOG_VARIANT + LIBRARY_INJECTION_TEST_APP_IMAGE=$APP_DOCKER_IMAGE_REPO:latest ./build.sh + cd .. - name: Install runner uses: ./.github/actions/install_runner diff --git a/lib-injection/build/docker/java/dd-lib-java-init-test-app/Dockerfile b/lib-injection/build/docker/java/dd-lib-java-init-test-app/Dockerfile index 203a294d197..c515fcec481 100644 --- a/lib-injection/build/docker/java/dd-lib-java-init-test-app/Dockerfile +++ b/lib-injection/build/docker/java/dd-lib-java-init-test-app/Dockerfile @@ -4,8 +4,8 @@ RUN apk add --no-cache bash COPY . . RUN ./gradlew build RUN echo -#ENTRYPOINT ["java", "-jar", "build/libs/k8s-lib-injection-app-0.0.1-SNAPSHOT.jar"] -CMD [ "app.sh" ] +ENTRYPOINT ["java", "-jar", "build/libs/k8s-lib-injection-app-0.0.1-SNAPSHOT.jar"] + diff --git a/lib-injection/build/docker/java/dd-lib-java-init-test-app/app.sh b/lib-injection/build/docker/java/dd-lib-java-init-test-app/app.sh deleted file mode 100755 index 2385b289a86..00000000000 --- a/lib-injection/build/docker/java/dd-lib-java-init-test-app/app.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -set -eu -# shellcheck disable=SC2086 -exec java -jar build/libs/k8s-lib-injection-app-0.0.1-SNAPSHOT.jar \ No newline at end of file diff --git a/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh b/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh index 769cad76f03..c88b61a7585 100755 --- a/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh +++ b/lib-injection/build/docker/java/dd-lib-java-init-test-app/build.sh @@ -1,4 +1,4 @@ #!/bin/bash -DOCKER_BUILDKIT=0 docker build --tag ${LIBRARY_INJECTION_TEST_APP_IMAGE} . -DOCKER_BUILDKIT=0 docker push ${LIBRARY_INJECTION_TEST_APP_IMAGE} \ No newline at end of file +./gradlew -PdockerImageRepo=${APP_DOCKER_IMAGE_REPO} -PdockerImageTag=${DOCKER_IMAGE_WEBLOG_TAG} clean bootBuildImage + docker push ${APP_DOCKER_IMAGE_REPO}:${DOCKER_IMAGE_WEBLOG_TAG} \ No newline at end of file diff --git a/utils/scripts/get_pr_merged_labels.sh b/utils/scripts/get_pr_merged_labels.sh index a539765ed79..c13a4b49ead 100755 --- a/utils/scripts/get_pr_merged_labels.sh +++ b/utils/scripts/get_pr_merged_labels.sh @@ -21,7 +21,8 @@ if [[ $CI_COMMIT_MESSAGE =~ ($PR_PATTERN) ]]; then is_build_buddies=$(echo "$PR_DATA" | jq -c '.[] | select(.name | contains("build-buddies-images"))'); is_build_python_base_images=$(echo "$PR_DATA" | jq -c '.[] | select(.name | contains("build-python-base-images"))'); - is_build_lib_injection_app_images=$(echo "$PR_DATA" | jq -c '.[] | select(.name | contains("build-lib-injection-app-images"))'); + #Disable build lib injection until problems with the java app are fixed + is_build_lib_injection_app_images=$(echo "$PR_DATA" | jq -c '.[] | select(.name | contains("build-lib-injection-app-images-DISABLED"))'); if [ -z "$is_build_buddies" ] && [ -z "$is_build_python_base_images" ] && [ -z "$is_build_lib_injection_app_images" ] then