diff --git a/.github/workflows/run-lib-injection.yml b/.github/workflows/run-lib-injection.yml index e6ad104b333..ff8733f2a14 100644 --- a/.github/workflows/run-lib-injection.yml +++ b/.github/workflows/run-lib-injection.yml @@ -75,11 +75,13 @@ jobs: id: buildx uses: docker/setup-buildx-action@v2 with: + version: v0.12.1 # 0.13.0 is causing the builds to fail install: true + provenance: false config-inline: | [worker.oci] max-parallelism = 1 - + - name: Log in to the Container registry uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # 3.0.0 with: @@ -96,7 +98,7 @@ jobs: cd lib-injection/build/docker/$TEST_LIBRARY/$WEBLOG_VARIANT LIBRARY_INJECTION_TEST_APP_IMAGE=$APP_DOCKER_IMAGE_REPO:${{ github.sha }} ./build.sh cd .. - + - name: Build weblog latest base images #If we execute on system-tests-dashboard, we can't push the images because we don't have the permissions. #To asign the permissions, we need to configure the image on ghcr, but due to a issue we can't do it diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b3061ee5c65..4fb77c7259a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -193,9 +193,12 @@ onboarding_parse_results: done check_merge_labels: + #Build docker images if it's needed. Check if the PR has the labels associated with the image build. image: registry.ddbuild.io/images/ci_docker_base tags: ["runner:docker"] + stage: before_tests + allow_failure: true before_script: - export GH_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.gh-token --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) diff --git a/lib-injection/build/build_lib_injection_images.sh b/lib-injection/build/build_lib_injection_images.sh new file mode 100755 index 00000000000..13f44329afa --- /dev/null +++ b/lib-injection/build/build_lib_injection_images.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -e + +export DOCKER_IMAGE_WEBLOG_TAG=latest +export BUILDX_PLATFORMS=linux/arm64/v8,linux/amd64 +declare -A variants +variants=(["dd-lib-dotnet-init-test-app"]="dotnet" + ["sample-app"]="nodejs" + ["dd-lib-python-init-test-django"]="python" + ["dd-lib-python-init-test-django-gunicorn"]="python" + ["dd-lib-python-init-test-django-uvicorn"]="python" + ["dd-lib-ruby-init-test-rails"]="ruby" + ["dd-lib-ruby-init-test-rails-bundle-deploy"]="ruby" + ["dd-lib-ruby-init-test-rails-conflict"]="ruby" + ["dd-lib-ruby-init-test-rails-explicit"]="ruby" + ["dd-lib-ruby-init-test-rails-gemsrb"]="ruby" + ["dd-lib-java-init-test-app"]="java" + ) +docker buildx create --name multiarch --driver docker-container --use + +for variant in "${!variants[@]}"; do + language="${variants[$variant]}" + echo "Building $variant - $language"; + echo "$(pwd)" + cd ./lib-injection/build/docker/$language/$variant/ && APP_DOCKER_IMAGE_REPO=ghcr.io/datadog/system-tests/$variant LIBRARY_INJECTION_TEST_APP_IMAGE=ghcr.io/datadog/system-tests/$variant:$DOCKER_IMAGE_WEBLOG_TAG ./build.sh && cd ../../../../../ +done \ No newline at end of file diff --git a/lib-injection/build/docker/java/dd-lib-java-init-test-app/Dockerfile b/lib-injection/build/docker/java/dd-lib-java-init-test-app/Dockerfile new file mode 100644 index 00000000000..c515fcec481 --- /dev/null +++ b/lib-injection/build/docker/java/dd-lib-java-init-test-app/Dockerfile @@ -0,0 +1,11 @@ +FROM alpine:latest +RUN apk --no-cache add openjdk11 --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community +RUN apk add --no-cache bash +COPY . . +RUN ./gradlew build +RUN echo +ENTRYPOINT ["java", "-jar", "build/libs/k8s-lib-injection-app-0.0.1-SNAPSHOT.jar"] + + + + diff --git a/lib-injection/build/docker/nodejs/sample-app/build.sh b/lib-injection/build/docker/nodejs/sample-app/build.sh index a44f6aefff1..985dd040619 100755 --- a/lib-injection/build/docker/nodejs/sample-app/build.sh +++ b/lib-injection/build/docker/nodejs/sample-app/build.sh @@ -3,4 +3,5 @@ if [ -z "${BUILDX_PLATFORMS}" ] ; then BUILDX_PLATFORMS=`docker buildx imagetools inspect --raw python:3.9 | jq -r 'reduce (.manifests[] | [ .platform.os, .platform.architecture, .platform.variant ] | join("/") | sub("\\/$"; "")) as $item (""; . + "," + $item)' | sed 's/,//'` fi +echo "Build for platforms: ${BUILDX_PLATFORMS}" docker buildx build --platform ${BUILDX_PLATFORMS} --tag ${LIBRARY_INJECTION_TEST_APP_IMAGE} --push . \ No newline at end of file diff --git a/utils/build/build_python_base_images.sh b/utils/build/build_python_base_images.sh new file mode 100755 index 00000000000..d694a4b5bd7 --- /dev/null +++ b/utils/build/build_python_base_images.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# build and push python base images + + +docker buildx build --load --progress=plain -f utils/build/docker/python/fastapi.base.Dockerfile -t datadog/system-tests:fastapi.base-v0 . +docker buildx build --load --progress=plain -f utils/build/docker/python/python3.12.base.Dockerfile -t datadog/system-tests:python3.12.base-v1 . +docker buildx build --load --progress=plain -f utils/build/docker/python/django-poc.base.Dockerfile -t datadog/system-tests:django-poc.base-v0 . +docker buildx build --load --progress=plain -f utils/build/docker/python/flask-poc.base.Dockerfile -t datadog/system-tests:flask-poc.base-v2 . +docker buildx build --load --progress=plain -f utils/build/docker/python/uwsgi-poc.base.Dockerfile -t datadog/system-tests:uwsgi-poc.base-v1 . + +if [ "$1" = "--push" ]; then + docker push datadog/system-tests:fastapi.base-v0 + docker push datadog/system-tests:python3.12.base-v1 + docker push datadog/system-tests:django-poc.base-v0 + docker push datadog/system-tests:flask-poc.base-v2 + docker push datadog/system-tests:uwsgi-poc.base-v1 +fi + diff --git a/utils/k8s_lib_injection/k8s_weblog.py b/utils/k8s_lib_injection/k8s_weblog.py index 2ee7dc4b347..95ca459607c 100644 --- a/utils/k8s_lib_injection/k8s_weblog.py +++ b/utils/k8s_lib_injection/k8s_weblog.py @@ -355,10 +355,13 @@ def export_debug_info(self): app_name = f"{self.library}-app" try: pods = self.k8s_wrapper.list_namespaced_pod("default", label_selector=f"app={app_name}") - if len(pods.items) > 0: - api_response = self.k8s_wrapper.read_namespaced_pod(pods.items[0].metadata.name) + for index in range(len(pods.items)): + k8s_logger(self.output_folder, self.test_name, "deployment.logs").info( + "-----------------------------------------------" + ) + api_response = self.k8s_wrapper.read_namespaced_pod(pods.items[index].metadata.name) k8s_logger(self.output_folder, self.test_name, "deployment.logs").info(api_response) - api_response = self.k8s_wrapper.read_namespaced_pod_log(name=pods.items[0].metadata.name) + api_response = self.k8s_wrapper.read_namespaced_pod_log(name=pods.items[index].metadata.name) k8s_logger(self.output_folder, self.test_name, "deployment.logs").info(api_response) except Exception as e: k8s_logger(self.output_folder, self.test_name, "deployment.logs").info( diff --git a/utils/scripts/get_pr_merged_labels.sh b/utils/scripts/get_pr_merged_labels.sh index 96d036957bb..c13a4b49ead 100755 --- a/utils/scripts/get_pr_merged_labels.sh +++ b/utils/scripts/get_pr_merged_labels.sh @@ -1,5 +1,5 @@ #!/bin/bash -# shellcheck disable=SC2164 +# shellcheck disable=SC2164 #Check the commit message to figure out if we are merging a PR. #We extract the PR number and using GitHub API we check the PR labels. #If the PR contains the label "build-buddies-images" we launch the build and push process @@ -20,15 +20,48 @@ if [[ $CI_COMMIT_MESSAGE =~ ($PR_PATTERN) ]]; then echo "We found PR labels: $PR_DATA" is_build_buddies=$(echo "$PR_DATA" | jq -c '.[] | select(.name | contains("build-buddies-images"))'); + is_build_python_base_images=$(echo "$PR_DATA" | jq -c '.[] | select(.name | contains("build-python-base-images"))'); + #Disable build lib injection until problems with the java app are fixed + is_build_lib_injection_app_images=$(echo "$PR_DATA" | jq -c '.[] | select(.name | contains("build-lib-injection-app-images-DISABLED"))'); + if [ -z "$is_build_buddies" ] && [ -z "$is_build_python_base_images" ] && [ -z "$is_build_lib_injection_app_images" ] + then + echo "The PR $PR_NUMBER doesn't contain any docker build label " + exit 0 + fi + echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin + + #BUILD BUDDIES IMAGES if [ -z "$is_build_buddies" ] then echo "The PR $PR_NUMBER doesn't contain the 'build-buddies-images' label " else echo "The PR $PR_NUMBER contains the 'build-buddies-images' label. Launching the images generation process " - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin ./utils/build/build_tracer_buddies.sh --push + echo "------------- The buddies images have been built and pushed ------------- " + fi + + #BUILD PYTHON BASE IMAGES + if [ -z "$is_build_python_base_images" ] + then + echo "The PR $PR_NUMBER doesn't contain the 'build-python-base-images' label " + else + echo "The PR $PR_NUMBER contains the 'build-python-base-images' label. Launching the images generation process " + ./utils/build/build_python_base_images.sh --push + echo "------------- The python base images have been built and pushed ------------- " + fi + + #BUILD LIB INJECTION IMAGES + if [ -z "$is_build_lib_injection_app_images" ] + then + echo "The PR $PR_NUMBER doesn't contain the 'build-lib-injection-app-images' label " + else + echo "The PR $PR_NUMBER contains the 'build-lib-injection-app-images' label. Launching the images generation process " + echo "$GH_TOKEN" | docker login ghcr.io --username "publisher" --password-stdin + ./lib-injection/build/build_lib_injection_images.sh + echo "------------- The lib injection weblog images have been built and pushed ------------- " fi + else echo "The commit message $CI_COMMIT_MESSAGE doesn't contain the PR number." fi \ No newline at end of file