Merge branch 'main' into vincent.whitchurch/procmontest

.circleci/config.yml

@@ -15,7 +15,7 @@ experimental:
 templates:
   job_template: &job_template
     docker:
-      - image: gcr.io/datadoghq/agent-circleci-runner:v32778237-8467c81e
+      - image: gcr.io/datadoghq/agent-circleci-runner:v34124919-31caaff7
         environment:
           USE_SYSTEM_LIBS: "1"
     working_directory: /go/src/github.com/DataDog/datadog-agent

.copyright-overrides.yml

@@ -100,6 +100,15 @@ github.com/jcmturner/*: Copyright 2016, Jonathan Turner <jt@jtnet.co.uk>
 # https://github.com/signalfx/com_signalfx_metrics_protobuf/README.md
 github.com/signalfx/com_signalfx_metrics_protobuf/*: Copyright © 2015-2017 SignalFx
 
+# https://github.com/SAP/go-hdb/blob/main/.reuse/dep5
+github.com/SAP/go-hdb/*: Copyright 2014-2024 SAP SE or an SAP affiliate company and go-hdb contributors
+
+# https://github.com/golang-sql/civil/blob/master/civil.go
+github.com/golang-sql/civil: Copyright 2016 Google LLC
+
+# https://github.com/grafana/loki/blob/main/clients/cmd/fluentd/README.md
+github.com/grafana/loki: Copyright 2018- Grafana Labs
+
 # For Apache projects, copyright is typically in individual source files and
 # not scanned automatically.
 github.com/tklauser/numcpus: Copyright 2018 Tobias Klauser
@@ -177,6 +186,7 @@ github.com/dgraph-io/badger:
  - "Copyright 2021 Dgraph Labs, Inc. and Contributors"
 github.com/mostynb/go-grpc-compression: "Copyright 2023 Mostyn Bramley-Moore."
 github.com/stormcat24/protodep: "Copyright Akinori Yamada"
+go4.org/mem: "Copyright 2020 The Go4 AUTHORS"
 
 # These have large AUTHORS/CONTRIBUTORS files but attribute copyright as a group
 github.com/aptly-dev/aptly: Copyright 2013-2015 aptly authors. All rights reserved.
@@ -323,6 +333,8 @@ github.com/aquasecurity/trivy-policies/pkg/spec: Copyright (c) 2024 Aqua Securit
 github.com/aquasecurity/trivy-policies/rules/specs: Copyright (c) 2024 Aqua Security
 github.com/pjbgf/sha1cd: Copyright 2023 pjbgf
 github.com/go-errors/errors: Copyright (c) 2015 Conrad Irwin <conrad@bugsnag.com>
+github.com/apache/arrow/*: Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
+github.com/tailscale/tailscale: Copyright (c) 2020 Tailscale Inc & AUTHORS.
 
 # Copyright information is in the source headers
 github.com/csaf-poc/csaf_distribution: Copyright 2021-2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
@@ -342,4 +354,3 @@ github.com/go-jose/go-jose/v4: Copyright 2014 Square Inc.
 
 github.com/vibrantbyte/go-antpath/antpath: vibrantbyte <codingrun@163.com>|suchao <suchao@renrenche.com>
 github.com/vibrantbyte/go-antpath/extend: vibrantbyte <codingrun@163.com>|suchao <suchao@renrenche.com>
-

.github/CODEOWNERS

@@ -315,6 +315,7 @@
 /comp/core/autodiscovery/providers/remote_config*.go  @DataDog/remote-config
 /pkg/cloudfoundry                       @Datadog/platform-integrations
 /pkg/clusteragent/                      @DataDog/container-platform
+/pkg/clusteragent/admission/mutate/autoinstrumentation/ @DataDog/container-platform @DataDog/apm-onboarding
 /pkg/clusteragent/orchestrator/         @DataDog/container-app
 /pkg/clusteragent/telemetry/            @DataDog/telemetry-and-analytics
 /pkg/collector/                         @DataDog/agent-metrics-logs
@@ -363,6 +364,7 @@
 /pkg/containerlifecycle/                @Datadog/container-integrations
 /pkg/diagnose/                          @Datadog/container-platform
 /pkg/diagnose/connectivity/             @DataDog/agent-shared-components
+/pkg/diagnose/ports/                    @DataDog/agent-shared-components
 /pkg/eventmonitor/                      @DataDog/ebpf-platform @DataDog/agent-security
 /pkg/dynamicinstrumentation/            @DataDog/debugger
 /pkg/epforwarder/                       @DataDog/agent-processing-and-routing
@@ -526,6 +528,7 @@
 /tasks/updater.py                       @DataDog/fleet
 /tasks/libs/common/omnibus.py           @DataDog/agent-build-and-releases
 /tasks/omnibus.py                       @DataDog/agent-build-and-releases
+/tasks/unit-tests/omnibus_tests.py      @DataDog/agent-build-and-releases
 /tasks/installer.py                     @DataDog/fleet
 /test/                                  @DataDog/agent-developer-tools
 /test/benchmarks/                       @DataDog/agent-metrics-logs

.github/workflows/cws-btfhub-sync.yml

@@ -17,13 +17,37 @@ on:
     - cron: '30 4 * * 5' # at 4:30 UTC on Friday
 
 jobs:
-  sync:
+  generate:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        cone:
+          - amzn
+          - centos
+          - debian
+          - fedora
+          - ol/7/arm64
+          - ol/7/x86_64
+          - ol/8/arm64
+          - ol/8/x86_64
+          - opensuse-leap
+          - rhel
+          - sles
+          - ubuntu/16.04/x86_64
+          - ubuntu/18.04/arm64
+          - ubuntu/18.04/x86_64
+          - ubuntu/20.04/arm64
+          - ubuntu/20.04/x86_64
     steps:
       - name: Cleanup runner
         run: |
           sudo rm -rf /usr/local/lib/android >/dev/null 2>&1
           docker rmi $(docker image ls -aq) >/dev/null 2>&1
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          df -h
 
       - name: Checkout datadog-agent repository
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
@@ -35,6 +59,43 @@ jobs:
         with:
           repository: DataDog/btfhub-archive
           path: dev/dist/archive
+          sparse-checkout: ${{ matrix.cone }}
+
+      - name: Install python
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: 3.11
+          cache: 'pip'
+      - run: pip install -r requirements.txt
+
+      - name: Install go
+        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        with:
+          go-version-file: '.go-version'
+
+      - name: Compute name
+        id: artifact-name
+        run: |
+          echo "ARTIFACT_NAME=constants-${{ matrix.cone }}" | tr '/' '-' >> $GITHUB_OUTPUT
+
+      - name: Sync constants
+        run: |
+          inv -e security-agent.generate-btfhub-constants --archive-path=./dev/dist/archive --output-path=./${{ steps.artifact-name.outputs.ARTIFACT_NAME }}.json ${{ inputs.force_refresh && '--force-refresh' || '' }}
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        with:
+          name: ${{ steps.artifact-name.outputs.ARTIFACT_NAME }}
+          path: ./${{ steps.artifact-name.outputs.ARTIFACT_NAME }}.json
+
+  combine:
+    needs: generate
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout datadog-agent repository
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+        with:
+          ref: ${{ inputs.base_branch || 'main' }}
 
       - name: Install python
         uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
@@ -48,19 +109,22 @@ jobs:
         with:
           go-version-file: '.go-version'
 
-      - name: Install go deps
+      - name: Download All Artifacts
+        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
+        with:
+          path: ./dev/dist/constants
+          pattern: constants-*
+          merge-multiple: true
+
+      - name: Combine constants
         run: |
-          inv -e deps
+          inv -e security-agent.combine-btfhub-constants --archive-path=./dev/dist/constants
 
       - name: Compute branch name
         id: branch-name
         run: |
           echo "BRANCH_NAME=cws/constants-sync-$(date +%s)" >> $GITHUB_OUTPUT
 
-      - name: Sync constants
-        run: |
-          inv -e security-agent.generate-btfhub-constants --archive-path=./dev/dist/archive ${{ inputs.force_refresh && '--force-refresh' || '' }}
-
       - uses: stefanzweifel/git-auto-commit-action@8621497c8c39c72f3e2a999a26b4ca1b5058a842 # v5.0.1
         id: commit-creator
         with:

.github/workflows/go_mod_tidy.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Install python
         uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
         with:
-          python-version: "3.9.12"
+          python-version: 3.11
           cache: "pip"
       - name: Install python requirements.txt
         run: python3 -m pip install -r requirements.txt

.github/workflows/windows-linters.yml

@@ -23,7 +23,7 @@ jobs:
       - name: Install python
         uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
         with:
-          python-version: "3.9.5"
+          python-version: 3.11
           cache: "pip"
       - run: |
           python -m pip install -r requirements.txt

.github/workflows/windows-unittests.yml

@@ -23,7 +23,7 @@ jobs:
       - name: Install python
         uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
         with:
-          python-version: "3.9.5"
+          python-version: 3.11
           cache: "pip"
       - run: |
           python -m pip install -r requirements.txt

.gitignore

@@ -9,9 +9,7 @@ bin/
 /site/
 __pycache__
 .pytest_cache
-venv/
-venv27/
-venv3[78]/
+venv*/
 
 **/*.tmp
 **/debug.test
@@ -33,7 +31,8 @@ Dockerfiles/cluster-agent/nosys-seccomp
 # local development environment control files
 .env
 .envrc
-.python-version
+# I explicitely include this file because it could be ignored globally in custom configurations
+!.python-version
 
 # go-generated files
 datadog.yaml

.gitlab-ci.yml

@@ -164,15 +164,15 @@ variables:
   # To use images from datadog-agent-buildimages dev branches, set the corresponding
   # SUFFIX variable to _test_only
   DATADOG_AGENT_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_BUILDIMAGES: v32778237-8467c81e
+  DATADOG_AGENT_BUILDIMAGES: v34124919-31caaff7
   DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_WINBUILDIMAGES: v32778237-8467c81e
+  DATADOG_AGENT_WINBUILDIMAGES: v34124919-31caaff7
   DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_ARMBUILDIMAGES: v32778237-8467c81e
+  DATADOG_AGENT_ARMBUILDIMAGES: v34124919-31caaff7
   DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v32778237-8467c81e
+  DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v34124919-31caaff7
   DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v32778237-8467c81e
+  DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v34124919-31caaff7
 
   DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded
   DEB_GPG_KEY_ID: ad9589b7
@@ -204,6 +204,7 @@ variables:
   ARTIFACTORY_BYPASS_SSM_NAME: ci.datadog-agent.artifactory_bypass  # agent-ci-experience
   CHANGELOG_COMMIT_SHA_SSM_NAME: ci.datadog-agent.gitlab_changelog_commit_sha  # agent-ci-experience
   CHOCOLATEY_API_KEY_SSM_NAME: ci.datadog-agent.chocolatey_api_key  # windows-agent
+  CODECOV_TOKEN_SSM_NAME: ci.datadog-agent.codecov_token  # agent-ci-experience
   DEB_GPG_KEY_SSM_NAME: ci.datadog-agent.deb_signing_private_key_v2_${DEB_GPG_KEY_ID}  # agent-build-and-releases
   DEB_SIGNING_PASSPHRASE_SSM_NAME: ci.datadog-agent.deb_signing_key_passphrase_v2_${DEB_GPG_KEY_ID}  # agent-build-and-releases
   DOCKER_REGISTRY_LOGIN_SSM_KEY: ci.datadog-agent.docker_hub_login  # agent-ci-experience
@@ -398,6 +399,9 @@ workflow:
       variables:
         USE_CACHING_PROXY_PYTHON: "false"
         USE_CACHING_PROXY_RUBY: "false"
+    - if: $DDR_WORKFLOW_ID != null
+      variables:
+        DEPLOY_AGENT: "true"
 
 #
 # List of rule blocks used in the pipeline
@@ -921,8 +925,11 @@ workflow:
   - !reference [.on_e2e_main_release_or_rc]
   - changes:
       paths:
-        # TODO: Add paths that should trigger tests for process
         - test/new-e2e/tests/process/**/*
+        - cmd/process-agent/**/*
+        - comp/process/**/*
+        - pkg/process/**/*
+        - pkg/config/setup/process.go
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
   - when: manual
     allow_failure: true

.gitlab/binary_build/otel_agent.yml

@@ -9,6 +9,9 @@
     - !reference [.retrieve_linux_go_deps]
     - inv -e otel-agent.build
   needs: ["go_deps"]
+  variables:
+    KUBERNETES_MEMORY_REQUEST: "8Gi"
+    KUBERNETES_MEMORY_LIMIT: "8Gi"
   artifacts:
     expire_in: 1 day
     paths:

.gitlab/deploy_packages/oci.yml

@@ -8,8 +8,8 @@ include:
   tags: ["arch:amd64"]
   stage: deploy_packages
   rules:
-    - !reference [.on_installer_or_e2e_changes_or_manual]
     - !reference [.on_deploy]
+    - !reference [.on_installer_or_e2e_changes_or_manual]
   before_script:
     - ls $OMNIBUS_PACKAGE_DIR
   script:

.gitlab/dev_container_deploy/docker_linux.yml

@@ -250,6 +250,7 @@ qa_agent:
   rules:
     - !reference [.on_container_or_e2e_changes_or_manual]
     - !reference [.on_apm_or_e2e_changes_or_manual]
+    - !reference [.on_npm_or_e2e_changes_or_manual]
   needs:
     - docker_build_agent7
     - docker_build_agent7_arm64

.gitlab/e2e/e2e.yml

@@ -215,17 +215,27 @@ new-e2e-language-detection:
     TARGETS: ./tests/language-detection
     TEAM: processes
 
-new-e2e-npm:
+new-e2e-npm-packages:
   extends: .new_e2e_template
   rules: !reference [.on_npm_or_e2e_changes_or_manual]
   needs:
-    - qa_agent
     - deploy_deb_testing-a7_x64
     - deploy_rpm_testing-a7_x64
     - deploy_windows_testing-a7
   variables:
     TARGETS: ./tests/npm
     TEAM: network-performance-monitoring
+    EXTRA_PARAMS: --run "TestEC2(VM|VMSELinux|VMWKit)Suite"
+
+new-e2e-npm-docker:
+  extends: .new_e2e_template
+  rules: !reference [.on_npm_or_e2e_changes_or_manual]
+  needs:
+    - qa_agent
+  variables:
+    TARGETS: ./tests/npm
+    TEAM: network-performance-monitoring
+    EXTRA_PARAMS: --run "Test(ECSVM|EC2VMContainerized)Suite"
 
 new-e2e-aml:
   extends: .new_e2e_template_needs_deb_windows_x64
@@ -241,6 +251,7 @@ new-e2e-cws:
     - deploy_deb_testing-a7_x64
     - qa_cws_instrumentation
     - qa_agent
+    - qa_dca
   variables:
     TARGETS: ./tests/cws
     TEAM: csm-threats-agent
@@ -249,6 +260,7 @@ new-e2e-cws:
     matrix:
       - EXTRA_PARAMS: --run TestAgentSuite
       - EXTRA_PARAMS: --run TestECSFargate
+      - EXTRA_PARAMS: --run TestKindSuite
   # Temporary, remove once we made sure the recent changes have no impact on the stability of these tests
   allow_failure: true
 
@@ -295,6 +307,7 @@ new-e2e-installer:
     - deploy_suse_rpm_testing-i7_arm64
     - deploy_suse_rpm_testing-i7_x64
     - qa_installer_oci
+    - qa_agent_oci
   variables:
     TARGETS: ./tests/installer
     TEAM: fleet

.gitlab/functional_test/common.yml

@@ -7,7 +7,7 @@
 
 .kitchen_test_system_probe:
   extends:
-    - .kitchen_common
+    - .kitchen_common_with_junit
     - .kitchen_datadog_agent_flavor
   rules:
     !reference [.on_system_probe_or_e2e_changes_or_manual]