Merge remote-tracking branch 'origin/main' into arbll/db

.gitlab-ci.yml

@@ -921,8 +921,11 @@ workflow:
   - !reference [.on_e2e_main_release_or_rc]
   - changes:
       paths:
-        # TODO: Add paths that should trigger tests for process
         - test/new-e2e/tests/process/**/*
+        - cmd/process-agent/**/*
+        - comp/process/**/*
+        - pkg/process/**/*
+        - pkg/config/setup/process.go
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
   - when: manual
     allow_failure: true

.gitlab/kernel_matrix_testing/security_agent.yml

@@ -127,6 +127,7 @@ kmt_run_secagent_tests_x64:
           - "ubuntu_20.04"
           - "ubuntu_22.04"
           - "ubuntu_23.10"
+          - "ubuntu_24.04"
           - "amazon_5.4"
           - "amazon_5.10"
           - "fedora_37"
@@ -164,6 +165,7 @@ kmt_run_secagent_tests_arm64:
       - TAG:
           - "ubuntu_22.04"
           - "ubuntu_23.10"
+          - "ubuntu_24.04"
           - "amazon_5.4"
           - "amazon_5.10"
           - "fedora_37"

cmd/installer/subcommands/installer/command.go

@@ -128,8 +128,11 @@ func newBootstraperCmd(operation string) *bootstraperCmd {
 func newTelemetry() *telemetry.Telemetry {
 	apiKey := os.Getenv(envAPIKey)
 	site := os.Getenv(envSite)
-	if apiKey == "" || site == "" {
-		fmt.Printf("telemetry disabled: missing DD_API_KEY or DD_SITE\n")
+	if site == "" {
+		site = "datadoghq.com"
+	}
+	if apiKey == "" {
+		fmt.Printf("telemetry disabled: missing DD_API_KEY\n")
 		return nil
 	}
 	t, err := telemetry.NewTelemetry(apiKey, site, "datadog-installer")

comp/core/autodiscovery/README.md

@@ -59,17 +59,16 @@ Each service has two entity identifiers: the AD service ID (from `svc.GetService
 These both uniquely identify an entity, but using different syntax.
 
 <!-- NOTE: a similar table appears in comp/core/tagger/README.md; please keep both in sync -->
-| *Service*                         | *Service ID*                                                      | *Tagger Entity*                                                    |
-|-----------------------------------|-------------------------------------------------------------------|--------------------------------------------------------------------|
-| workloadmeta.KindContainer        | `<runtime>://<sha>`                                               | `container_id://<sha>`                                             |
-| workloadmeta.KindGardenContainer  | `garden_container://<sha>`                                        | `container_id://<sha>`                                             |
-| workloadmeta.KindKubernetesPod    | `kubernetes_pod://<uid>`                                          | `kubernetes_pod_uid://<uid>`                                       |
-| workloadmeta.KindECSTask          | `ecs_task://<task-id>`                                            | `ecs_task://<task-id>`                                             |
-| CloudFoundry LRP                  | `<processGuid>/<svcName>/<instanceGuid>` or `<appGuid>/<svcName>` | `<processGuid>/<svcName>/<instanceGuid>`  or `<appGuid>/<svcName>` |
-| Container runtime or orchestrator | `_<name>` e.g., `_containerd`                                     | (none)                                                             |
-| Kubernetes Endpoint               | `kube_endpoint_uid://<namespace>/<name>/<ip>`                     | `kube_endpoint_uid://<namespace>/<name>/<ip>`                      |
-| Kubernetes Service                | `kube_service://<namespace>/<name>`                               | `kube_service://<namespace>/<name>`                                |
-| SNMP Config                       | config hash                                                       | config hash                                                        |
+| *Service*                         | *Service ID*                                                      | *Tagger Entity*              |
+|-----------------------------------|-------------------------------------------------------------------|------------------------------|
+| workloadmeta.KindContainer        | `<runtime>://<sha>`                                               | `container_id://<sha>`       |
+| workloadmeta.KindKubernetesPod    | `kubernetes_pod://<uid>`                                          | `kubernetes_pod_uid://<uid>` |
+| workloadmeta.KindECSTask          | `ecs_task://<task-id>`                                            | `ecs_task://<task-id>`       |
+| CloudFoundry LRP                  | `<processGuid>/<svcName>/<instanceGuid>` or `<appGuid>/<svcName>` | (none)                       |
+| Container runtime or orchestrator | `_<name>` e.g., `_containerd`                                     | (none)                       |
+| Kubernetes Endpoint               | `kube_endpoint_uid://<namespace>/<name>/<ip>`                     | (none)                       |
+| Kubernetes Service                | `kube_service://<namespace>/<name>`                               | (none)                       |
+| SNMP Config                       | config hash                                                       | (none)                       |
 
 ## MetaScheduler
 

comp/core/tagger/README.md

@@ -1,50 +1,36 @@
 # package `tagger`
 
 The **Tagger** is the central source of truth for client-side entity tagging. It
-runs **Collector**s that detect entities and collect their tags. Tags are then
-stored in memory (by the **TagStore**) and can be queried by the tagger.Tag()
-method. Calling once tagger.Init() after the **config** package is ready is
-needed to enable collection.
+subscribes to workloadmeta to get updates for all the entity kinds (containers,
+kubernetes pods, kubernetes nodes, etc.) and extracts the tags for each of them.
+Tags are then stored in memory (by the **TagStore**) and can be queried by the
+`tagger.Tag()` method. Calling once `tagger.Start()` after the **config**
+package is ready is needed to enable collection.
 
 The package methods use a common **defaultTagger** object, but we can create
 a custom **Tagger** object for testing.
 
-The package will implement an IPC mechanism (a server and a client) to allow
-other agents to query the **DefaultTagger** and avoid duplicating the information
-in their process. Switch between local and client mode will be done via a build flag.
+The package implements an IPC mechanism (a server and a client) to allow other
+agents to query the **DefaultTagger** and avoid duplicating the information in
+their process. Check the `remote` package for more details.
 
 The tagger is also available to python checks via the `tagger` module exporting
 the `get_tags()` function. This function accepts the same arguments as the Go `Tag()`
 function, and returns an empty list on errors.
 
-## Collector
+## Workloadmeta
 
-A **Collector** connects to a single information source and pushes **types.TagInfo**
-structs to a channel, towards the **Tagger**. It can either run in streaming
-mode, pull or fetchonly mode, depending of what's most efficient for the data source:
-
-### Streamer
-
-The **DockerCollector** runs in stream mode as it collects events from the docker
-daemon and reacts to them, sending updates incrementally.
-
-### Puller
-
-The **KubernetesCollector** will run in pull mode as it needs to query and filter a full entity list every time. It will only push
-updates to the store though, by keeping an internal state of the latest
-revision.
-
-### FetchOnly
-
-The **ECSCollector** does not push updates to the Store by itself, but is only triggered on cache misses. As tasks don't change after creation, there's no need for periodic pulling. It is designed to run alongside DockerCollector, that will trigger deletions in the store.
+The entities that need to be tagged are collected by workloadmeta. The tagger
+subscribes to workloadmeta to get updates for all the entity kinds (containers,
+kubernetes pods, kubernetes nodes, etc.) and extracts the tags for each of them.
 
 ## TagStore
 
-The **TagStore** reads **types.TagInfo** structs and stores them in a in-memory
+The **TagStore** reads **types.TagInfo** structs and stores them in an in-memory
 cache. Cache invalidation is triggered by the collectors (or source) by either:
 
 * sending new tags for the same `Entity`, all the tags from this `Source`
-  will be removed and replaced by the new tags
+  will be removed and replaced by the new tags.
 * sending a **types.TagInfo** with **DeleteEntity** set, all the tags collected for
   this entity by the specified source (but not others) will be deleted when
   **prune()** is called.
@@ -62,31 +48,29 @@ cache. Cache invalidation is triggered by the collectors (or source) by either:
 Tagger entities are identified by a string-typed ID, with one of the following forms:
 
 <!-- NOTE: a similar table appears in comp/core/autodiscovery/README.md; please keep both in sync -->
-| *Service*                               | *Tagger Entity*                                                    |
-|-----------------------------------------|--------------------------------------------------------------------|
-| workloadmeta.KindContainer              | `container_id://<sha>`                                             |
-| workloadmeta.KindContainerImageMetadata | `container_image_metadata://<sha>`                                 |
-| workloadmeta.KindGardenContainer        | `container_id://<sha>`                                             |
-| workloadmeta.KindKubernetesPod          | `kubernetes_pod_uid://<uid>`                                       |
-| workloadmeta.KindECSTask                | `ecs_task://<task-id>`                                             |
-| CloudFoundry LRP                        | `<processGuid>/<svcName>/<instanceGuid>`  or `<appGuid>/<svcName>` |
-| Container runtime or orchestrator       | (none)                                                             |
-| Kubernetes Endpoint                     | `kube_endpoint_uid://<namespace>/<name>/<ip>`                      |
-| Kubernetes Service                      | `kube_service://<namespace>/<name>`                                |
-| SNMP Config                             | config hash                                                        |
+| *Entity*                                | *ID*                               |
+|-----------------------------------------|------------------------------------|
+| workloadmeta.KindContainer              | `container_id://<sha>`             |
+| workloadmeta.KindContainerImageMetadata | `container_image_metadata://<sha>` |
+| workloadmeta.KindECSTask                | `ecs_task://<task-id>`             |
+| workloadmeta.KindHost                   | `host`                             |
+| workloadmeta.KindKubernetesDeployment   | `deployment://<namespace>/<name>`  |
+| workloadmeta.KindKubernetesNamespace    | `namespace://<name>`               |
+| workloadmeta.KindKubernetesNode         | `kubernetes_node_uid://<name>`     |
+| workloadmeta.KindKubernetesPod          | `kubernetes_pod_uid://<uid>`       |
+| workloadmeta.KindProcess                | `process://<pid>`                  |
 
 ## Tagger
 
-The Tagger handles the glue between **Collectors** and **TagStore** and the
-cache miss logic. If the tags from the **TagStore** are missing some sources,
-they will be manually queried in a block way, and the cache will be updated.
+The Tagger handles the glue between the workloadmeta collector and the
+**TagStore**.
 
 For convenience, the package creates a **defaultTagger** object that is used
 when calling the `tagger.Tag()` method.
 
-                   +-----------+
-                   | Collector |
-                   +---+-------+
+                   +--------------+
+                   | Workloadmeta |
+                   +---+----------+
                        |
                        |
     +--------+      +--+-------+       +-------------+

comp/core/tagger/component.go

@@ -4,9 +4,11 @@
 // Copyright 2016-present Datadog, Inc.
 
 // Package tagger implements the Tagger component. The Tagger is the central
-// source of truth for client-side entity tagging. It runs Collectors that
-// detect entities and collect their tags. Tags are then stored in memory (by
-// the TagStore) and can be queried by the tagger.Tag() method.
+// source of truth for client-side entity tagging. It subscribes to workloadmeta
+// to get updates for all the entity kinds (containers, kubernetes pods,
+// kubernetes nodes, etc.) and extracts the tags for each of them. Tags are then
+// stored in memory (by the TagStore) and can be queried by the tagger.Tag()
+// method.
 
 // Package tagger provides the tagger component for the Datadog Agent
 package tagger

comp/core/tagger/taggerimpl/collectors/doc.go

@@ -3,5 +3,6 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2016-present Datadog, Inc.
 
-// Package collectors implements the collectors for the Tagger component.
+// Package collectors implements a collector for the Tagger component that
+// subscribes to workloadmeta
 package collectors

comp/core/tagger/taggerimpl/local/tagger.go

@@ -45,9 +45,7 @@ func NewTagger(workloadStore workloadmeta.Component) *Tagger {
 	}
 }
 
-// Start goes through a catalog and tries to detect which are relevant
-// for this host. It then starts the collection logic and is ready for
-// requests.
+// Start starts the workloadmeta collector and then it is ready for requests.
 func (t *Tagger) Start(ctx context.Context) error {
 	t.ctx, t.cancel = context.WithCancel(ctx)
 

docs/cloud-workload-security/agent_expressions.md

@@ -1,5 +1,5 @@
 ---
-title: Creating Custom Agent Rules
+title: Creating Agent Rule Expressions
 kind: documentation
 description: "Agent expression attributes and operators for CSM Threats Rules"
 disable_edit: true

docs/cloud-workload-security/scripts/templates/agent_expressions.md

@@ -1,5 +1,5 @@
 ---
-title: Creating Custom Agent Rules
+title: Creating Agent Rule Expressions
 kind: documentation
 description: "Agent expression attributes and operators for CSM Threats Rules"
 disable_edit: true