Add OTel integration test (#25519)

.gitlab/integration_test/include.yml

@@ -4,4 +4,5 @@
 
 include:
   - .gitlab/integration_test/dogstatsd.yml
+  - .gitlab/integration_test/otel.yml
   - .gitlab/integration_test/windows.yml

.gitlab/integration_test/otel.yml

@@ -0,0 +1,23 @@
+---
+# integration_test stage
+# Contains jobs to run integration tests in otel go binaries
+
+integration_tests_otel:
+  stage: integration_test
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  tags: ["arch:amd64"]
+  script:
+    - inv check-otel-build
+    - inv check-otel-module-versions
+  rules: # this should only run on dev branches
+    - if: $CI_PIPELINE_SOURCE =~ /^schedule.*$/
+      when: never
+    - if: $CI_COMMIT_TAG
+      when: never
+    - !reference [.except_main_or_release_branch]
+    - if: $CI_COMMIT_MESSAGE =~ /.*\[skip cancel\].*/
+      when: never
+    - if: $CI_COMMIT_REF_NAME =~ /.*-skip-cancel$/
+      when: never
+    - when: always
+      allow_failure: true

tasks/__init__.py

@@ -71,6 +71,8 @@
     tidy_all,
 )
 from tasks.gotest import (
+    check_otel_build,
+    check_otel_module_versions,
     codecov,
     e2e_tests,
     get_impacted_packages,
@@ -123,6 +125,8 @@
 ns.add_task(invoke_unit_tests)
 ns.add_task(check_mod_tidy)
 ns.add_task(check_go_mod_replaces)
+ns.add_task(check_otel_build)
+ns.add_task(check_otel_module_versions)
 ns.add_task(tidy)
 ns.add_task(tidy_all)
 ns.add_task(internal_deps_checker)

tasks/gotest.py

@@ -11,6 +11,7 @@
 import os
 import platform
 import re
+import shutil
 import sys
 from collections import defaultdict
 from datetime import datetime
@@ -35,13 +36,15 @@
 from tasks.test_core import ModuleTestResult, process_input_args, process_module_results, test_core
 from tasks.testwasher import TestWasher
 from tasks.trace_agent import integration_tests as trace_integration_tests
+from tasks.update_go import PATTERN_MAJOR_MINOR_BUGFIX
 
 PROFILE_COV = "coverage.out"
 TMP_PROFILE_COV_PREFIX = "coverage.out.rerun"
 GO_COV_TEST_PATH = "test_with_coverage"
 GO_TEST_RESULT_TMP_JSON = 'module_test_output.json'
 WINDOWS_MAX_PACKAGES_NUMBER = 150
 TRIGGER_ALL_TESTS_PATHS = ["tasks/gotest.py", "tasks/build_tags.py", ".gitlab/source_test/*"]
+OTEL_UPSTREAM_GO_VERSION = "1.21.0"
 
 
 class TestProfiler:
@@ -943,3 +946,38 @@ def lint_go(
     only_modified_packages=False,
 ):
     raise Exit("This task is deprecated, please use `inv linter.go`", 1)
+
+
+@task
+def check_otel_build(ctx):
+    with ctx.cd("test/otel"):
+        # Rename fixtures
+        shutil.copy("test/otel/dependencies.go.fake", "test/otel/dependencies.go")
+        shutil.copy("test/otel/go.mod.fake", "test/otel/go.mod")
+
+        # Update dependencies to latest local version
+        res = ctx.run("go mod tidy")
+        if not res.ok:
+            raise Exit(f"Error running `go mod tidy`: {res.stderr}")
+
+        # Build test/otel/dependencies.go with same settings as `make otelcontribcol`
+        res = ctx.run("GO111MODULE=on CGO_ENABLED=0 go build -trimpath -o . .", warn=True)
+        if res is None or not res.ok:
+            raise Exit(f"Error building otel components with datadog-agent dependencies: {res.stderr}")
+
+
+@task
+def check_otel_module_versions(ctx):
+    for path, module in DEFAULT_MODULES.items():
+        if module.used_by_otel:
+            mod_file = f"./{path}/go.mod"
+            pattern = f"^go {PATTERN_MAJOR_MINOR_BUGFIX}\r?$"
+            with open(mod_file, newline='', encoding='utf-8') as reader:
+                content = reader.read()
+                matches = re.findall(pattern, content, flags=re.MULTILINE)
+                if len(matches) != 1:
+                    raise Exit(f"{mod_file} does not match expected go directive format")
+                if matches[0] != f"go {OTEL_UPSTREAM_GO_VERSION}":
+                    raise Exit(
+                        f"{mod_file} version {matches[0]} does not match upstream version: {OTEL_UPSTREAM_GO_VERSION}"
+                    )

tasks/modules.py

@@ -132,128 +132,130 @@ def dependency_path(self, agent_version):
         targets=["./pkg", "./cmd", "./comp"],
     ),
     "cmd/agent/common/path": GoModule("cmd/agent/common/path", independent=True),
-    "comp/core/config": GoModule("comp/core/config", independent=True),
+    "comp/core/config": GoModule("comp/core/config", independent=True, used_by_otel=True),
     "comp/core/flare/builder": GoModule("comp/core/flare/builder", independent=True),
     "comp/core/flare/types": GoModule("comp/core/flare/types", independent=True),
-    "comp/core/hostname/hostnameinterface": GoModule("comp/core/hostname/hostnameinterface", independent=True),
-    "comp/core/log": GoModule("comp/core/log", independent=True),
-    "comp/core/secrets": GoModule("comp/core/secrets", independent=True),
-    "comp/core/status": GoModule("comp/core/status", independent=True),
+    "comp/core/hostname/hostnameinterface": GoModule(
+        "comp/core/hostname/hostnameinterface", independent=True, used_by_otel=True
+    ),
+    "comp/core/log": GoModule("comp/core/log", independent=True, used_by_otel=True),
+    "comp/core/secrets": GoModule("comp/core/secrets", independent=True, used_by_otel=True),
+    "comp/core/status": GoModule("comp/core/status", independent=True, used_by_otel=True),
     "comp/core/status/statusimpl": GoModule("comp/core/status/statusimpl", independent=True),
-    "comp/core/telemetry": GoModule("comp/core/telemetry", independent=True),
-    "comp/def": GoModule("comp/def", independent=True),
-    "comp/forwarder/defaultforwarder": GoModule("comp/forwarder/defaultforwarder", independent=True),
+    "comp/core/telemetry": GoModule("comp/core/telemetry", independent=True, used_by_otel=True),
+    "comp/def": GoModule("comp/def", independent=True, used_by_otel=True),
+    "comp/forwarder/defaultforwarder": GoModule("comp/forwarder/defaultforwarder", independent=True, used_by_otel=True),
     "comp/forwarder/orchestrator/orchestratorinterface": GoModule(
-        "comp/forwarder/orchestrator/orchestratorinterface", independent=True
+        "comp/forwarder/orchestrator/orchestratorinterface", independent=True, used_by_otel=True
     ),
-    "comp/logs/agent/config": GoModule("comp/logs/agent/config", independent=True),
+    "comp/logs/agent/config": GoModule("comp/logs/agent/config", independent=True, used_by_otel=True),
     "comp/netflow/payload": GoModule("comp/netflow/payload", independent=True),
     "comp/otelcol/collector-contrib/def": GoModule(
         "comp/otelcol/collector-contrib/def", independent=True, used_by_otel=True
     ),
     "comp/otelcol/collector-contrib/impl": GoModule(
         "comp/otelcol/collector-contrib/impl", independent=True, used_by_otel=True
     ),
-    "comp/otelcol/logsagentpipeline": GoModule("comp/otelcol/logsagentpipeline", independent=True),
+    "comp/otelcol/logsagentpipeline": GoModule("comp/otelcol/logsagentpipeline", independent=True, used_by_otel=True),
     "comp/otelcol/logsagentpipeline/logsagentpipelineimpl": GoModule(
-        "comp/otelcol/logsagentpipeline/logsagentpipelineimpl", independent=True
+        "comp/otelcol/logsagentpipeline/logsagentpipelineimpl", independent=True, used_by_otel=True
     ),
     "comp/otelcol/otlp/components/exporter/datadogexporter": GoModule(
         "comp/otelcol/otlp/components/exporter/datadogexporter", independent=True
     ),
     "comp/otelcol/otlp/components/exporter/logsagentexporter": GoModule(
-        "comp/otelcol/otlp/components/exporter/logsagentexporter", independent=True
+        "comp/otelcol/otlp/components/exporter/logsagentexporter", independent=True, used_by_otel=True
     ),
     "comp/otelcol/otlp/components/exporter/serializerexporter": GoModule(
-        "comp/otelcol/otlp/components/exporter/serializerexporter", independent=True
+        "comp/otelcol/otlp/components/exporter/serializerexporter", independent=True, used_by_otel=True
     ),
     "comp/otelcol/otlp/components/metricsclient": GoModule(
-        "comp/otelcol/otlp/components/metricsclient", independent=True
+        "comp/otelcol/otlp/components/metricsclient", independent=True, used_by_otel=True
     ),
     "comp/otelcol/otlp/components/statsprocessor": GoModule(
-        "comp/otelcol/otlp/components/statsprocessor", independent=True
+        "comp/otelcol/otlp/components/statsprocessor", independent=True, used_by_otel=True
     ),
     "comp/otelcol/otlp/testutil": GoModule("comp/otelcol/otlp/testutil", independent=True),
     "comp/otelcol/converter/def": GoModule("comp/otelcol/converter/def", independent=True),
     "comp/otelcol/converter/impl": GoModule("comp/otelcol/converter/impl", independent=True),
-    "comp/serializer/compression": GoModule("comp/serializer/compression", independent=True),
+    "comp/serializer/compression": GoModule("comp/serializer/compression", independent=True, used_by_otel=True),
     "internal/tools": GoModule("internal/tools", condition=lambda: False, should_tag=False),
     "internal/tools/independent-lint": GoModule(
         "internal/tools/independent-lint", condition=lambda: False, should_tag=False
     ),
     "internal/tools/modformatter": GoModule("internal/tools/modformatter", condition=lambda: False, should_tag=False),
     "internal/tools/modparser": GoModule("internal/tools/modparser", condition=lambda: False, should_tag=False),
     "internal/tools/proto": GoModule("internal/tools/proto", condition=lambda: False, should_tag=False),
-    "pkg/aggregator/ckey": GoModule("pkg/aggregator/ckey", independent=True),
+    "pkg/aggregator/ckey": GoModule("pkg/aggregator/ckey", independent=True, used_by_otel=True),
     "pkg/api": GoModule("pkg/api", independent=True),
-    "pkg/collector/check/defaults": GoModule("pkg/collector/check/defaults", independent=True),
-    "pkg/config/env": GoModule("pkg/config/env", independent=True),
+    "pkg/collector/check/defaults": GoModule("pkg/collector/check/defaults", independent=True, used_by_otel=True),
+    "pkg/config/env": GoModule("pkg/config/env", independent=True, used_by_otel=True),
     "pkg/config/logs": GoModule("pkg/config/logs", independent=True),
-    "pkg/config/model": GoModule("pkg/config/model", independent=True),
+    "pkg/config/model": GoModule("pkg/config/model", independent=True, used_by_otel=True),
     "pkg/config/remote": GoModule("pkg/config/remote", independent=True),
-    "pkg/config/setup": GoModule("pkg/config/setup", independent=True),
-    "pkg/config/utils": GoModule("pkg/config/utils", independent=True),
+    "pkg/config/setup": GoModule("pkg/config/setup", independent=True, used_by_otel=True),
+    "pkg/config/utils": GoModule("pkg/config/utils", independent=True, used_by_otel=True),
     "pkg/errors": GoModule("pkg/errors", independent=True),
     "pkg/gohai": GoModule("pkg/gohai", independent=True, importable=False),
-    "pkg/logs/auditor": GoModule("pkg/logs/auditor", independent=True),
-    "pkg/logs/client": GoModule("pkg/logs/client", independent=True),
-    "pkg/logs/diagnostic": GoModule("pkg/logs/diagnostic", independent=True),
-    "pkg/logs/message": GoModule("pkg/logs/message", independent=True),
-    "pkg/logs/metrics": GoModule("pkg/logs/metrics", independent=True),
-    "pkg/logs/pipeline": GoModule("pkg/logs/pipeline", independent=True),
-    "pkg/logs/processor": GoModule("pkg/logs/processor", independent=True),
-    "pkg/logs/sds": GoModule("pkg/logs/sds", independent=True),
-    "pkg/logs/sender": GoModule("pkg/logs/sender", independent=True),
-    "pkg/logs/sources": GoModule("pkg/logs/sources", independent=True),
-    "pkg/logs/status/statusinterface": GoModule("pkg/logs/status/statusinterface", independent=True),
-    "pkg/logs/status/utils": GoModule("pkg/logs/status/utils", independent=True),
+    "pkg/logs/auditor": GoModule("pkg/logs/auditor", independent=True, used_by_otel=True),
+    "pkg/logs/client": GoModule("pkg/logs/client", independent=True, used_by_otel=True),
+    "pkg/logs/diagnostic": GoModule("pkg/logs/diagnostic", independent=True, used_by_otel=True),
+    "pkg/logs/message": GoModule("pkg/logs/message", independent=True, used_by_otel=True),
+    "pkg/logs/metrics": GoModule("pkg/logs/metrics", independent=True, used_by_otel=True),
+    "pkg/logs/pipeline": GoModule("pkg/logs/pipeline", independent=True, used_by_otel=True),
+    "pkg/logs/processor": GoModule("pkg/logs/processor", independent=True, used_by_otel=True),
+    "pkg/logs/sds": GoModule("pkg/logs/sds", independent=True, used_by_otel=True),
+    "pkg/logs/sender": GoModule("pkg/logs/sender", independent=True, used_by_otel=True),
+    "pkg/logs/sources": GoModule("pkg/logs/sources", independent=True, used_by_otel=True),
+    "pkg/logs/status/statusinterface": GoModule("pkg/logs/status/statusinterface", independent=True, used_by_otel=True),
+    "pkg/logs/status/utils": GoModule("pkg/logs/status/utils", independent=True, used_by_otel=True),
     "pkg/logs/util/testutils": GoModule("pkg/logs/util/testutils", independent=True),
-    "pkg/metrics": GoModule("pkg/metrics", independent=True),
+    "pkg/metrics": GoModule("pkg/metrics", independent=True, used_by_otel=True),
     "pkg/networkdevice/profile": GoModule("pkg/networkdevice/profile", independent=True),
     "pkg/obfuscate": GoModule("pkg/obfuscate", independent=True, used_by_otel=True),
-    "pkg/orchestrator/model": GoModule("pkg/orchestrator/model", independent=True),
-    "pkg/process/util/api": GoModule("pkg/process/util/api", independent=True),
+    "pkg/orchestrator/model": GoModule("pkg/orchestrator/model", independent=True, used_by_otel=True),
+    "pkg/process/util/api": GoModule("pkg/process/util/api", independent=True, used_by_otel=True),
     "pkg/proto": GoModule("pkg/proto", independent=True, used_by_otel=True),
     "pkg/remoteconfig/state": GoModule("pkg/remoteconfig/state", independent=True, used_by_otel=True),
     "pkg/security/secl": GoModule("pkg/security/secl", independent=True, legacy_go_mod_version=True),
     "pkg/security/seclwin": GoModule(
         "pkg/security/seclwin", independent=True, condition=lambda: False, legacy_go_mod_version=True
     ),
-    "pkg/serializer": GoModule("pkg/serializer", independent=True),
-    "pkg/status/health": GoModule("pkg/status/health", independent=True),
-    "pkg/tagger/types": GoModule("pkg/tagger/types", independent=True),
-    "pkg/tagset": GoModule("pkg/tagset", independent=True),
-    "pkg/telemetry": GoModule("pkg/telemetry", independent=True),
+    "pkg/serializer": GoModule("pkg/serializer", independent=True, used_by_otel=True),
+    "pkg/status/health": GoModule("pkg/status/health", independent=True, used_by_otel=True),
+    "pkg/tagger/types": GoModule("pkg/tagger/types", independent=True, used_by_otel=True),
+    "pkg/tagset": GoModule("pkg/tagset", independent=True, used_by_otel=True),
+    "pkg/telemetry": GoModule("pkg/telemetry", independent=True, used_by_otel=True),
     "pkg/trace": GoModule("pkg/trace", independent=True, used_by_otel=True),
     "pkg/trace/stats/oteltest": GoModule("pkg/trace/stats/oteltest", independent=True, used_by_otel=True),
-    "pkg/util/backoff": GoModule("pkg/util/backoff", independent=True),
-    "pkg/util/buf": GoModule("pkg/util/buf", independent=True),
+    "pkg/util/backoff": GoModule("pkg/util/backoff", independent=True, used_by_otel=True),
+    "pkg/util/buf": GoModule("pkg/util/buf", independent=True, used_by_otel=True),
     "pkg/util/cache": GoModule("pkg/util/cache", independent=True),
     "pkg/util/cgroups": GoModule(
         "pkg/util/cgroups", independent=True, condition=lambda: sys.platform == "linux", used_by_otel=True
     ),
-    "pkg/util/common": GoModule("pkg/util/common", independent=True),
-    "pkg/util/executable": GoModule("pkg/util/executable", independent=True),
-    "pkg/util/filesystem": GoModule("pkg/util/filesystem", independent=True),
+    "pkg/util/common": GoModule("pkg/util/common", independent=True, used_by_otel=True),
+    "pkg/util/executable": GoModule("pkg/util/executable", independent=True, used_by_otel=True),
+    "pkg/util/filesystem": GoModule("pkg/util/filesystem", independent=True, used_by_otel=True),
     "pkg/util/flavor": GoModule("pkg/util/flavor", independent=True),
-    "pkg/util/fxutil": GoModule("pkg/util/fxutil", independent=True),
+    "pkg/util/fxutil": GoModule("pkg/util/fxutil", independent=True, used_by_otel=True),
     "pkg/util/grpc": GoModule("pkg/util/grpc", independent=True),
-    "pkg/util/hostname/validate": GoModule("pkg/util/hostname/validate", independent=True),
-    "pkg/util/http": GoModule("pkg/util/http", independent=True),
-    "pkg/util/json": GoModule("pkg/util/json", independent=True),
+    "pkg/util/hostname/validate": GoModule("pkg/util/hostname/validate", independent=True, used_by_otel=True),
+    "pkg/util/http": GoModule("pkg/util/http", independent=True, used_by_otel=True),
+    "pkg/util/json": GoModule("pkg/util/json", independent=True, used_by_otel=True),
     "pkg/util/log": GoModule("pkg/util/log", independent=True, used_by_otel=True),
-    "pkg/util/optional": GoModule("pkg/util/optional", independent=True),
+    "pkg/util/optional": GoModule("pkg/util/optional", independent=True, used_by_otel=True),
     "pkg/util/pointer": GoModule("pkg/util/pointer", independent=True, used_by_otel=True),
     "pkg/util/scrubber": GoModule("pkg/util/scrubber", independent=True, used_by_otel=True),
-    "pkg/util/sort": GoModule("pkg/util/sort", independent=True),
-    "pkg/util/startstop": GoModule("pkg/util/startstop", independent=True),
-    "pkg/util/statstracker": GoModule("pkg/util/statstracker", independent=True),
-    "pkg/util/system": GoModule("pkg/util/system", independent=True),
-    "pkg/util/system/socket": GoModule("pkg/util/system/socket", independent=True),
+    "pkg/util/sort": GoModule("pkg/util/sort", independent=True, used_by_otel=True),
+    "pkg/util/startstop": GoModule("pkg/util/startstop", independent=True, used_by_otel=True),
+    "pkg/util/statstracker": GoModule("pkg/util/statstracker", independent=True, used_by_otel=True),
+    "pkg/util/system": GoModule("pkg/util/system", independent=True, used_by_otel=True),
+    "pkg/util/system/socket": GoModule("pkg/util/system/socket", independent=True, used_by_otel=True),
     "pkg/util/testutil": GoModule("pkg/util/testutil", independent=True),
     "pkg/util/uuid": GoModule("pkg/util/uuid", independent=True),
-    "pkg/util/winutil": GoModule("pkg/util/winutil", independent=True),
-    "pkg/version": GoModule("pkg/version", independent=True),
+    "pkg/util/winutil": GoModule("pkg/util/winutil", independent=True, used_by_otel=True),
+    "pkg/version": GoModule("pkg/version", independent=True, used_by_otel=True),
     "test/e2e/containers/otlp_sender": GoModule(
         "test/e2e/containers/otlp_sender", condition=lambda: False, should_tag=False
     ),

test/otel/.gitignore

@@ -0,0 +1,4 @@
+*.go
+go.mod
+go.sum
+otel

test/otel/README.md

@@ -0,0 +1,11 @@
+# OTel Integration Test Files
+
+This folder contains test files used as fixtures for the tests in `.gitlab/integration_test/otel.yml`. This integration test calls the following invoke tasks: `check-otel-build`, `check-otel-module-versions`.
+
+## `check-otel-build`
+This test attempts to build the files in `test/otel` with the same build args used in the make command of opentelemetry-collector-contrib. If an incompatibility is found, such as a dependency that requires CGO, this test will fail.
+
+`dependencies.go` is a skeleton Go file that imports all direct imports that are used in the upstream [datadogexporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/datadogexporter/go.mod) and [datadogconnector](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/connector/datadogconnector/go.mod).
+
+## `check-otel-module-versions`
+This test reads the modules in `modules.py` with the `used_by_otel` flag and verifies that all of their `go.mod` versions match the same version that opentelemetry-collector-contrib uses. If these versions are out of sync, the modules can no longer be imported in upstream and this test will fail.