diff --git a/.github/workflows/run-external-processing.yml b/.github/workflows/run-external-processing.yml new file mode 100644 index 00000000000..1bdc5c23dab --- /dev/null +++ b/.github/workflows/run-external-processing.yml @@ -0,0 +1,81 @@ +name: External-processing tests + +on: + workflow_call: + inputs: + binaries_artifact: + description: "Artifact name containing the binaries to test" + default: '' + required: false + type: string + ci_environment: + description: "Which CI environment is running the tests, used for FPD" + default: 'custom' + required: false + type: string + build_proxy_image: + description: "Shall we build proxy image" + default: false + required: false + type: boolean + +env: + REGISTRY: ghcr.io + + +jobs: + external-processing: + runs-on: + group: "APM Larger Runners" + + env: + SYSTEM_TESTS_REPORT_ENVIRONMENT: ${{ inputs.ci_environment }} + SYSTEM_TESTS_REPORT_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + repository: 'DataDog/system-tests' + - name: Install runner + uses: ./.github/actions/install_runner + + - name: Log in to the Container registry + run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin + + - name: Get binaries artifact + if : ${{ inputs.binaries_artifact != '' }} + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.binaries_artifact }} + path: binaries/ + + - name: Pull images + uses: ./.github/actions/pull_images + with: + library: golang + weblog: golang-dummy + scenarios: '["EXTERNAL_PROCESSING"]' + + - name: Build proxy image + if: inputs.build_proxy_image + run: ./build.sh -i proxy + + - name: Build agent image + run: ./build.sh -i agent + + - name: Run EXTERNAL_PROCESSING scenario + run: ./run.sh EXTERNAL_PROCESSING + env: + DD_API_KEY: ${{ secrets.DD_API_KEY }} + + - name: Compress logs + id: compress_logs + if: always() && steps.build.outcome == 'success' + run: tar -czvf artifact.tar.gz $(ls | grep logs) + - name: Upload artifact + if: always() && steps.compress_logs.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: logs_externalprocessing_golang_golang-dummy_${{ inputs.ci_environment }} + path: artifact.tar.gz diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index ec5f5a0418c..9b9406b6a02 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -143,3 +143,14 @@ jobs: with: library: ${{ inputs.library }} weblogs: ${{ needs.compute_parameters.outputs.dockerssi_weblogs }} + + external-processing: + needs: + - compute_parameters + if: ${{ needs.compute_parameters.outputs.externalprocessing_scenarios != '[]' && inputs.library == 'golang' && inputs.binaries_artifact != ''}} + uses: ./.github/workflows/run-external-processing.yml + secrets: inherit + with: + build_proxy_image: ${{ inputs.build_proxy_image }} + ci_environment: ${{ inputs.ci_environment }} + binaries_artifact: ${{ inputs.binaries_artifact }} diff --git a/docs/scenarios/external_processing.md b/docs/scenarios/external_processing.md new file mode 100644 index 00000000000..2646668dbdd --- /dev/null +++ b/docs/scenarios/external_processing.md @@ -0,0 +1,17 @@ +```mermaid +flowchart LR +%% Nodes + A("Test runner") + B("Envoy") + C("External Processing") + D("HTTP app") + E("Proxy") + F("Agent") + G("Backend") + +%% Edge connections between nodes + A --> B --> D + B --> C --> B + C --> E --> F --> G + %% D -- Mermaid js --> I --> J +``` \ No newline at end of file diff --git a/tests/external_processing/envoy.yaml b/tests/external_processing/envoy.yaml new file mode 100644 index 00000000000..95fa7cca2aa --- /dev/null +++ b/tests/external_processing/envoy.yaml @@ -0,0 +1,78 @@ +static_resources: + listeners: + - name: listener_0 + address: + socket_address: + address: 0.0.0.0 + port_value: 80 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + codec_type: AUTO + route_config: + name: local_route + virtual_hosts: + - name: backend + domains: + - "*" + routes: + - match: + prefix: "/" + route: + cluster: web_service + http_filters: + - name: envoy.filters.http.ext_proc + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor + grpc_service: + envoy_grpc: + cluster_name: ext_proc_cluster + timeout: 0.25s + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: web_service + connect_timeout: 0.25s + type: STRICT_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: web_service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: http-app + port_value: 8080 + + - name: ext_proc_cluster + connect_timeout: 0.25s + type: STRICT_DNS + lb_policy: ROUND_ROBIN + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + sni: extproc + load_assignment: + cluster_name: ext_proc_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: extproc + port_value: 443 + +# used for health checking +# admin: +# access_log_path: "/tmp/admin_access.log" +# address: +# socket_address: +# address: 0.0.0.0 +# port_value: 9901 diff --git a/tests/external_processing/test_external_processing.py b/tests/external_processing/test_external_processing.py new file mode 100644 index 00000000000..8afaca34279 --- /dev/null +++ b/tests/external_processing/test_external_processing.py @@ -0,0 +1,16 @@ +from utils import weblog, interfaces, scenarios, features + + +@features.not_reported # it's just a POC. We'll need to figure out how we want to see results in FPD +@scenarios.external_processing +class Test_ExternalProcessing: + def setup_main(self): + self.r = weblog.get("/mock", params={"status_code": 200}) + + def test_main(self): + assert self.r.status_code == 200 + + interfaces.library.assert_trace_exists(self.r) + + for _, span in interfaces.library.get_root_spans(request=self.r): + assert span["meta"]["http.url"] == "http://localhost:7777/mock?status_code=200" diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 4185cee5293..21c19ade7a0 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -16,6 +16,7 @@ from .auto_injection import InstallerAutoInjectionScenario from .k8s_lib_injection import KubernetesScenario, WeblogInjectionScenario from .docker_ssi import DockerSSIScenario +from .external_processing import ExternalProcessingScenario update_environ_with_local_env() @@ -696,6 +697,8 @@ def all_endtoend_scenarios(test_object): scenario_groups=[ScenarioGroup.APPSEC], ) + external_processing = ExternalProcessingScenario("EXTERNAL_PROCESSING") + def get_all_scenarios() -> list[Scenario]: result = [] diff --git a/utils/_context/_scenarios/core.py b/utils/_context/_scenarios/core.py index 8b8b8205154..3266f15ab8b 100644 --- a/utils/_context/_scenarios/core.py +++ b/utils/_context/_scenarios/core.py @@ -23,6 +23,7 @@ class ScenarioGroup(Enum): ONBOARDING = "onboarding" DOCKER_SSI = "docker-ssi" ESSENTIALS = "essentials" + EXTERNAL_PROCESSING = "external-processing" VALID_GITHUB_WORKFLOWS = { @@ -34,6 +35,7 @@ class ScenarioGroup(Enum): "parametric", "testthetest", "dockerssi", + "externalprocessing", } diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index cc284ee71c0..d1525b495c1 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -106,6 +106,30 @@ def get_container_by_dd_integration_name(self, name): return container return None + def _start_interfaces_watchdog(self, interfaces): + class Event(FileSystemEventHandler): + def __init__(self, interface) -> None: + super().__init__() + self.interface = interface + + def _ingest(self, event): + if event.is_directory: + return + + self.interface.ingest_file(event.src_path) + + on_modified = _ingest + on_created = _ingest + + # lot of issue using the default OS dependant notifiers (not working on WSL, reaching some inotify watcher + # limits on Linux) -> using the good old bare polling system + observer = PollingObserver() + + for interface in interfaces: + observer.schedule(Event(interface), path=interface._log_folder) + + observer.start() + def get_warmups(self): warmups = super().get_warmups() @@ -306,41 +330,19 @@ def _create_interface_folders(self): for container in self.buddies: self._create_log_subfolder(f"interfaces/{container.interface.name}") - def _start_interface_watchdog(self): + def _start_interfaces_watchdog(self, _=None): from utils import interfaces - class Event(FileSystemEventHandler): - def __init__(self, interface) -> None: - super().__init__() - self.interface = interface - - def _ingest(self, event): - if event.is_directory: - return - - self.interface.ingest_file(event.src_path) - - on_modified = _ingest - on_created = _ingest - - # lot of issue using the default OS dependant notifiers (not working on WSL, reaching some inotify watcher - # limits on Linux) -> using the good old bare polling system - observer = PollingObserver() - - observer.schedule(Event(interfaces.library), path=f"{self.host_log_folder}/interfaces/library") - observer.schedule(Event(interfaces.agent), path=f"{self.host_log_folder}/interfaces/agent") - - for container in self.buddies: - observer.schedule(Event(container.interface), path=container.interface._log_folder) - - observer.start() + super()._start_interfaces_watchdog( + [interfaces.library, interfaces.agent] + [container.interface for container in self.buddies] + ) def get_warmups(self): warmups = super().get_warmups() if not self.replay: warmups.insert(0, self._create_interface_folders) - warmups.insert(1, self._start_interface_watchdog) + warmups.insert(1, self._start_interfaces_watchdog) warmups.append(self._get_weblog_system_info) warmups.append(self._wait_for_app_readiness) diff --git a/utils/_context/_scenarios/external_processing.py b/utils/_context/_scenarios/external_processing.py new file mode 100644 index 00000000000..f96c11fd82e --- /dev/null +++ b/utils/_context/_scenarios/external_processing.py @@ -0,0 +1,124 @@ +import pytest + +from utils._context.containers import DummyServerContainer, ExternalProcessingContainer, EnvoyContainer, AgentContainer +from utils.tools import logger + +from .endtoend import DockerScenario, ScenarioGroup + + +class ExternalProcessingScenario(DockerScenario): + def __init__(self, name): + super().__init__( + name, + doc="Envoy + external processing", + github_workflow="externalprocessing", + scenario_groups=[ScenarioGroup.END_TO_END, ScenarioGroup.EXTERNAL_PROCESSING], + use_proxy=True, + ) + + self._agent_container = AgentContainer(self.host_log_folder) + self._external_processing_container = ExternalProcessingContainer(self.host_log_folder) + self._envoy_container = EnvoyContainer(self.host_log_folder) + self._http_app_container = DummyServerContainer(self.host_log_folder) + + self._agent_container.depends_on.append(self.proxy_container) + self._external_processing_container.depends_on.append(self.proxy_container) + + self._required_containers.append(self._agent_container) + self._required_containers.append(self._external_processing_container) + self._required_containers.append(self._envoy_container) + self._required_containers.append(self._http_app_container) + + # start envoyproxy/envoy:v1.31-latest⁠ + # -> envoy.yaml configuration in tests/external_processing/envoy.yaml + + # start dummy http app on weblog port + # -> server.py in tests/external_processing/server.py + + # start system-tests proxy + # start agent + # start service extension + # with agent url threw system-tests proxy + + # service extension image: + # https://github.com/DataDog/dd-trace-go/pkgs/container/dd-trace-go%2Fservice-extensions-callout + # Version: + # tag: dev + # base: latest/v*.*.* + + def _create_interface_folders(self): + self._create_log_subfolder("interfaces/agent") + self._create_log_subfolder("interfaces/library") + + def _start_interfaces_watchdog(self, _=None): + from utils import interfaces + + super()._start_interfaces_watchdog([interfaces.library, interfaces.agent]) + + def _wait_for_app_readiness(self): + from utils import interfaces # import here to avoid circular import + + logger.debug("Wait for app readiness") + + if not interfaces.library.ready.wait(40): + pytest.exit("Nothing received from external processing", 1) + logger.debug("Library ready") + + if not interfaces.agent.ready.wait(40): + pytest.exit("Datadog agent not ready", 1) + logger.debug("Agent ready") + + def get_warmups(self) -> list: + warmups = super().get_warmups() + + if not self.replay: + warmups.insert(0, self._create_interface_folders) + warmups.insert(1, self._start_interfaces_watchdog) + warmups.append(self._wait_for_app_readiness) + + return warmups + + def post_setup(self): + try: + self._wait_and_stop_containers() + finally: + self.close_targets() + + def _wait_and_stop_containers(self): + from utils import interfaces + + if self.replay: + logger.terminal.write_sep("-", "Load all data from logs") + logger.terminal.flush() + + interfaces.library.load_data_from_logs() + interfaces.library.check_deserialization_errors() + + interfaces.agent.load_data_from_logs() + interfaces.agent.check_deserialization_errors() + + else: + self._wait_interface(interfaces.library, 5) + + self._http_app_container.stop() + self._envoy_container.stop() + self._external_processing_container.stop() + + interfaces.library.check_deserialization_errors() + + self._agent_container.stop() + interfaces.agent.check_deserialization_errors() + + def _wait_interface(self, interface, timeout): + logger.terminal.write_sep("-", f"Wait for {interface} ({timeout}s)") + logger.terminal.flush() + + interface.wait(timeout) + + @property + def weblog_variant(self): + return "external-processing" + + @property + def library(self): + return self._external_processing_container.library diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 47aff86fc9b..9e4359706a0 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -1071,3 +1071,57 @@ def get_env(self, env_var): """Get env variables from the container """ env = self.image.env | self.environment return env.get(env_var) + + +class DummyServerContainer(TestedContainer): + def __init__(self, host_log_folder) -> None: + super().__init__( + image_name="jasonrm/dummy-server:latest", + name="http-app", + host_log_folder=host_log_folder, + healthcheck={"test": "wget http://localhost:8080", "retries": 10,}, + ) + + +class EnvoyContainer(TestedContainer): + def __init__(self, host_log_folder) -> None: + + from utils import weblog + + super().__init__( + image_name="envoyproxy/envoy:v1.31-latest", + name="envoy", + host_log_folder=host_log_folder, + volumes={"./tests/external_processing/envoy.yaml": {"bind": "/etc/envoy/envoy.yaml", "mode": "ro",}}, + ports={"80": ("127.0.0.1", weblog.port)}, + # healthcheck={"test": "wget http://localhost:9901/ready", "retries": 10,}, # no wget on envoy + ) + + +class ExternalProcessingContainer(TestedContainer): + library: LibraryVersion + + def __init__(self, host_log_folder) -> None: + try: + with open("binaries/golang-service-extensions-callout-image", "r", encoding="utf-8") as f: + image = f.read().strip() + except FileNotFoundError: + image = "ghcr.io/datadog/dd-trace-go/service-extensions-callout:latest" + + super().__init__( + image_name=image, + name="extproc", + host_log_folder=host_log_folder, + environment={"DD_APPSEC_ENABLED": "true", "DD_AGENT_HOST": "proxy", "DD_TRACE_AGENT_PORT": 8126,}, + healthcheck={"test": "wget -qO- http://localhost:80/", "retries": 10,}, + ) + + def post_start(self): + with open(self.healthcheck_log_file, mode="r", encoding="utf-8") as f: + data = json.load(f) + lib = data["library"] + + self.library = LibraryVersion(lib["language"], lib["version"]) + + logger.stdout(f"Library: {self.library}") + logger.stdout(f"Image: {self.image.name}") diff --git a/utils/scripts/load-binary.sh b/utils/scripts/load-binary.sh index 9e104faf2d5..743f23bb545 100755 --- a/utils/scripts/load-binary.sh +++ b/utils/scripts/load-binary.sh @@ -197,6 +197,9 @@ elif [ "$TARGET" = "golang" ]; then echo "Using gopkg.in/DataDog/dd-trace-go.v1@main" echo "gopkg.in/DataDog/dd-trace-go.v1@main" > golang-load-from-go-get + echo "Using ghcr.io/datadog/dd-trace-go/service-extensions-callout:dev" + echo "ghcr.io/datadog/dd-trace-go/service-extensions-callout:dev" > golang-service-extensions-callout-image + elif [ "$TARGET" = "cpp" ]; then assert_version_is_dev # get_circleci_artifact "gh/DataDog/dd-opentracing-cpp" "build_test_deploy" "build" "TBD"