From 00233ba4bdc0f7a36e98c33053db6a4317bdcc31 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 5 Nov 2025 11:56:27 +0100 Subject: [PATCH 1/4] opamp: extend client for mTLS --- src/opentelemetry/_opamp/client.py | 15 +++- src/opentelemetry/_opamp/transport/base.py | 14 ++- .../_opamp/transport/requests.py | 23 ++++- tests/opamp/test_client.py | 6 ++ tests/opamp/transport/test_requests.py | 87 ++++++++++++++++++- 5 files changed, 137 insertions(+), 8 deletions(-) diff --git a/src/opentelemetry/_opamp/client.py b/src/opentelemetry/_opamp/client.py index 0522ff1..d5aa608 100644 --- a/src/opentelemetry/_opamp/client.py +++ b/src/opentelemetry/_opamp/client.py @@ -60,6 +60,10 @@ def __init__( timeout_millis: int = _DEFAULT_OPAMP_TIMEOUT_MS, agent_identifying_attributes: Mapping[str, AnyValue], agent_non_identifying_attributes: Mapping[str, AnyValue] | None = None, + # this matches requests but can be mapped to other http libraries APIs + tls_certificate: str | bool = True, + tls_client_certificate: str | None = None, + tls_client_key: str | None = None, ): self._timeout_millis = timeout_millis self._transport = RequestsTransport() @@ -67,6 +71,9 @@ def __init__( self._endpoint = endpoint headers = headers or {} self._headers = {**_OPAMP_HTTP_HEADERS, **headers} + self._tls_certificate = tls_certificate + self._tls_client_certificate = tls_client_certificate + self._tls_client_key = tls_client_key self._agent_description = messages._build_agent_description( identifying_attributes=agent_identifying_attributes, @@ -154,7 +161,13 @@ def _send(self, data: bytes): token = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)) try: response = self._transport.send( - url=self._endpoint, headers=self._headers, data=data, timeout_millis=self._timeout_millis + url=self._endpoint, + headers=self._headers, + data=data, + timeout_millis=self._timeout_millis, + tls_certificate=self._tls_certificate, + tls_client_certificate=self._tls_client_certificate, + tls_client_key=self._tls_client_key, ) return response finally: diff --git a/src/opentelemetry/_opamp/transport/base.py b/src/opentelemetry/_opamp/transport/base.py index 1ca426c..e819184 100644 --- a/src/opentelemetry/_opamp/transport/base.py +++ b/src/opentelemetry/_opamp/transport/base.py @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + import abc from typing import Mapping @@ -27,5 +29,15 @@ class HttpTransport(abc.ABC): @abc.abstractmethod - def send(self, url: str, headers: Mapping[str, str], data: bytes, timeout_millis: int) -> opamp_pb2.ServerToAgent: + def send( + self, + *, + url: str, + headers: Mapping[str, str], + data: bytes, + timeout_millis: int, + tls_certificate: str | bool, + tls_client_certificate: str | None = None, + tls_client_key: str | None = None, + ) -> opamp_pb2.ServerToAgent: pass diff --git a/src/opentelemetry/_opamp/transport/requests.py b/src/opentelemetry/_opamp/transport/requests.py index 555c75a..235719f 100644 --- a/src/opentelemetry/_opamp/transport/requests.py +++ b/src/opentelemetry/_opamp/transport/requests.py @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + import logging from typing import Mapping @@ -32,11 +34,28 @@ def __init__(self): self.session = requests.Session() # TODO: support basic-auth? - def send(self, url: str, headers: Mapping[str, str], data: bytes, timeout_millis: int): + def send( + self, + *, + url: str, + headers: Mapping[str, str], + data: bytes, + timeout_millis: int, + tls_certificate: str | bool, + tls_client_certificate: str | None = None, + tls_client_key: str | None = None, + ): headers = {**base_headers, **headers} timeout: float = timeout_millis / 1e3 + client_cert = ( + (tls_client_certificate, tls_client_key) + if tls_client_certificate and tls_client_key + else tls_client_certificate + ) try: - response = self.session.post(url, headers=headers, data=data, timeout=timeout) + response = self.session.post( + url, headers=headers, data=data, timeout=timeout, verify=tls_certificate, cert=client_cert + ) response.raise_for_status() except Exception as exc: logger.error(str(exc)) diff --git a/tests/opamp/test_client.py b/tests/opamp/test_client.py index a07c050..62f2f62 100644 --- a/tests/opamp/test_client.py +++ b/tests/opamp/test_client.py @@ -88,6 +88,9 @@ def test_client_headers_override_defaults(): headers={"Content-Type": "application/x-protobuf", "User-Agent": "Custom"}, data=b"", timeout_millis=1000, + tls_certificate=True, + tls_client_certificate=None, + tls_client_key=None, ) @@ -347,6 +350,9 @@ def test_send(client): headers={"Content-Type": "application/x-protobuf", "User-Agent": "OTel-OpAMP-Python/" + __version__}, data=b"foo", timeout_millis=1000, + tls_certificate=True, + tls_client_certificate=None, + tls_client_key=None, ) diff --git a/tests/opamp/transport/test_requests.py b/tests/opamp/transport/test_requests.py index bc7e4f4..1d39b09 100644 --- a/tests/opamp/transport/test_requests.py +++ b/tests/opamp/transport/test_requests.py @@ -39,10 +39,87 @@ def test_can_send(): data = b"" with mock.patch.object(transport, "session") as session_mock: session_mock.post.return_value = response_mock - response = transport.send("http://127.0.0.1/v1/opamp", headers=headers, data=data, timeout_millis=1_000) + response = transport.send( + url="http://127.0.0.1/v1/opamp", headers=headers, data=data, timeout_millis=1_000, tls_certificate=True + ) + + session_mock.post.assert_called_once_with( + "http://127.0.0.1/v1/opamp", headers=expected_headers, data=data, timeout=1, verify=True, cert=None + ) + + assert isinstance(response, opamp_pb2.ServerToAgent) + + +def test_send_tls_certiticate_mapped_to_verify(): + transport = RequestsTransport() + serialized_message = opamp_pb2.ServerToAgent().SerializeToString() + response_mock = mock.Mock(content=serialized_message) + data = b"" + with mock.patch.object(transport, "session") as session_mock: + session_mock.post.return_value = response_mock + response = transport.send( + url="http://127.0.0.1/v1/opamp", headers={}, data=data, timeout_millis=1_000, tls_certificate=False + ) + + session_mock.post.assert_called_once_with( + "http://127.0.0.1/v1/opamp", headers=base_headers, data=data, timeout=1, verify=False, cert=None + ) + + assert isinstance(response, opamp_pb2.ServerToAgent) + + +def test_send_mtls(): + transport = RequestsTransport() + serialized_message = opamp_pb2.ServerToAgent().SerializeToString() + response_mock = mock.Mock(content=serialized_message) + data = b"" + with mock.patch.object(transport, "session") as session_mock: + session_mock.post.return_value = response_mock + response = transport.send( + url="http://127.0.0.1/v1/opamp", + headers={}, + data=data, + timeout_millis=1_000, + tls_certificate="server.pem", + tls_client_certificate="client.pem", + tls_client_key="client.key", + ) + + session_mock.post.assert_called_once_with( + "http://127.0.0.1/v1/opamp", + headers=base_headers, + data=data, + timeout=1, + verify="server.pem", + cert=("client.pem", "client.key"), + ) + + assert isinstance(response, opamp_pb2.ServerToAgent) + + +def test_send_mtls_no_client_key(): + transport = RequestsTransport() + serialized_message = opamp_pb2.ServerToAgent().SerializeToString() + response_mock = mock.Mock(content=serialized_message) + data = b"" + with mock.patch.object(transport, "session") as session_mock: + session_mock.post.return_value = response_mock + response = transport.send( + url="http://127.0.0.1/v1/opamp", + headers={}, + data=data, + timeout_millis=1_000, + tls_certificate="server.pem", + tls_client_certificate="client.pem", + ) session_mock.post.assert_called_once_with( - "http://127.0.0.1/v1/opamp", headers=expected_headers, data=data, timeout=1 + "http://127.0.0.1/v1/opamp", + headers=base_headers, + data=data, + timeout=1, + verify="server.pem", + cert="client.pem", ) assert isinstance(response, opamp_pb2.ServerToAgent) @@ -58,8 +135,10 @@ def test_send_exceptions_raises_opamp_exception(): session_mock.post.return_value = response_mock response_mock.raise_for_status.side_effect = Exception with pytest.raises(OpAMPException): - transport.send("http://127.0.0.1/v1/opamp", headers=headers, data=data, timeout_millis=1_000) + transport.send( + url="http://127.0.0.1/v1/opamp", headers=headers, data=data, timeout_millis=1_000, tls_certificate=True + ) session_mock.post.assert_called_once_with( - "http://127.0.0.1/v1/opamp", headers=expected_headers, data=data, timeout=1 + "http://127.0.0.1/v1/opamp", headers=expected_headers, data=data, timeout=1, verify=True, cert=None ) From 98c7499996d02a4af2b0c646a970fcfb32127ffc Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 5 Nov 2025 14:50:14 +0100 Subject: [PATCH 2/4] distro: get mTLS configuration from env vars --- src/elasticotel/distro/__init__.py | 10 ++++ .../distro/environment_variables.py | 27 ++++++++++ tests/distro/test_distro.py | 53 ++++++++++++++++++- 3 files changed, 89 insertions(+), 1 deletion(-) diff --git a/src/elasticotel/distro/__init__.py b/src/elasticotel/distro/__init__.py index 76e0503..7fd7f44 100644 --- a/src/elasticotel/distro/__init__.py +++ b/src/elasticotel/distro/__init__.py @@ -60,6 +60,9 @@ ELASTIC_OTEL_OPAMP_ENDPOINT, ELASTIC_OTEL_OPAMP_HEADERS, ELASTIC_OTEL_SYSTEM_METRICS_ENABLED, + ELASTIC_OTEL_OPAMP_CERTIFICATE, + ELASTIC_OTEL_OPAMP_CLIENT_CERTIFICATE, + ELASTIC_OTEL_OPAMP_CLIENT_KEY, ) from elasticotel.distro.resource_detectors import get_cloud_resource_detectors from elasticotel.distro.config import opamp_handler, _initialize_config, DEFAULT_SAMPLING_RATE @@ -129,10 +132,17 @@ def _configure(self, **kwargs): else: headers = None + # If string is a path to the certificate, if bool means to check the server certificate. Behaviour inherited from requests + tls_certificate: str | bool = os.environ.get(ELASTIC_OTEL_OPAMP_CERTIFICATE, True) + tls_client_certificate: str | None = os.environ.get(ELASTIC_OTEL_OPAMP_CLIENT_CERTIFICATE) + tls_client_key: str | None = os.environ.get(ELASTIC_OTEL_OPAMP_CLIENT_KEY) opamp_client = OpAMPClient( endpoint=endpoint_url, agent_identifying_attributes=agent_identifying_attributes, headers=headers, + tls_certificate=tls_certificate, + tls_client_certificate=tls_client_certificate, + tls_client_key=tls_client_key, ) opamp_agent = OpAMPAgent( interval=30, diff --git a/src/elasticotel/distro/environment_variables.py b/src/elasticotel/distro/environment_variables.py index 878186d..ce30921 100644 --- a/src/elasticotel/distro/environment_variables.py +++ b/src/elasticotel/distro/environment_variables.py @@ -40,3 +40,30 @@ **Default value:** ``not set`` """ + +ELASTIC_OTEL_OPAMP_CERTIFICATE = "ELASTIC_OTEL_OPAMP_CERTIFICATE" +""" +.. envvar:: ELASTIC_OTEL_OPAMP_CERTIFICATE + +The path of the trusted certificate to use when verifying a server’s TLS credentials, this is needed for mTLS or when the server is using a self-signed certificate. + +**Default value:** ``not set`` +""" + +ELASTIC_OTEL_OPAMP_CLIENT_CERTIFICATE = "ELASTIC_OTEL_OPAMP_CLIENT_CERTIFICATE" +""" +.. envvar:: ELASTIC_OTEL_OPAMP_CLIENT_CERTIFICATE + +Client certificate/chain trust for clients private key path to use in mTLS communication in PEM format. + +**Default value:** ``not set`` +""" + +ELASTIC_OTEL_OPAMP_CLIENT_KEY = "ELASTIC_OTEL_OPAMP_CLIENT_KEY" +""" +.. envvar:: ELASTIC_OTEL_OPAMP_CLIENT_KEY + +Client private key path to use in mTLS communication in PEM format. + +**Default value:** ``not set`` +""" diff --git a/tests/distro/test_distro.py b/tests/distro/test_distro.py index 58250da..4746fad 100644 --- a/tests/distro/test_distro.py +++ b/tests/distro/test_distro.py @@ -21,7 +21,13 @@ from elasticotel.distro import ElasticOpenTelemetryConfigurator, ElasticOpenTelemetryDistro, logger as distro_logger from elasticotel.distro.config import opamp_handler, logger as config_logger, Config -from elasticotel.distro.environment_variables import ELASTIC_OTEL_OPAMP_ENDPOINT, ELASTIC_OTEL_SYSTEM_METRICS_ENABLED +from elasticotel.distro.environment_variables import ( + ELASTIC_OTEL_OPAMP_ENDPOINT, + ELASTIC_OTEL_OPAMP_CERTIFICATE, + ELASTIC_OTEL_OPAMP_CLIENT_CERTIFICATE, + ELASTIC_OTEL_OPAMP_CLIENT_KEY, + ELASTIC_OTEL_SYSTEM_METRICS_ENABLED, +) from elasticotel.sdk.sampler import DefaultSampler from opentelemetry.environment_variables import ( OTEL_LOGS_EXPORTER, @@ -162,6 +168,9 @@ def test_configurator_sets_up_opamp_with_http_endpoint(self, client_mock, agent_ endpoint="http://localhost:4320/v1/opamp", agent_identifying_attributes={"service.name": "service", "deployment.environment.name": "dev"}, headers=None, + tls_certificate=True, + tls_client_certificate=None, + tls_client_key=None, ) agent_mock.assert_called_once_with(interval=30, message_handler=opamp_handler, client=client_mock) agent_mock.start.assert_called_once_with() @@ -186,6 +195,9 @@ def test_configurator_sets_up_opamp_with_https_endpoint(self, client_mock, agent endpoint="https://localhost:4320/v1/opamp", agent_identifying_attributes={"service.name": "service", "deployment.environment.name": "dev"}, headers=None, + tls_certificate=True, + tls_client_certificate=None, + tls_client_key=None, ) agent_mock.assert_called_once_with(interval=30, message_handler=opamp_handler, client=client_mock) agent_mock.start.assert_called_once_with() @@ -211,6 +223,9 @@ def test_configurator_sets_up_opamp_with_headers_from_environment_variable(self, endpoint="http://localhost:4320/v1/opamp", agent_identifying_attributes={"service.name": "service", "deployment.environment.name": "dev"}, headers={"authorization": "ApiKey foobar==="}, + tls_certificate=True, + tls_client_certificate=None, + tls_client_key=None, ) agent_mock.assert_called_once_with(interval=30, message_handler=opamp_handler, client=client_mock) agent_mock.start.assert_called_once_with() @@ -235,6 +250,9 @@ def test_configurator_adds_path_to_opamp_endpoint_if_missing(self, client_mock, endpoint="https://localhost:4320/v1/opamp", agent_identifying_attributes={"service.name": "service", "deployment.environment.name": "dev"}, headers=None, + tls_certificate=True, + tls_client_certificate=None, + tls_client_key=None, ) agent_mock.assert_called_once_with(interval=30, message_handler=opamp_handler, client=client_mock) agent_mock.start.assert_called_once_with() @@ -259,6 +277,39 @@ def test_configurator_sets_up_opamp_without_deployment_environment_name(self, cl endpoint="https://localhost:4320/v1/opamp", agent_identifying_attributes={"service.name": "service"}, headers=None, + tls_certificate=True, + tls_client_certificate=None, + tls_client_key=None, + ) + agent_mock.assert_called_once_with(interval=30, message_handler=opamp_handler, client=client_mock) + agent_mock.start.assert_called_once_with() + + @mock.patch.dict( + "os.environ", + { + ELASTIC_OTEL_OPAMP_ENDPOINT: "https://localhost:4320/v1/opamp", + ELASTIC_OTEL_OPAMP_CERTIFICATE: "server.pem", + ELASTIC_OTEL_OPAMP_CLIENT_CERTIFICATE: "client.pem", + ELASTIC_OTEL_OPAMP_CLIENT_KEY: "client.key", + "OTEL_RESOURCE_ATTRIBUTES": "service.name=service", + }, + clear=True, + ) + @mock.patch("elasticotel.distro.OpAMPAgent") + @mock.patch("elasticotel.distro.OpAMPClient") + def test_configurator_sets_up_opamp_with_mTLS_variables(self, client_mock, agent_mock): + client_mock.return_value = client_mock + agent_mock.return_value = agent_mock + + ElasticOpenTelemetryConfigurator()._configure() + + client_mock.assert_called_once_with( + endpoint="https://localhost:4320/v1/opamp", + agent_identifying_attributes={"service.name": "service"}, + headers=None, + tls_certificate="server.pem", + tls_client_certificate="client.pem", + tls_client_key="client.key", ) agent_mock.assert_called_once_with(interval=30, message_handler=opamp_handler, client=client_mock) agent_mock.start.assert_called_once_with() From 4df55fa1ce1f905b32aea5b7a1c508d432186048 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Thu, 6 Nov 2025 16:22:03 +0100 Subject: [PATCH 3/4] Add documentation --- docs/reference/edot-python/configuration.md | 23 ++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/reference/edot-python/configuration.md b/docs/reference/edot-python/configuration.md index cbf468e..a422009 100644 --- a/docs/reference/edot-python/configuration.md +++ b/docs/reference/edot-python/configuration.md @@ -67,10 +67,31 @@ product: If the OpAMP server is configured to require authentication set the `ELASTIC_OTEL_OPAMP_HEADERS` environment variable. -``` +```sh export ELASTIC_OTEL_OPAMP_HEADERS="Authorization=ApiKey an_api_key" ``` +### Configure mTLS for Central configuration + +```{applies_to} +serverless: unavailable +stack: preview 9.1 +product: + edot_python: preview 1.10.0 +``` + +If the OpAMP Central configuration server requires mutual TLS to encrypt data in transit you need to set the following environment variables: + +- `ELASTIC_OTEL_OPAMP_CERTIFICATE`: The path of the trusted certificate to use when verifying a server’s TLS credentials, this may also be used if the server is using a self-signed certificate. +- `ELASTIC_OTEL_OPAMP_CLIENT_CERTIFICATE`: Client certificate/chain trust for clients private key path to use in mTLS communication in PEM format. +- `ELASTIC_OTEL_OPAMP_CLIENT_KEY`: Client private key path to use in mTLS communication in PEM format. + +```sh +export ELASTIC_OTEL_OPAMP_CERTIFICATE=/path/to/rootCA.pem +export ELASTIC_OTEL_OPAMP_CLIENT_CERTIFICATE=/path/to/client.pem +export ELASTIC_OTEL_OPAMP_CLIENT_KEY=/path/to/client-key.pem +``` + ### Central configuration settings You can modify the following settings for EDOT Python through APM Agent Central Configuration: From d1fe2436bbdf76dfb6cfafe6155a04b734a5d93c Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Fri, 7 Nov 2025 09:47:45 +0100 Subject: [PATCH 4/4] Aplly feedback from review --- tests/opamp/transport/test_requests.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/opamp/transport/test_requests.py b/tests/opamp/transport/test_requests.py index 1d39b09..139bb37 100644 --- a/tests/opamp/transport/test_requests.py +++ b/tests/opamp/transport/test_requests.py @@ -50,7 +50,7 @@ def test_can_send(): assert isinstance(response, opamp_pb2.ServerToAgent) -def test_send_tls_certiticate_mapped_to_verify(): +def test_send_tls_certificate_mapped_to_verify(): transport = RequestsTransport() serialized_message = opamp_pb2.ServerToAgent().SerializeToString() response_mock = mock.Mock(content=serialized_message) @@ -58,11 +58,11 @@ def test_send_tls_certiticate_mapped_to_verify(): with mock.patch.object(transport, "session") as session_mock: session_mock.post.return_value = response_mock response = transport.send( - url="http://127.0.0.1/v1/opamp", headers={}, data=data, timeout_millis=1_000, tls_certificate=False + url="https://127.0.0.1/v1/opamp", headers={}, data=data, timeout_millis=1_000, tls_certificate=False ) session_mock.post.assert_called_once_with( - "http://127.0.0.1/v1/opamp", headers=base_headers, data=data, timeout=1, verify=False, cert=None + "https://127.0.0.1/v1/opamp", headers=base_headers, data=data, timeout=1, verify=False, cert=None ) assert isinstance(response, opamp_pb2.ServerToAgent) @@ -76,7 +76,7 @@ def test_send_mtls(): with mock.patch.object(transport, "session") as session_mock: session_mock.post.return_value = response_mock response = transport.send( - url="http://127.0.0.1/v1/opamp", + url="https://127.0.0.1/v1/opamp", headers={}, data=data, timeout_millis=1_000, @@ -86,7 +86,7 @@ def test_send_mtls(): ) session_mock.post.assert_called_once_with( - "http://127.0.0.1/v1/opamp", + "https://127.0.0.1/v1/opamp", headers=base_headers, data=data, timeout=1, @@ -105,7 +105,7 @@ def test_send_mtls_no_client_key(): with mock.patch.object(transport, "session") as session_mock: session_mock.post.return_value = response_mock response = transport.send( - url="http://127.0.0.1/v1/opamp", + url="https://127.0.0.1/v1/opamp", headers={}, data=data, timeout_millis=1_000, @@ -114,7 +114,7 @@ def test_send_mtls_no_client_key(): ) session_mock.post.assert_called_once_with( - "http://127.0.0.1/v1/opamp", + "https://127.0.0.1/v1/opamp", headers=base_headers, data=data, timeout=1,