From 715ae0ed7b6b7ac21fae1f79ae1e417c35578a8a Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Tue, 9 Jan 2024 10:17:47 -0800 Subject: [PATCH 1/2] distro-detect --- .../opentelemetry/exporter/_constants.py | 2 + .../opentelemetry/exporter/export/_base.py | 1 + .../exporter/statsbeat/_statsbeat.py | 1 + .../exporter/statsbeat/_statsbeat_metrics.py | 4 ++ .../tests/statsbeat/test_statsbeat.py | 59 ++++++++++++++++++- .../tests/test_base_exporter.py | 2 + .../azure/monitor/opentelemetry/_constants.py | 1 + .../opentelemetry/_util/configurations.py | 3 + 8 files changed, 71 insertions(+), 2 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_constants.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_constants.py index 566c1b53680c..7d2f8cbf3008 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_constants.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_constants.py @@ -119,6 +119,8 @@ "urllib", "urllib3", _AZURE_SDK_OPENTELEMETRY_NAME, + "cassandra", + "tortoiseorm", ] _INSTRUMENTATIONS_BIT_MAP = {_INSTRUMENTATIONS_LIST[i]: _BASE**i for i in range(len(_INSTRUMENTATIONS_LIST))} diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/_base.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/_base.py index 696694fe4cf3..4c7e988d9135 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/_base.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/_base.py @@ -100,6 +100,7 @@ def __init__(self, **kwargs: Any) -> None: self._storage_directory = kwargs.get('storage_directory', default_storage_directory) # Storage path in which to store retry files. self._storage_retention_period = kwargs.get('storage_retention_period', 48 * 60 * 60) # Retention period in seconds (default 48 hrs) self._timeout = kwargs.get('timeout', 10.0) # networking timeout in seconds + self._distro_version = kwargs.get('distro_version', '') # If set, indicates the exporter is instantiated via Azure monitor OpenTelemetry distro. Versions corresponds to distro version. config = AzureMonitorClientConfiguration(self._endpoint, **kwargs) policies = [ diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/statsbeat/_statsbeat.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/statsbeat/_statsbeat.py index 2cf290cf8e21..c57c5a77b589 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/statsbeat/_statsbeat.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/statsbeat/_statsbeat.py @@ -62,6 +62,7 @@ def collect_statsbeat_metrics(exporter) -> None: exporter._disable_offline_storage, long_interval_threshold, exporter._credential is not None, + exporter._distro_version, ) # Export some initial stats on program start mp.force_flush() diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/statsbeat/_statsbeat_metrics.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/statsbeat/_statsbeat_metrics.py index 28b13c665373..ebcfe545b5fc 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/statsbeat/_statsbeat_metrics.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/statsbeat/_statsbeat_metrics.py @@ -53,6 +53,7 @@ class _StatsbeatFeature: DISK_RETRY = 1 AAD = 2 CUSTOM_EVENTS_EXTENSION = 4 + DISTRO = 8 class _AttachTypes: @@ -97,6 +98,7 @@ def __init__( disable_offline_storage: bool, long_interval_threshold: int, has_credential: bool, + distro_version: str = "", ) -> None: self._ikey = instrumentation_key self._feature = _StatsbeatFeature.NONE @@ -104,6 +106,8 @@ def __init__( self._feature |= _StatsbeatFeature.DISK_RETRY if has_credential: self._feature |= _StatsbeatFeature.AAD + if distro_version: + self._feature |= _StatsbeatFeature.DISTRO if get_statsbeat_custom_events_feature_set(): self._feature |= _StatsbeatFeature.CUSTOM_EVENTS_EXTENSION self._ikey = instrumentation_key diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/statsbeat/test_statsbeat.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/statsbeat/test_statsbeat.py index 66cf05863fea..7ca17ba44743 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/statsbeat/test_statsbeat.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/statsbeat/test_statsbeat.py @@ -176,6 +176,7 @@ def test_collect_statsbeat_metrics_aad( exporter._instrumentation_key = TEST_IKEY exporter._disable_offline_storage = False exporter._credential = TEST_CREDENTIAL + exporter._distro_version = "" _statsbeat.collect_statsbeat_metrics(exporter) mock_statsbeat_metrics.assert_called_once_with( mock.ANY, @@ -184,6 +185,7 @@ def test_collect_statsbeat_metrics_aad( False, _DEFAULT_STATS_LONG_EXPORT_INTERVAL / _DEFAULT_STATS_SHORT_EXPORT_INTERVAL, True, + "", ) @@ -207,6 +209,7 @@ def test_collect_statsbeat_metrics_no_aad( exporter._instrumentation_key = TEST_IKEY exporter._disable_offline_storage = False exporter._credential = TEST_CREDENTIAL + exporter._distro_version = "" _statsbeat.collect_statsbeat_metrics(exporter) mock_statsbeat_metrics.assert_called_once_with( mock.ANY, @@ -215,6 +218,39 @@ def test_collect_statsbeat_metrics_no_aad( False, _DEFAULT_STATS_LONG_EXPORT_INTERVAL / _DEFAULT_STATS_SHORT_EXPORT_INTERVAL, False, + "", + ) + + @mock.patch('azure.monitor.opentelemetry.exporter.statsbeat._statsbeat._StatsbeatMetrics') + @mock.patch.dict( + "os.environ", + { + "APPLICATION_INSIGHTS_STATS_SHORT_EXPORT_INTERVAL": "", + "APPLICATION_INSIGHTS_STATS_LONG_EXPORT_INTERVAL": "", + }, + ) + def test_collect_statsbeat_metrics_distro_version( + self, + mock_statsbeat_metrics, + ): + exporter = mock.Mock() + TEST_ENDPOINT = "test endpoint" + TEST_IKEY = "test ikey" + TEST_CREDENTIAL = None + exporter._endpoint = TEST_ENDPOINT + exporter._instrumentation_key = TEST_IKEY + exporter._disable_offline_storage = False + exporter._credential = TEST_CREDENTIAL + exporter._distro_version = "1.0.0" + _statsbeat.collect_statsbeat_metrics(exporter) + mock_statsbeat_metrics.assert_called_once_with( + mock.ANY, + TEST_IKEY, + TEST_ENDPOINT, + False, + _DEFAULT_STATS_LONG_EXPORT_INTERVAL / _DEFAULT_STATS_SHORT_EXPORT_INTERVAL, + False, + "1.0.0", ) def test_shutdown_statsbeat_metrics(self): @@ -634,7 +670,7 @@ def test_get_feature_metric_does_not_meet_threshold(self): self.assertEqual(metric._long_interval_count_map[_FEATURE_METRIC_NAME[0]], 1) # pylint: disable=protected-access - def test_get_feature_metric(self): + def test_get_feature_metric_storage(self): mp = MeterProvider() ikey = "1aa11111-bbbb-1ccc-8ddd-eeeeffff3334" endpoint = "https://westus-1.in.applicationinsights.azure.com/" @@ -648,7 +684,7 @@ def test_get_feature_metric(self): ) attributes = dict(_StatsbeatMetrics._COMMON_ATTRIBUTES) attributes.update(_StatsbeatMetrics._FEATURE_ATTRIBUTES) - self.assertEqual(attributes["feature"], 1) + self.assertEqual(attributes["feature"], _StatsbeatFeature.DISK_RETRY) self.assertEqual(attributes["type"], _FEATURE_TYPES.FEATURE) observations = metric._get_feature_metric(options=None) for obs in observations: @@ -740,6 +776,25 @@ def test_get_feature_metric_custom_events_runtime(self): self.assertEqual(obs.value, 1) self.assertEqual(obs.attributes, attributes) + # pylint: disable=protected-access + def test_get_feature_metric_distro(self): + mp = MeterProvider() + ikey = "1aa11111-bbbb-1ccc-8ddd-eeeeffff3334" + endpoint = "https://westus-1.in.applicationinsights.azure.com/" + metric = _StatsbeatMetrics( + mp, + ikey, + endpoint, + True, + 0, + False, + "1.0.0" + ) + self.assertEqual(_StatsbeatMetrics._FEATURE_ATTRIBUTES["feature"], _StatsbeatFeature.DISTRO) + observations = metric._get_feature_metric(options=None) + self.assertEqual(len(observations), 1) + self.assertEqual(_StatsbeatMetrics._FEATURE_ATTRIBUTES["feature"], _StatsbeatFeature.DISTRO) + # pylint: disable=protected-access def test_get_feature_metric_instrumentation(self): mp = MeterProvider() diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_exporter.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_exporter.py index 70b3f7c71cf8..6cbfd34beb9d 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_exporter.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_exporter.py @@ -101,6 +101,7 @@ def test_constructor(self): api_version="2021-02-10_Preview", connection_string="InstrumentationKey=4321abcd-5678-4efa-8abc-1234567890ab;IngestionEndpoint=https://westus-0.in.applicationinsights.azure.com/", disable_offline_storage=False, + distro_version="1.0.0", storage_maintenance_period=30, storage_max_size=1000, storage_min_retry_interval=100, @@ -115,6 +116,7 @@ def test_constructor(self): base._endpoint, "https://westus-0.in.applicationinsights.azure.com/", ) + self.assertEqual(base._distro_version, "1.0.0") self.assertIsNotNone(base.storage) self.assertEqual(base.storage._max_size, 1000) self.assertEqual(base.storage._retention_period, 2000) diff --git a/sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_constants.py b/sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_constants.py index 1fdd37ee1f86..fd224dd41a43 100644 --- a/sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_constants.py +++ b/sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_constants.py @@ -21,6 +21,7 @@ DISABLE_LOGGING_ARG = "disable_logging" DISABLE_METRICS_ARG = "disable_metrics" DISABLE_TRACING_ARG = "disable_tracing" +DISTRO_VERSION = "distro_version" LOGGER_NAME_ARG = "logger_name" INSTRUMENTATION_OPTIONS_ARG = "instrumentation_options" RESOURCE_ARG = "resource" diff --git a/sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_util/configurations.py b/sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_util/configurations.py index 6489dd853c10..dcc5312d9ad4 100644 --- a/sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_util/configurations.py +++ b/sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_util/configurations.py @@ -30,12 +30,14 @@ DISABLE_LOGGING_ARG, DISABLE_METRICS_ARG, DISABLE_TRACING_ARG, + DISTRO_VERSION, INSTRUMENTATION_OPTIONS_ARG, LOGGER_NAME_ARG, RESOURCE_ARG, SAMPLING_RATIO_ARG, ) from azure.monitor.opentelemetry._types import ConfigurationValue +from azure.monitor.opentelemetry._version import VERSION _INVALID_FLOAT_MESSAGE = "Value of %s must be a float. Defaulting to %s: %s" @@ -55,6 +57,7 @@ def _get_configurations(**kwargs) -> Dict[str, ConfigurationValue]: for key, val in kwargs.items(): configurations[key] = val + configurations[DISTRO_VERSION] = VERSION _default_disable_logging(configurations) _default_disable_metrics(configurations) From ede5af883e2e649c724ae091f894f2701ff02b5e Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Tue, 9 Jan 2024 13:01:00 -0800 Subject: [PATCH 2/2] move to constant --- sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md | 2 ++ .../azure/monitor/opentelemetry/exporter/_constants.py | 4 ++++ .../azure/monitor/opentelemetry/exporter/export/_base.py | 3 ++- .../monitor/opentelemetry/exporter/export/logs/_exporter.py | 4 +++- .../tests/logs/test_logs.py | 4 +++- sdk/monitor/azure-monitor-opentelemetry/CHANGELOG.md | 4 ++-- .../azure/monitor/opentelemetry/_constants.py | 3 ++- .../azure/monitor/opentelemetry/_util/configurations.py | 4 ++-- 8 files changed, 20 insertions(+), 8 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md b/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md index 4844cd402931..460f3a11c5bb 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md @@ -22,6 +22,8 @@ ([#33667](https://github.com/Azure/azure-sdk-for-python/pull/33667)) - Readme examples are updated with correct imports ([#33691](https://github.com/Azure/azure-sdk-for-python/pull/33691)) +- Implement distro detection for statsbeat feature + ([#33761](https://github.com/Azure/azure-sdk-for-python/pull/33761)) ## 1.0.0b19 (2023-11-20) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_constants.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_constants.py index 7d2f8cbf3008..a6d26f1ca607 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_constants.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_constants.py @@ -46,6 +46,10 @@ _REQUEST_ENVELOPE_NAME = "Microsoft.ApplicationInsights.Request" _REMOTE_DEPENDENCY_ENVELOPE_NAME = "Microsoft.ApplicationInsights.RemoteDependency" +# Feature constants +_APPLICATION_INSIGHTS_EVENT_MARKER_ATTRIBUTE = "APPLICATION_INSIGHTS_EVENT_MARKER_ATTRIBUTE" +_AZURE_MONITOR_DISTRO_VERSION_ARG = "distro_version" + # Statsbeat # (OpenTelemetry metric name, Statsbeat metric name) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/_base.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/_base.py index 4c7e988d9135..3a2dabad3f40 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/_base.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/_base.py @@ -29,6 +29,7 @@ TelemetryItem, ) from azure.monitor.opentelemetry.exporter._constants import ( + _AZURE_MONITOR_DISTRO_VERSION_ARG, _INVALID_STATUS_CODES, _REACHED_INGESTION_STATUS_CODES, _REDIRECT_STATUS_CODES, @@ -100,7 +101,7 @@ def __init__(self, **kwargs: Any) -> None: self._storage_directory = kwargs.get('storage_directory', default_storage_directory) # Storage path in which to store retry files. self._storage_retention_period = kwargs.get('storage_retention_period', 48 * 60 * 60) # Retention period in seconds (default 48 hrs) self._timeout = kwargs.get('timeout', 10.0) # networking timeout in seconds - self._distro_version = kwargs.get('distro_version', '') # If set, indicates the exporter is instantiated via Azure monitor OpenTelemetry distro. Versions corresponds to distro version. + self._distro_version = kwargs.get(_AZURE_MONITOR_DISTRO_VERSION_ARG, '') # If set, indicates the exporter is instantiated via Azure monitor OpenTelemetry distro. Versions corresponds to distro version. config = AzureMonitorClientConfiguration(self._endpoint, **kwargs) policies = [ diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/logs/_exporter.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/logs/_exporter.py index 384d83d1ed1f..cd3a637f90bf 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/logs/_exporter.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/logs/_exporter.py @@ -26,6 +26,9 @@ BaseExporter, ExportResult, ) +from azure.monitor.opentelemetry.exporter._constants import ( + _APPLICATION_INSIGHTS_EVENT_MARKER_ATTRIBUTE, +) from azure.monitor.opentelemetry.exporter.statsbeat._state import ( get_statsbeat_shutdown, get_statsbeat_custom_events_feature_set, @@ -40,7 +43,6 @@ __all__ = ["AzureMonitorLogExporter"] -_APPLICATION_INSIGHTS_EVENT_MARKER_ATTRIBUTE = "APPLICATION_INSIGHTS_EVENT_MARKER_ATTRIBUTE" class AzureMonitorLogExporter(BaseExporter, LogExporter): """Azure Monitor Log exporter for OpenTelemetry.""" diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/logs/test_logs.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/logs/test_logs.py index bfa5e3db6aee..cbdad3eebb40 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/logs/test_logs.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/logs/test_logs.py @@ -17,11 +17,13 @@ from azure.monitor.opentelemetry.exporter.export._base import ExportResult from azure.monitor.opentelemetry.exporter.export.logs._exporter import ( - _APPLICATION_INSIGHTS_EVENT_MARKER_ATTRIBUTE, AzureMonitorLogExporter, _get_log_export_result, _get_severity_level, ) +from azure.monitor.opentelemetry.exporter._constants import ( + _APPLICATION_INSIGHTS_EVENT_MARKER_ATTRIBUTE, +) from azure.monitor.opentelemetry.exporter._generated.models import ContextTagKeys from azure.monitor.opentelemetry.exporter._utils import ( azure_monitor_context, diff --git a/sdk/monitor/azure-monitor-opentelemetry/CHANGELOG.md b/sdk/monitor/azure-monitor-opentelemetry/CHANGELOG.md index e116fbc9ee6f..32718ccf16bc 100644 --- a/sdk/monitor/azure-monitor-opentelemetry/CHANGELOG.md +++ b/sdk/monitor/azure-monitor-opentelemetry/CHANGELOG.md @@ -10,8 +10,8 @@ ### Other Changes -- Shutdown statsbeat on customer exporter 400 status code - ([#31881](https://github.com/Azure/azure-sdk-for-python/pull/31881)) +- Implement distro detection for statsbeat feature + ([#33761](https://github.com/Azure/azure-sdk-for-python/pull/33761)) ## 1.1.1 (2023-12-04) diff --git a/sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_constants.py b/sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_constants.py index fd224dd41a43..2428729d4a5d 100644 --- a/sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_constants.py +++ b/sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_constants.py @@ -21,7 +21,8 @@ DISABLE_LOGGING_ARG = "disable_logging" DISABLE_METRICS_ARG = "disable_metrics" DISABLE_TRACING_ARG = "disable_tracing" -DISTRO_VERSION = "distro_version" +# TODO: Use constant in exporter once available +DISTRO_VERSION_ARG = "distro_version" LOGGER_NAME_ARG = "logger_name" INSTRUMENTATION_OPTIONS_ARG = "instrumentation_options" RESOURCE_ARG = "resource" diff --git a/sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_util/configurations.py b/sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_util/configurations.py index dcc5312d9ad4..405cd7071e41 100644 --- a/sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_util/configurations.py +++ b/sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_util/configurations.py @@ -30,7 +30,7 @@ DISABLE_LOGGING_ARG, DISABLE_METRICS_ARG, DISABLE_TRACING_ARG, - DISTRO_VERSION, + DISTRO_VERSION_ARG, INSTRUMENTATION_OPTIONS_ARG, LOGGER_NAME_ARG, RESOURCE_ARG, @@ -57,7 +57,7 @@ def _get_configurations(**kwargs) -> Dict[str, ConfigurationValue]: for key, val in kwargs.items(): configurations[key] = val - configurations[DISTRO_VERSION] = VERSION + configurations[DISTRO_VERSION_ARG] = VERSION _default_disable_logging(configurations) _default_disable_metrics(configurations)