Add metrics client for trace agent (open-telemetry#31179)

**Description:** <Describe what has changed.>
<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->

Implement `datadog-agent/pkg/trace/metrics.StatsClient` so that we can
record telemetry metrics on the trace agent running in the Datadog
Exporter and Datadog Connector. We implemented the StatsClient in OTel
metric format so that the metrics can be recorded in the OTel Collector
and picked up by the prometheus receiver under the `otelcol_` prefix.

**Link to tracking Issue:** <Issue number if applicable>

**Testing:** <Describe what testing was performed and which tests were
added.>
Added unit tests and tested change by running modified Collector with
opentelemetry-demo and observing generated metrics.

**Documentation:** <Describe the documentation added.>

---------

Co-authored-by: Pablo Baeyens <pbaeyens31+github@gmail.com>

.chloggen/stanley.liu_add-metrics-client-connector.yaml

@@ -0,0 +1,28 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: connector/datadogconnector
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Internal telemetry metrics for the Datadog traces exporter are now reported through the Collector's self-telemetry 
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [31179]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: | 
+   - These internal metrics may be dropped or change name without prior notice
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

.chloggen/stanley.liu_add-metrics-client.yaml

@@ -0,0 +1,28 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: exporter/datadogexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Internal telemetry metrics for the Datadog traces exporter are now reported through the Collector's self-telemetry 
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [31179]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: | 
+   - These internal metrics may be dropped or change name without prior notice
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

connector/datadogconnector/factory.go

@@ -13,6 +13,7 @@ import (
 	"go.opentelemetry.io/collector/consumer"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/connector/datadogconnector/internal/metadata"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog"
 )
 
 // NewFactory creates a factory for tailtracer connector.
@@ -37,6 +38,7 @@ func createDefaultConfig() component.Config {
 // defines the consumer type of the connector
 // we want to consume traces and export metrics therefore define nextConsumer as metrics, consumer is the next component in the pipeline
 func createTracesToMetricsConnector(_ context.Context, params connector.CreateSettings, cfg component.Config, nextConsumer consumer.Metrics) (connector.Traces, error) {
+	initializeMetricsClient(params)
 	c, err := newTraceToMetricConnector(params.TelemetrySettings, cfg, nextConsumer)
 	if err != nil {
 		return nil, err
@@ -45,5 +47,10 @@ func createTracesToMetricsConnector(_ context.Context, params connector.CreateSe
 }
 
 func createTracesToTracesConnector(_ context.Context, params connector.CreateSettings, _ component.Config, nextConsumer consumer.Traces) (connector.Traces, error) {
+	initializeMetricsClient(params)
 	return newTraceToTraceConnector(params.Logger, nextConsumer), nil
 }
+
+func initializeMetricsClient(params connector.CreateSettings) {
+	datadog.InitializeMetricClient(params.MeterProvider)
+}

exporter/datadogexporter/factory.go

@@ -139,6 +139,7 @@ func (f *factory) StopReporter() {
 }
 
 func (f *factory) TraceAgent(ctx context.Context, params exporter.CreateSettings, cfg *Config, sourceProvider source.Provider) (*agent.Agent, error) {
+	datadog.InitializeMetricClient(params.MeterProvider)
 	agnt, err := newTraceAgent(ctx, params, cfg, sourceProvider)
 	if err != nil {
 		return nil, err

internal/datadog/go.mod

@@ -9,6 +9,9 @@ require (
 	github.com/stretchr/testify v1.8.4
 	go.opentelemetry.io/collector/featuregate v1.1.0
 	go.opentelemetry.io/collector/pdata v1.1.0
+	go.opentelemetry.io/otel v1.23.0
+	go.opentelemetry.io/otel/metric v1.23.0
+	go.opentelemetry.io/otel/sdk/metric v1.23.0
 )
 
 require (
@@ -32,6 +35,8 @@ require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect
 	github.com/godbus/dbus/v5 v5.0.6 // indirect
@@ -69,8 +74,7 @@ require (
 	go.opentelemetry.io/collector/config/configtelemetry v0.94.1 // indirect
 	go.opentelemetry.io/collector/confmap v0.94.1 // indirect
 	go.opentelemetry.io/collector/semconv v0.94.1 // indirect
-	go.opentelemetry.io/otel v1.23.0 // indirect
-	go.opentelemetry.io/otel/metric v1.23.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.23.0 // indirect
 	go.opentelemetry.io/otel/trace v1.23.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect

internal/datadog/metrics_client.go

@@ -0,0 +1,98 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package datadog // import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog"
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/DataDog/datadog-agent/pkg/trace/metrics"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+)
+
+type metricsClient struct {
+	meter  metric.Meter
+	gauges map[string]float64
+	mutex  sync.Mutex
+}
+
+var initializeOnce sync.Once
+
+// InitializeMetricClient using a meter provider. If the client has already been initialized,
+// this function just returns the previous one.
+func InitializeMetricClient(mp metric.MeterProvider) metrics.StatsClient {
+	initializeOnce.Do(func() {
+		m := &metricsClient{
+			meter:  mp.Meter("datadog"),
+			gauges: make(map[string]float64),
+		}
+		metrics.Client = m
+	})
+	return metrics.Client
+}
+
+func (m *metricsClient) Gauge(name string, value float64, tags []string, _ float64) error {
+	// The last parameter is rate, but we're omitting it because rate does not have effect for gauge points: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/dedd44436ae064f5a0b43769d24adf897533957b/receiver/statsdreceiver/internal/protocol/metric_translator.go#L153-L156
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	if _, ok := m.gauges[name]; ok {
+		m.gauges[name] = value
+		return nil
+	}
+	m.gauges[name] = value
+	_, err := m.meter.Float64ObservableGauge(name, metric.WithFloat64Callback(func(_ context.Context, f metric.Float64Observer) error {
+		attr := attributeFromTags(tags)
+		if v, ok := m.gauges[name]; ok {
+			f.Observe(v, metric.WithAttributeSet(attr))
+		}
+		return nil
+	}))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *metricsClient) Count(name string, value int64, tags []string, _ float64) error {
+	counter, err := m.meter.Int64Counter(name)
+	if err != nil {
+		return err
+	}
+	attr := attributeFromTags(tags)
+	counter.Add(context.Background(), value, metric.WithAttributeSet(attr))
+	return nil
+}
+
+func attributeFromTags(tags []string) attribute.Set {
+	attr := make([]attribute.KeyValue, 0, len(tags))
+	for _, t := range tags {
+		kv := strings.Split(t, ":")
+		attr = append(attr, attribute.KeyValue{
+			Key:   attribute.Key(kv[0]),
+			Value: attribute.StringValue(kv[1]),
+		})
+	}
+	return attribute.NewSet(attr...)
+}
+
+func (m *metricsClient) Histogram(name string, value float64, tags []string, _ float64) error {
+	hist, err := m.meter.Float64Histogram(name)
+	if err != nil {
+		return err
+	}
+	attr := attributeFromTags(tags)
+	hist.Record(context.Background(), value, metric.WithAttributeSet(attr))
+	return nil
+}
+
+func (m *metricsClient) Timing(name string, value time.Duration, tags []string, rate float64) error {
+	return m.Histogram(name, float64(value.Milliseconds()), tags, rate)
+}
+
+func (m *metricsClient) Flush() error {
+	return nil
+}