Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(agent): add metrics to agent workers (#3674)
* wip * adding basic metrics for workers * adding better timeouts for github actions * fixing agent tests * update scheduled jobs timeout
- Loading branch information
1 parent
d60b884
commit ed7540c
Showing
10 changed files
with
235 additions
and
27 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| package telemetry | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "os" | ||
|
|
||
| "github.com/kubeshop/tracetest/server/version" | ||
| "go.opentelemetry.io/otel/sdk/resource" | ||
| semconv "go.opentelemetry.io/otel/semconv/v1.21.0" | ||
| ) | ||
|
|
||
| func getAgentServiceName(serviceName string) string { | ||
| return fmt.Sprintf("tracetest.agent-%s", serviceName) | ||
| } | ||
|
|
||
| func getResource(serviceName string) (*resource.Resource, error) { | ||
| hostname, err := os.Hostname() | ||
| if err != nil { | ||
| return nil, fmt.Errorf("could not get OS hostname: %w", err) | ||
| } | ||
|
|
||
| resource, err := resource.Merge( | ||
| resource.Default(), | ||
| resource.NewWithAttributes( | ||
| semconv.SchemaURL, | ||
| semconv.ServiceNameKey.String(serviceName), | ||
| semconv.HostName(hostname), | ||
| semconv.ServiceVersion(version.Version), // TODO: should we consider a version file for the agent? | ||
| ), | ||
| ) | ||
|
|
||
| if err != nil { | ||
| return nil, fmt.Errorf("could not merge resources: %w", err) | ||
| } | ||
|
|
||
| return resource, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| package telemetry | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "time" | ||
|
|
||
| "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" | ||
| "go.opentelemetry.io/otel/metric" | ||
| "go.opentelemetry.io/otel/metric/noop" | ||
| metricsdk "go.opentelemetry.io/otel/sdk/metric" | ||
| ) | ||
|
|
||
| const ( | ||
| metricsReaderInterval = 30 * time.Second | ||
| metricExporterTimeout = 5 * time.Second | ||
| ) | ||
|
|
||
| func GetNoopMeter() metric.Meter { | ||
| return noop.NewMeterProvider().Meter("noop") | ||
| } | ||
|
|
||
| func GetMeter(ctx context.Context, otelExporterEndpoint, serviceName string) (metric.Meter, error) { | ||
| if otelExporterEndpoint == "" { | ||
| // fallback, return noop | ||
| return GetNoopMeter(), nil | ||
| } | ||
|
|
||
| realServiceName := getAgentServiceName(serviceName) | ||
|
|
||
| provider, err := newMeterProvider(ctx, otelExporterEndpoint, realServiceName) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("could not create meter provider: %w", err) | ||
| } | ||
|
|
||
| return provider.Meter("tracetest.agent"), nil | ||
| } | ||
|
|
||
| func newMeterProvider(ctx context.Context, otelExporterEndpoint, serviceName string) (metric.MeterProvider, error) { | ||
| resource, err := getResource(serviceName) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("could not get resource: %w", err) | ||
| } | ||
|
|
||
| exporter, err := getMetricExporter(ctx, otelExporterEndpoint) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("could not create metric exporter: %w", err) | ||
| } | ||
|
|
||
| periodicReader := metricsdk.NewPeriodicReader( | ||
| exporter, | ||
| metricsdk.WithInterval(metricsReaderInterval), | ||
| ) | ||
|
|
||
| provider := metricsdk.NewMeterProvider( | ||
| metricsdk.WithResource(resource), | ||
| metricsdk.WithReader(periodicReader), | ||
| ) | ||
|
|
||
| return provider, nil | ||
| } | ||
|
|
||
| func getMetricExporter(ctx context.Context, otelExporterEndpoint string) (*otlpmetricgrpc.Exporter, error) { | ||
| ctx, cancel := context.WithTimeout(ctx, metricExporterTimeout) | ||
| defer cancel() | ||
|
|
||
| exporter, err := otlpmetricgrpc.New(ctx, | ||
| otlpmetricgrpc.WithEndpoint(otelExporterEndpoint), | ||
| otlpmetricgrpc.WithCompressor("gzip"), | ||
| otlpmetricgrpc.WithInsecure(), | ||
| ) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("could not create metric exporter: %w", err) | ||
| } | ||
|
|
||
| return exporter, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.