diff --git a/internal/pkg/otel/samples/linux/profiles_metrics.yml b/internal/pkg/otel/samples/linux/profiles_metrics.yml index be01ec3bda6..499a7a8c90b 100644 --- a/internal/pkg/otel/samples/linux/profiles_metrics.yml +++ b/internal/pkg/otel/samples/linux/profiles_metrics.yml @@ -1,6 +1,6 @@ receivers: profiling: - SamplesPerSecond: 19 + samples_per_second: 19 connectors: profilingmetrics: diff --git a/internal/pkg/otel/samples/windows/logs_metrics_traces.yml b/internal/pkg/otel/samples/windows/logs_metrics_traces.yml index 5f8d6c856dc..94899677071 100644 --- a/internal/pkg/otel/samples/windows/logs_metrics_traces.yml +++ b/internal/pkg/otel/samples/windows/logs_metrics_traces.yml @@ -36,10 +36,6 @@ receivers: grpc: http: -extensions: - file_storage: - directory: ${env:STORAGE_DIR} - connectors: elasticapm: @@ -85,7 +81,6 @@ exporters: mode: otel service: - extensions: [file_storage] pipelines: traces/fromsdk: receivers: [otlp/fromsdk] diff --git a/internal/pkg/otel/samples/windows/managed_otlp/logs_metrics_traces.yml b/internal/pkg/otel/samples/windows/managed_otlp/logs_metrics_traces.yml new file mode 100644 index 00000000000..3a3d773cbf7 --- /dev/null +++ b/internal/pkg/otel/samples/windows/managed_otlp/logs_metrics_traces.yml @@ -0,0 +1,124 @@ +receivers: + windowseventlog: + channel: application + + # Receiver for CPU, Disk, Memory, and Filesystem metrics + hostmetrics/system: + collection_interval: 60s + scrapers: + filesystem: + memory: + metrics: + system.memory.utilization: + enabled: true + # process scraper is disabled for now: https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/39423 + #process: + # mute_process_exe_error: true + # mute_process_io_error: true + # mute_process_user_error: true + # metrics: + # process.threads: + # enabled: true + # process.open_file_descriptors: + # enabled: true + # process.memory.utilization: + # enabled: true + # process.disk.operations: + # enabled: true + network: + processes: + load: + + # Receiver for logs, traces, and metrics from SDKs + otlp/fromsdk: + protocols: + grpc: + http: + +processors: + resourcedetection: + detectors: ["system"] + system: + hostname_sources: ["os"] + resource_attributes: + host.name: + enabled: true + host.id: + enabled: false + host.arch: + enabled: true + host.ip: + enabled: true + host.mac: + enabled: true + host.cpu.vendor.id: + enabled: true + host.cpu.family: + enabled: true + host.cpu.model.id: + enabled: true + host.cpu.model.name: + enabled: true + host.cpu.stepping: + enabled: true + host.cpu.cache.l2.size: + enabled: true + os.description: + enabled: true + os.type: + enabled: true + +exporters: + otlp/ingest_logs: + endpoint: ${env:ELASTIC_OTLP_ENDPOINT} + headers: + Authorization: ApiKey ${env:ELASTIC_API_KEY} + sending_queue: + enabled: true + sizer: bytes + queue_size: 50000000 # 50MB uncompressed + block_on_overflow: true + batch: + flush_timeout: 1s + min_size: 1_000_000 # 1MB uncompressed + max_size: 4_000_000 # 4MB uncompressed + otlp/ingest_metrics_traces: + endpoint: ${env:ELASTIC_OTLP_ENDPOINT} + headers: + Authorization: ApiKey ${env:ELASTIC_API_KEY} + sending_queue: + enabled: true + sizer: bytes + queue_size: 50000000 # 50MB uncompressed + block_on_overflow: true + batch: + flush_timeout: 1s + min_size: 1_000_000 # 1MB uncompressed + max_size: 4_000_000 # 4MB uncompressed + +service: + pipelines: + traces/fromsdk: + receivers: [otlp/fromsdk] + processors: [] + exporters: [otlp/ingest_metrics_traces] + + metrics/fromsdk: + receivers: [otlp/fromsdk] + processors: [] + exporters: [otlp/ingest_metrics_traces] + + logs/fromsdk: + receivers: [otlp/fromsdk] + processors: [] + exporters: [otlp/ingest_logs] + + metrics/hostmetrics: + receivers: [hostmetrics/system] + processors: [resourcedetection] + exporters: [otlp/ingest_metrics_traces] + + logs/platformlogs: + receivers: [windowseventlog] + processors: [resourcedetection] + exporters: [otlp/ingest_logs] diff --git a/internal/pkg/otel/samples/windows/managed_otlp/platformlogs.yml b/internal/pkg/otel/samples/windows/managed_otlp/platformlogs.yml new file mode 100644 index 00000000000..7e46d926661 --- /dev/null +++ b/internal/pkg/otel/samples/windows/managed_otlp/platformlogs.yml @@ -0,0 +1,65 @@ +receivers: + windowseventlog: + channel: application + +processors: + resourcedetection: + detectors: ["system"] + system: + hostname_sources: ["os"] + resource_attributes: + host.name: + enabled: true + host.id: + enabled: false + host.arch: + enabled: true + host.ip: + enabled: true + host.mac: + enabled: true + host.cpu.vendor.id: + enabled: true + host.cpu.family: + enabled: true + host.cpu.model.id: + enabled: true + host.cpu.model.name: + enabled: true + host.cpu.stepping: + enabled: true + host.cpu.cache.l2.size: + enabled: true + os.description: + enabled: true + os.type: + enabled: true + +exporters: + # Exporter to print the first 5 logs/metrics and then every 1000th + debug: + verbosity: detailed + sampling_initial: 5 + sampling_thereafter: 1000 + +# Exporter to send logs and metrics to Elasticsearch Managed OTLP Input + otlp/ingest: + endpoint: ${env:ELASTIC_OTLP_ENDPOINT} + headers: + Authorization: ApiKey ${env:ELASTIC_API_KEY} + sending_queue: + enabled: true + sizer: bytes + queue_size: 50000000 # 50MB uncompressed + block_on_overflow: true + batch: + flush_timeout: 1s + min_size: 1_000_000 # 1MB uncompressed + max_size: 4_000_000 # 4MB uncompressed + +service: + pipelines: + logs/platformlogs: + receivers: [windowseventlog] + processors: [resourcedetection] + exporters: [debug, otlp/ingest] diff --git a/internal/pkg/otel/samples/windows/managed_otlp/platformlogs_hostmetrics.yml b/internal/pkg/otel/samples/windows/managed_otlp/platformlogs_hostmetrics.yml new file mode 100644 index 00000000000..e9ecb5672b2 --- /dev/null +++ b/internal/pkg/otel/samples/windows/managed_otlp/platformlogs_hostmetrics.yml @@ -0,0 +1,90 @@ +receivers: + windowseventlog: + channel: application + + # Receiver for CPU, Disk, Memory, and Filesystem metrics + hostmetrics/system: + collection_interval: 60s + scrapers: + filesystem: + memory: + metrics: + system.memory.utilization: + enabled: true + # process scraper is disabled for now: https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/39423 + #process: + # mute_process_exe_error: true + # mute_process_io_error: true + # mute_process_user_error: true + # metrics: + # process.threads: + # enabled: true + # process.open_file_descriptors: + # enabled: true + # process.memory.utilization: + # enabled: true + # process.disk.operations: + # enabled: true + network: + processes: + load: + +processors: + resourcedetection: + detectors: ["system"] + system: + hostname_sources: ["os"] + resource_attributes: + host.name: + enabled: true + host.id: + enabled: false + host.arch: + enabled: true + host.ip: + enabled: true + host.mac: + enabled: true + host.cpu.vendor.id: + enabled: true + host.cpu.family: + enabled: true + host.cpu.model.id: + enabled: true + host.cpu.model.name: + enabled: true + host.cpu.stepping: + enabled: true + host.cpu.cache.l2.size: + enabled: true + os.description: + enabled: true + os.type: + enabled: true + +exporters: + # Exporter to send logs and metrics to Elasticsearch Managed OTLP Input + otlp/ingest: + endpoint: ${env:ELASTIC_OTLP_ENDPOINT} + headers: + Authorization: ApiKey ${env:ELASTIC_API_KEY} + sending_queue: + enabled: true + sizer: bytes + queue_size: 50000000 # 50MB uncompressed + block_on_overflow: true + batch: + flush_timeout: 1s + min_size: 1_000_000 # 1MB uncompressed + max_size: 4_000_000 # 4MB uncompressed + +service: + pipelines: + metrics/hostmetrics: + receivers: [hostmetrics/system] + processors: [resourcedetection] + exporters: [otlp/ingest] + logs/platformlogs: + receivers: [windowseventlog] + processors: [resourcedetection] + exporters: [otlp/ingest] diff --git a/internal/pkg/otel/samples_test.go b/internal/pkg/otel/samples_test.go new file mode 100644 index 00000000000..4e45226df02 --- /dev/null +++ b/internal/pkg/otel/samples_test.go @@ -0,0 +1,63 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package otel + +import ( + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/featuregate" + "go.opentelemetry.io/collector/otelcol" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func TestSamples(t *testing.T) { + // Set environment variables required by the sample configurations. + // These are just sample values to allow the collector to start. + t.Setenv("STORAGE_DIR", t.TempDir()) + t.Setenv("ELASTIC_ENDPOINT", "http://localhost:9200") + t.Setenv("ELASTIC_API_KEY", "test_api_key") + t.Setenv("ELASTIC_OTLP_ENDPOINT", "http://localhost:4318") + t.Setenv("AUTOOPS_ES_URL", "http://localhost:9200") + t.Setenv("AUTOOPS_TOKEN", "token") + t.Setenv("AUTOOPS_TEMP_RESOURCE_ID", "temp") + t.Setenv("AUTOOPS_OTEL_URL", "http://localhost:4318") + + // Enable service.profilesSupport featuregate to test the profiling samples. + assert.NoError(t, featuregate.GlobalRegistry().Set("service.profilesSupport", true)) + defer func() { + assert.NoError(t, featuregate.GlobalRegistry().Set("service.profilesSupport", false)) + }() + + err := filepath.WalkDir(filepath.Join(".", "samples", runtime.GOOS), func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && (filepath.Ext(d.Name()) == ".yaml" || filepath.Ext(d.Name()) == ".yml") { + t.Run(d.Name(), func(t *testing.T) { + testSample(t, filepath.Join(".", "samples", runtime.GOOS, d.Name())) + }) + } + return nil + }) + assert.NoError(t, err) +} + +func testSample(t *testing.T, configFile string) { + settings := NewSettings("test", []string{configFile}) + settings.LoggingOptions = []zap.Option{zap.WrapCore(func(zapcore.Core) zapcore.Core { + // TODO: Replace with observer core. Right now, it results in a race condition. + return zapcore.NewNopCore() + })} + collector, err := otelcol.NewCollector(*settings) + assert.NoError(t, err) + assert.NotNil(t, collector) + assert.NoError(t, collector.DryRun(t.Context())) + collector.Shutdown() +}