From 698f24bc0896432ffd531f72192f9f4d83a23021 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Fri, 14 Nov 2025 15:22:31 +0100 Subject: [PATCH 1/4] Fix OTel histogram metrics --- .../Extensions.cs | 10 +++- .../Synchronization/AwsS3SyncApplyStrategy.cs | 56 ++++++++++--------- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/src/Elastic.Documentation.ServiceDefaults/Extensions.cs b/src/Elastic.Documentation.ServiceDefaults/Extensions.cs index 3ba8f44e4..12df1d813 100644 --- a/src/Elastic.Documentation.ServiceDefaults/Extensions.cs +++ b/src/Elastic.Documentation.ServiceDefaults/Extensions.cs @@ -80,7 +80,15 @@ private static TBuilder AddOpenTelemetryExporters(this TBuilder builde var useOtlpExporter = !string.IsNullOrWhiteSpace(builder.Configuration["OTEL_EXPORTER_OTLP_ENDPOINT"]); if (useOtlpExporter) - _ = builder.Services.AddOpenTelemetry().UseOtlpExporter(); + { + _ = builder.Services.AddOpenTelemetry() + .UseOtlpExporter(otlpExporterOptions => + { + // Configure delta temporality for histograms to work with Elasticsearch + // See: https://www.elastic.co/docs/reference/opentelemetry/compatibility/limitations#histograms-in-delta-temporality-only + otlpExporterOptions.MetricTemporalityPreference = MetricReaderTemporalityPreference.Delta; + }); + } // Uncomment the following lines to enable the Azure Monitor exporter (requires the Azure.Monitor.OpenTelemetry.AspNetCore package) //if (!string.IsNullOrEmpty(builder.Configuration["APPLICATIONINSIGHTS_CONNECTION_STRING"])) diff --git a/src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/AwsS3SyncApplyStrategy.cs b/src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/AwsS3SyncApplyStrategy.cs index 2b334c385..3bcd3adb8 100644 --- a/src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/AwsS3SyncApplyStrategy.cs +++ b/src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/AwsS3SyncApplyStrategy.cs @@ -27,33 +27,40 @@ IDiagnosticsCollector collector // Meter for OpenTelemetry metrics private static readonly Meter SyncMeter = new(TelemetryConstants.AssemblerSyncInstrumentationName); - // Deployment-level metrics (low cardinality) - private static readonly Histogram FilesPerDeploymentHistogram = SyncMeter.CreateHistogram( + // Deployment-level metrics (histograms for distribution analysis, counters for totals) + // Note: Histograms require delta temporality to work with Elasticsearch + // See Extensions.cs where MetricTemporalityPreference.Delta is configured + private static readonly Histogram FilesPerDeploymentHistogram = SyncMeter.CreateHistogram( "docs.deployment.files.count", "files", - "Number of files synced per deployment operation"); + "Distribution of files per deployment operation"); - private static readonly Counter FilesAddedCounter = SyncMeter.CreateCounter( + private static readonly Counter FilesTotalCounter = SyncMeter.CreateCounter( + "docs.deployment.files.total", + "files", + "Total number of files in deployment (added + updated + deleted + skipped)"); + + private static readonly Counter FilesAddedCounter = SyncMeter.CreateCounter( "docs.sync.files.added.total", "files", "Total number of files added to S3"); - private static readonly Counter FilesUpdatedCounter = SyncMeter.CreateCounter( + private static readonly Counter FilesUpdatedCounter = SyncMeter.CreateCounter( "docs.sync.files.updated.total", "files", "Total number of files updated in S3"); - private static readonly Counter FilesDeletedCounter = SyncMeter.CreateCounter( + private static readonly Counter FilesDeletedCounter = SyncMeter.CreateCounter( "docs.sync.files.deleted.total", "files", "Total number of files deleted from S3"); - private static readonly Histogram FileSizeHistogram = SyncMeter.CreateHistogram( + private static readonly Histogram FileSizeHistogram = SyncMeter.CreateHistogram( "docs.sync.file.size", "By", "Distribution of file sizes synced to S3"); - private static readonly Counter FilesByExtensionCounter = SyncMeter.CreateCounter( + private static readonly Counter FilesByExtensionCounter = SyncMeter.CreateCounter( "docs.sync.files.by_extension", "files", "File operations grouped by extension"); @@ -61,7 +68,7 @@ IDiagnosticsCollector collector private static readonly Histogram SyncDurationHistogram = SyncMeter.CreateHistogram( "docs.sync.duration", "s", - "Duration of sync operations"); + "Distribution of sync operation durations"); private readonly ILogger _logger = logFactory.CreateLogger(); @@ -95,7 +102,7 @@ public async Task Apply(SyncPlan plan, Cancel ctx = default) var updateCount = plan.UpdateRequests.Count; var deleteCount = plan.DeleteRequests.Count; var skipCount = plan.SkipRequests.Count; - var totalFiles = addCount + updateCount + deleteCount; + var totalFiles = addCount + updateCount + deleteCount + skipCount; // Add aggregate metrics to span _ = applyActivity?.SetTag("docs.sync.files.added", addCount); @@ -105,14 +112,23 @@ public async Task Apply(SyncPlan plan, Cancel ctx = default) _ = applyActivity?.SetTag("docs.sync.files.total", totalFiles); // Record deployment-level metrics (always emit, even if 0) + // Histogram for distribution analysis (p50, p95, p99) FilesPerDeploymentHistogram.Record(totalFiles); - // Always record per-operation counts (even if 0) so metrics show consistent data + // Record per-operation histograms (for distribution analysis by operation type) FilesPerDeploymentHistogram.Record(addCount, [new("operation", "add")]); FilesPerDeploymentHistogram.Record(updateCount, [new("operation", "update")]); FilesPerDeploymentHistogram.Record(deleteCount, [new("operation", "delete")]); FilesPerDeploymentHistogram.Record(skipCount, [new("operation", "skip")]); + // Counter for simple totals and rates + FilesTotalCounter.Add(totalFiles); + + // Record counter versions for easy dashboard queries (always emit, even if 0) + FilesAddedCounter.Add(addCount); + FilesUpdatedCounter.Add(updateCount); + FilesDeletedCounter.Add(deleteCount); + _logger.LogInformation( "Deployment sync: {TotalFiles} files ({AddCount} added, {UpdateCount} updated, {DeleteCount} deleted, {SkipCount} skipped) in {Environment}", totalFiles, addCount, updateCount, deleteCount, skipCount, context.Environment.Name); @@ -120,9 +136,8 @@ public async Task Apply(SyncPlan plan, Cancel ctx = default) await Upload(plan, ctx); await Delete(plan, ctx); - // Record sync duration - SyncDurationHistogram.Record(sw.Elapsed.TotalSeconds, - [new("operation", "sync")]); + // Record sync duration (both histogram for distribution and counter for total) + SyncDurationHistogram.Record(sw.Elapsed.TotalSeconds); } private async Task Upload(SyncPlan plan, Cancel ctx) @@ -147,14 +162,8 @@ private async Task Upload(SyncPlan plan, Cancel ctx) var fileSize = context.WriteFileSystem.FileInfo.New(upload.LocalPath).Length; var extension = Path.GetExtension(upload.DestinationPath).ToLowerInvariant(); - // Record counters - if (operation == "add") - FilesAddedCounter.Add(1); - else - FilesUpdatedCounter.Add(1); - - // Record file size distribution - FileSizeHistogram.Record(fileSize, [new("operation", operation)]); + // Record file size distribution (histogram for p50, p95, p99 analysis) + FileSizeHistogram.Record(fileSize); // Record by extension (low cardinality) if (!string.IsNullOrEmpty(extension)) @@ -222,9 +231,6 @@ private async Task Delete(SyncPlan plan, Cancel ctx) { var extension = Path.GetExtension(delete.DestinationPath).ToLowerInvariant(); - // Record counter - FilesDeletedCounter.Add(1); - // Record by extension (low cardinality) if (!string.IsNullOrEmpty(extension)) { From 7a37789a860a0495ea5bd7bd59db91f4df30b6d0 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Fri, 14 Nov 2025 15:29:29 +0100 Subject: [PATCH 2/4] Also add skipped files counter --- .../Deploying/Synchronization/AwsS3SyncApplyStrategy.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/AwsS3SyncApplyStrategy.cs b/src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/AwsS3SyncApplyStrategy.cs index 3bcd3adb8..d9882958f 100644 --- a/src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/AwsS3SyncApplyStrategy.cs +++ b/src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/AwsS3SyncApplyStrategy.cs @@ -55,6 +55,11 @@ IDiagnosticsCollector collector "files", "Total number of files deleted from S3"); + private static readonly Counter FilesSkippedCounter = SyncMeter.CreateCounter( + "docs.sync.files.skipped.total", + "files", + "Total number of files skipped (unchanged)"); + private static readonly Histogram FileSizeHistogram = SyncMeter.CreateHistogram( "docs.sync.file.size", "By", @@ -128,6 +133,7 @@ public async Task Apply(SyncPlan plan, Cancel ctx = default) FilesAddedCounter.Add(addCount); FilesUpdatedCounter.Add(updateCount); FilesDeletedCounter.Add(deleteCount); + FilesSkippedCounter.Add(skipCount); _logger.LogInformation( "Deployment sync: {TotalFiles} files ({AddCount} added, {UpdateCount} updated, {DeleteCount} deleted, {SkipCount} skipped) in {Environment}", From 79c85bdfe196157fb4f15fbd2a3dee2e8bea780c Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Fri, 14 Nov 2025 15:44:22 +0100 Subject: [PATCH 3/4] Fix descriptions --- .../Deploying/Synchronization/AwsS3SyncApplyStrategy.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/AwsS3SyncApplyStrategy.cs b/src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/AwsS3SyncApplyStrategy.cs index d9882958f..e8ab46b8e 100644 --- a/src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/AwsS3SyncApplyStrategy.cs +++ b/src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/AwsS3SyncApplyStrategy.cs @@ -33,7 +33,7 @@ IDiagnosticsCollector collector private static readonly Histogram FilesPerDeploymentHistogram = SyncMeter.CreateHistogram( "docs.deployment.files.count", "files", - "Distribution of files per deployment operation"); + "Number of files per deployment operation (added + updated + deleted + skipped)"); private static readonly Counter FilesTotalCounter = SyncMeter.CreateCounter( "docs.deployment.files.total", @@ -73,7 +73,7 @@ IDiagnosticsCollector collector private static readonly Histogram SyncDurationHistogram = SyncMeter.CreateHistogram( "docs.sync.duration", "s", - "Distribution of sync operation durations"); + "Duration of sync operations"); private readonly ILogger _logger = logFactory.CreateLogger(); From f8b6327a134ea5ea3decae708a9f81bf5d277efe Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Fri, 14 Nov 2025 15:52:09 +0100 Subject: [PATCH 4/4] Fix --- .../Extensions.cs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Elastic.Documentation.ServiceDefaults/Extensions.cs b/src/Elastic.Documentation.ServiceDefaults/Extensions.cs index 12df1d813..6d415040a 100644 --- a/src/Elastic.Documentation.ServiceDefaults/Extensions.cs +++ b/src/Elastic.Documentation.ServiceDefaults/Extensions.cs @@ -81,13 +81,14 @@ private static TBuilder AddOpenTelemetryExporters(this TBuilder builde if (useOtlpExporter) { - _ = builder.Services.AddOpenTelemetry() - .UseOtlpExporter(otlpExporterOptions => - { - // Configure delta temporality for histograms to work with Elasticsearch - // See: https://www.elastic.co/docs/reference/opentelemetry/compatibility/limitations#histograms-in-delta-temporality-only - otlpExporterOptions.MetricTemporalityPreference = MetricReaderTemporalityPreference.Delta; - }); + // Configure delta temporality for Elasticsearch compatibility + // See: https://www.elastic.co/docs/reference/opentelemetry/compatibility/limitations#histograms-in-delta-temporality-only + _ = builder.Services.Configure(options => + { + options.TemporalityPreference = MetricReaderTemporalityPreference.Delta; + }); + + _ = builder.Services.AddOpenTelemetry().UseOtlpExporter(); } // Uncomment the following lines to enable the Azure Monitor exporter (requires the Azure.Monitor.OpenTelemetry.AspNetCore package)