diff --git a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md index 8168dd9e04..9e4e725830 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +* Fixed a race condition happening in simultaneous OpenMetrics and PlainText + requests which could cause malformed response + ([#5517](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5517)) + ## 1.8.0-rc.1 Released 2024-Mar-27 diff --git a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusExporterMiddleware.cs b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusExporterMiddleware.cs index 7e815c3e18..da9db13b19 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusExporterMiddleware.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusExporterMiddleware.cs @@ -53,11 +53,13 @@ public async Task InvokeAsync(HttpContext httpContext) try { var openMetricsRequested = AcceptsOpenMetrics(httpContext.Request); - var collectionResponse = await this.exporter.CollectionManager.EnterCollect(openMetricsRequested).ConfigureAwait(false); + var collectionResponse = await this.exporter.CollectionManager.EnterCollect().Response.ConfigureAwait(false); try { - if (collectionResponse.View.Count > 0) + var dataView = openMetricsRequested ? collectionResponse.OpenMetricsView : collectionResponse.PlainTextView; + + if (dataView.Count > 0) { response.StatusCode = 200; #if NET8_0_OR_GREATER @@ -69,7 +71,7 @@ public async Task InvokeAsync(HttpContext httpContext) ? "application/openmetrics-text; version=1.0.0; charset=utf-8" : "text/plain; charset=utf-8; version=0.0.4"; - await response.Body.WriteAsync(collectionResponse.View.Array, 0, collectionResponse.View.Count).ConfigureAwait(false); + await response.Body.WriteAsync(dataView.Array, 0, dataView.Count).ConfigureAwait(false); } else { @@ -91,8 +93,6 @@ public async Task InvokeAsync(HttpContext httpContext) response.StatusCode = 500; } } - - this.exporter.OnExport = null; } private static bool AcceptsOpenMetrics(HttpRequest request) diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md index ba522fc60a..22a81bb2df 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +* Fixed a race condition happening in simultaneous OpenMetrics and PlainText + requests which could cause malformed response + ([#5517](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5517)) + ## 1.8.0-rc.1 Released 2024-Mar-27 diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusCollectionManager.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusCollectionManager.cs index 27ab845164..30444efff0 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusCollectionManager.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusCollectionManager.cs @@ -16,14 +16,14 @@ internal sealed class PrometheusCollectionManager private readonly Dictionary metricsCache; private readonly HashSet scopes; private int metricsCacheCount; - private byte[] buffer = new byte[85000]; // encourage the object to live in LOH (large object heap) + private byte[] plainTextBuffer = new byte[85000]; // encourage the object to live in LOH (large object heap) + private byte[] openMetricsBuffer = new byte[85000]; // encourage the object to live in LOH (large object heap) private int targetInfoBufferLength = -1; // zero or positive when target_info has been written for the first time - private int globalLockState; - private ArraySegment previousDataView; - private DateTime? previousDataViewGeneratedAtUtc; - private int readerCount; - private bool collectionRunning; - private TaskCompletionSource collectionTcs; + private ArraySegment previousPlainTextDataView; + private ArraySegment previousOpenMetricsDataView; + private volatile int globalLockState; + private volatile int readerCount; + private volatile TaskCompletionSource collectionTcs; public PrometheusCollectionManager(PrometheusExporter exporter) { @@ -32,85 +32,65 @@ public PrometheusCollectionManager(PrometheusExporter exporter) this.onCollectRef = this.OnCollect; this.metricsCache = new Dictionary(); this.scopes = new HashSet(); + + this.collectionTcs = new TaskCompletionSource(); + this.collectionTcs.SetResult(default); } #if NET6_0_OR_GREATER - public ValueTask EnterCollect(bool openMetricsRequested) + public (ValueTask Response, bool FromCache) EnterCollect() #else - public Task EnterCollect(bool openMetricsRequested) + public (Task Response, bool FromCache) EnterCollect() #endif { this.EnterGlobalLock(); + var tcs = this.collectionTcs; - // If we are within {ScrapeResponseCacheDurationMilliseconds} of the - // last successful collect, return the previous view. - if (this.previousDataViewGeneratedAtUtc.HasValue - && this.scrapeResponseCacheDurationMilliseconds > 0 - && this.previousDataViewGeneratedAtUtc.Value.AddMilliseconds(this.scrapeResponseCacheDurationMilliseconds) >= DateTime.UtcNow) + if (tcs.Task.IsCompleted) { - Interlocked.Increment(ref this.readerCount); - this.ExitGlobalLock(); + if (this.scrapeResponseCacheDurationMilliseconds > 0 + && tcs.Task.Result.GeneratedAtUtc.AddMilliseconds(this.scrapeResponseCacheDurationMilliseconds) >= DateTime.UtcNow) + { + Interlocked.Increment(ref this.readerCount); + this.ExitGlobalLock(); #if NET6_0_OR_GREATER - return new ValueTask(new CollectionResponse(this.previousDataView, this.previousDataViewGeneratedAtUtc.Value, fromCache: true)); + return (new ValueTask(tcs.Task), true); #else - return Task.FromResult(new CollectionResponse(this.previousDataView, this.previousDataViewGeneratedAtUtc.Value, fromCache: true)); + return (tcs.Task, true); #endif + } } - - // If a collection is already running, return a task to wait on the result. - if (this.collectionRunning) + else { - if (this.collectionTcs == null) - { - this.collectionTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - } - Interlocked.Increment(ref this.readerCount); this.ExitGlobalLock(); #if NET6_0_OR_GREATER - return new ValueTask(this.collectionTcs.Task); + return (new ValueTask(tcs.Task), false); #else - return this.collectionTcs.Task; + return (tcs.Task, false); #endif } this.WaitForReadersToComplete(); - - // Start a collection on the current thread. - this.collectionRunning = true; - this.previousDataViewGeneratedAtUtc = null; Interlocked.Increment(ref this.readerCount); - this.ExitGlobalLock(); - - CollectionResponse response; - var result = this.ExecuteCollect(openMetricsRequested); - if (result) - { - this.previousDataViewGeneratedAtUtc = DateTime.UtcNow; - response = new CollectionResponse(this.previousDataView, this.previousDataViewGeneratedAtUtc.Value, fromCache: false); - } - else - { - response = default; - } - - this.EnterGlobalLock(); - - this.collectionRunning = false; + var newTcs = new TaskCompletionSource(); + SpinWait spinWait = default; - if (this.collectionTcs != null) + while (true) { - this.collectionTcs.SetResult(response); - this.collectionTcs = null; - } - - this.ExitGlobalLock(); - + if (Interlocked.CompareExchange(ref this.collectionTcs, newTcs, tcs) == tcs) + { + Task.Factory.StartNew(this.ExecuteCollect, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); + this.ExitGlobalLock(); #if NET6_0_OR_GREATER - return new ValueTask(response); + return (new ValueTask(newTcs.Task), false); #else - return Task.FromResult(response); + return (newTcs.Task, false); #endif + } + + spinWait.SpinOnce(); + } } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -147,7 +127,7 @@ private void WaitForReadersToComplete() SpinWait readWait = default; while (true) { - if (Interlocked.CompareExchange(ref this.readerCount, 0, this.readerCount) != 0) + if (Interlocked.CompareExchange(ref this.readerCount, 0, 0) != 0) { readWait.SpinOnce(); continue; @@ -157,63 +137,77 @@ private void WaitForReadersToComplete() } } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool ExecuteCollect(bool openMetricsRequested) + private void ExecuteCollect() { this.exporter.OnExport = this.onCollectRef; - this.exporter.OpenMetricsRequested = openMetricsRequested; var result = this.exporter.Collect(Timeout.Infinite); this.exporter.OnExport = null; - return result; + + CollectionResponse response; + + if (result) + { + response = new CollectionResponse(this.previousOpenMetricsDataView, this.previousPlainTextDataView, DateTime.UtcNow); + } + else + { + response = default; + } + + // Set the result and notify all waiting readers. + // We are not calling `tcs.TrySetResult(response)` directly here + // because we don't want any continuation to be run inlined on the current thread. + var tcs = this.collectionTcs; + Task.Factory.StartNew(s => ((TaskCompletionSource)s!).TrySetResult(response), tcs, CancellationToken.None, TaskCreationOptions.PreferFairness, TaskScheduler.Default); + tcs.Task.Wait(); } private ExportResult OnCollect(Batch metrics) { - var cursor = 0; - try { - if (this.exporter.OpenMetricsRequested) - { - cursor = this.WriteTargetInfo(); + var openMetricsCursor = this.WriteTargetInfo(); + var plainTextCursor = 0; - this.scopes.Clear(); + this.scopes.Clear(); - foreach (var metric in metrics) + foreach (var metric in metrics) + { + if (!PrometheusSerializer.CanWriteMetric(metric)) { - if (!PrometheusSerializer.CanWriteMetric(metric)) - { - continue; - } + continue; + } - if (this.scopes.Add(metric.MeterName)) + if (this.scopes.Add(metric.MeterName)) + { + while (true) { - while (true) + try { - try - { - cursor = PrometheusSerializer.WriteScopeInfo(this.buffer, cursor, metric.MeterName); + openMetricsCursor = PrometheusSerializer.WriteScopeInfo(this.openMetricsBuffer, openMetricsCursor, metric.MeterName); - break; - } - catch (IndexOutOfRangeException) + break; + } + catch (IndexOutOfRangeException) + { + if (!this.IncreaseBufferSize(ref this.openMetricsBuffer)) { - if (!this.IncreaseBufferSize()) - { - // there are two cases we might run into the following condition: - // 1. we have many metrics to be exported - in this case we probably want - // to put some upper limit and allow the user to configure it. - // 2. we got an IndexOutOfRangeException which was triggered by some other - // code instead of the buffer[cursor++] - in this case we should give up - // at certain point rather than allocating like crazy. - throw; - } + // there are two cases we might run into the following condition: + // 1. we have many metrics to be exported - in this case we probably want + // to put some upper limit and allow the user to configure it. + // 2. we got an IndexOutOfRangeException which was triggered by some other + // code instead of the buffer[cursor++] - in this case we should give up + // at certain point rather than allocating like crazy. + throw; } } } } } + // TODO: caching the response based on the request type on demand, + // instead of always caching two responses regardless the request type + foreach (var metric in metrics) { if (!PrometheusSerializer.CanWriteMetric(metric)) @@ -221,51 +215,92 @@ private ExportResult OnCollect(Batch metrics) continue; } + var prometheusMetric = this.GetPrometheusMetric(metric); + while (true) { try { - cursor = PrometheusSerializer.WriteMetric( - this.buffer, - cursor, + openMetricsCursor = PrometheusSerializer.WriteMetric( + this.openMetricsBuffer, + openMetricsCursor, metric, - this.GetPrometheusMetric(metric), - this.exporter.OpenMetricsRequested); + prometheusMetric, + true); break; } catch (IndexOutOfRangeException) { - if (!this.IncreaseBufferSize()) + if (!this.IncreaseBufferSize(ref this.openMetricsBuffer)) { throw; } } } + + while (true) + { + try + { + plainTextCursor = PrometheusSerializer.WriteMetric( + this.plainTextBuffer, + plainTextCursor, + metric, + prometheusMetric, + false); + + break; + } + catch (IndexOutOfRangeException) + { + if (!this.IncreaseBufferSize(ref this.plainTextBuffer)) + { + throw; + } + } + } + } + + while (true) + { + try + { + openMetricsCursor = PrometheusSerializer.WriteEof(this.openMetricsBuffer, openMetricsCursor); + break; + } + catch (IndexOutOfRangeException) + { + if (!this.IncreaseBufferSize(ref this.openMetricsBuffer)) + { + throw; + } + } } while (true) { try { - cursor = PrometheusSerializer.WriteEof(this.buffer, cursor); + plainTextCursor = PrometheusSerializer.WriteEof(this.plainTextBuffer, plainTextCursor); break; } catch (IndexOutOfRangeException) { - if (!this.IncreaseBufferSize()) + if (!this.IncreaseBufferSize(ref this.plainTextBuffer)) { throw; } } } - this.previousDataView = new ArraySegment(this.buffer, 0, cursor); + this.previousOpenMetricsDataView = new ArraySegment(this.openMetricsBuffer, 0, openMetricsCursor); + this.previousPlainTextDataView = new ArraySegment(this.plainTextBuffer, 0, plainTextCursor); return ExportResult.Success; } catch (Exception) { - this.previousDataView = new ArraySegment(Array.Empty(), 0, 0); + this.previousOpenMetricsDataView = this.previousPlainTextDataView = new ArraySegment(Array.Empty(), 0, 0); return ExportResult.Failure; } } @@ -278,13 +313,13 @@ private int WriteTargetInfo() { try { - this.targetInfoBufferLength = PrometheusSerializer.WriteTargetInfo(this.buffer, 0, this.exporter.Resource); + this.targetInfoBufferLength = PrometheusSerializer.WriteTargetInfo(this.openMetricsBuffer, 0, this.exporter.Resource); break; } catch (IndexOutOfRangeException) { - if (!this.IncreaseBufferSize()) + if (!this.IncreaseBufferSize(ref this.openMetricsBuffer)) { throw; } @@ -295,9 +330,9 @@ private int WriteTargetInfo() return this.targetInfoBufferLength; } - private bool IncreaseBufferSize() + private bool IncreaseBufferSize(ref byte[] buffer) { - var newBufferSize = this.buffer.Length * 2; + var newBufferSize = buffer.Length * 2; if (newBufferSize > 100 * 1024 * 1024) { @@ -305,8 +340,8 @@ private bool IncreaseBufferSize() } var newBuffer = new byte[newBufferSize]; - this.buffer.CopyTo(newBuffer, 0); - this.buffer = newBuffer; + buffer.CopyTo(newBuffer, 0); + buffer = newBuffer; return true; } @@ -331,17 +366,17 @@ private PrometheusMetric GetPrometheusMetric(Metric metric) public readonly struct CollectionResponse { - public CollectionResponse(ArraySegment view, DateTime generatedAtUtc, bool fromCache) + public CollectionResponse(ArraySegment openMetricsView, ArraySegment plainTextView, DateTime generatedAtUtc) { - this.View = view; + this.OpenMetricsView = openMetricsView; + this.PlainTextView = plainTextView; this.GeneratedAtUtc = generatedAtUtc; - this.FromCache = fromCache; } - public ArraySegment View { get; } + public ArraySegment OpenMetricsView { get; } - public DateTime GeneratedAtUtc { get; } + public ArraySegment PlainTextView { get; } - public bool FromCache { get; } + public DateTime GeneratedAtUtc { get; } } } diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListener.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListener.cs index 6f9663ba42..ed68a640c3 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListener.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListener.cs @@ -148,12 +148,13 @@ private async Task ProcessRequestAsync(HttpListenerContext context) try { var openMetricsRequested = AcceptsOpenMetrics(context.Request); - var collectionResponse = await this.exporter.CollectionManager.EnterCollect(openMetricsRequested).ConfigureAwait(false); + var collectionResponse = await this.exporter.CollectionManager.EnterCollect().Response.ConfigureAwait(false); try { context.Response.Headers.Add("Server", string.Empty); - if (collectionResponse.View.Count > 0) + var dataView = openMetricsRequested ? collectionResponse.OpenMetricsView : collectionResponse.PlainTextView; + if (dataView.Count > 0) { context.Response.StatusCode = 200; context.Response.Headers.Add("Last-Modified", collectionResponse.GeneratedAtUtc.ToString("R")); @@ -161,7 +162,7 @@ private async Task ProcessRequestAsync(HttpListenerContext context) ? "application/openmetrics-text; version=1.0.0; charset=utf-8" : "text/plain; charset=utf-8; version=0.0.4"; - await context.Response.OutputStream.WriteAsync(collectionResponse.View.Array, 0, collectionResponse.View.Count).ConfigureAwait(false); + await context.Response.OutputStream.WriteAsync(dataView.Array, 0, dataView.Count).ConfigureAwait(false); } else { diff --git a/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs b/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs index 46da01e641..c460d02e71 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs @@ -248,6 +248,48 @@ public Task PrometheusExporterMiddlewareIntegration_UseOpenMetricsVersionHeader( acceptHeader: "application/openmetrics-text; version=1.0.0"); } + [Fact] + public async Task PrometheusExporterMiddlewareIntegration_MultipleRequestsOfDifferentFormatsInParallel() + { + using var host = await StartTestHostAsync( + app => app.UseOpenTelemetryPrometheusScrapingEndpoint()); + + var tags = new KeyValuePair[] + { + new KeyValuePair("key1", "value1"), + new KeyValuePair("key2", "value2"), + }; + + using var meter = new Meter(MeterName, MeterVersion); + + var beginTimestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds(); + + var counter = meter.CreateCounter("counter_double"); + counter.Add(100.18D, tags); + counter.Add(0.99D, tags); + + bool[] requestOpenMetricsTestCases = Enumerable.Repeat([true, false, false, true], 5000000).SelectMany(i => i).ToArray(); + using var client = host.GetTestClient(); + + await Parallel.ForEachAsync(requestOpenMetricsTestCases, async (requestOpenMetrics, _) => + { + using var request = new HttpRequestMessage + { + Headers = { { "Accept", requestOpenMetrics ? "application/openmetrics-text" : "text/plain" } }, + RequestUri = new Uri("/metrics", UriKind.Relative), + Method = HttpMethod.Get, + }; + + using var response = await client.SendAsync(request); + + var endTimestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds(); + + await VerifyAsync(beginTimestamp, endTimestamp, response, requestOpenMetrics); + }); + + await host.StopAsync(); + } + private static async Task RunPrometheusExporterMiddlewareIntegrationTest( string path, Action configure, @@ -260,26 +302,7 @@ public Task PrometheusExporterMiddlewareIntegration_UseOpenMetricsVersionHeader( { var requestOpenMetrics = acceptHeader.StartsWith("application/openmetrics-text"); - using var host = await new HostBuilder() - .ConfigureWebHost(webBuilder => webBuilder - .UseTestServer() - .ConfigureServices(services => - { - if (registerMeterProvider) - { - services.AddOpenTelemetry().WithMetrics(builder => builder - .ConfigureResource(x => x.Clear().AddService("my_service", serviceInstanceId: "id1")) - .AddMeter(MeterName) - .AddPrometheusExporter(o => - { - configureOptions?.Invoke(o); - })); - } - - configureServices?.Invoke(services); - }) - .Configure(configure)) - .StartAsync(); + using var host = await StartTestHostAsync(configure, configureServices, registerMeterProvider, configureOptions); var tags = new KeyValuePair[] { @@ -310,51 +333,90 @@ public Task PrometheusExporterMiddlewareIntegration_UseOpenMetricsVersionHeader( var endTimestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds(); if (!skipMetrics) + { + await VerifyAsync(beginTimestamp, endTimestamp, response, requestOpenMetrics); + } + else { Assert.Equal(HttpStatusCode.OK, response.StatusCode); - Assert.True(response.Content.Headers.Contains("Last-Modified")); - - if (requestOpenMetrics) - { - Assert.Equal("application/openmetrics-text; version=1.0.0; charset=utf-8", response.Content.Headers.ContentType.ToString()); - } - else - { - Assert.Equal("text/plain; charset=utf-8; version=0.0.4", response.Content.Headers.ContentType.ToString()); - } - - string content = await response.Content.ReadAsStringAsync(); - - string expected = requestOpenMetrics - ? "# TYPE target info\n" - + "# HELP target Target metadata\n" - + "target_info{service_name='my_service',service_instance_id='id1'} 1\n" - + "# TYPE otel_scope_info info\n" - + "# HELP otel_scope_info Scope metadata\n" - + $"otel_scope_info{{otel_scope_name='{MeterName}'}} 1\n" - + "# TYPE counter_double_total counter\n" - + $"counter_double_total{{otel_scope_name='{MeterName}',otel_scope_version='{MeterVersion}',key1='value1',key2='value2'}} 101.17 (\\d+\\.\\d{{3}})\n" - + "# EOF\n" - : "# TYPE counter_double_total counter\n" - + $"counter_double_total{{otel_scope_name='{MeterName}',otel_scope_version='{MeterVersion}',key1='value1',key2='value2'}} 101.17 (\\d+)\n" - + "# EOF\n"; + } - var matches = Regex.Matches(content, ("^" + expected + "$").Replace('\'', '"')); + validateResponse?.Invoke(response); - Assert.Single(matches); + await host.StopAsync(); + } - var timestamp = long.Parse(matches[0].Groups[1].Value.Replace(".", string.Empty)); + private static async Task VerifyAsync(long beginTimestamp, long endTimestamp, HttpResponseMessage response, bool requestOpenMetrics) + { + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.True(response.Content.Headers.Contains("Last-Modified")); - Assert.True(beginTimestamp <= timestamp && timestamp <= endTimestamp); + if (requestOpenMetrics) + { + Assert.Equal("application/openmetrics-text; version=1.0.0; charset=utf-8", response.Content.Headers.ContentType.ToString()); } else { - Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("text/plain; charset=utf-8; version=0.0.4", response.Content.Headers.ContentType.ToString()); } - validateResponse?.Invoke(response); + string content = (await response.Content.ReadAsStringAsync()).ReplaceLineEndings(); - await host.StopAsync(); + string expected = requestOpenMetrics + ? $$""" + # TYPE target info + # HELP target Target metadata + target_info{service_name="my_service",service_instance_id="id1"} 1 + # TYPE otel_scope_info info + # HELP otel_scope_info Scope metadata + otel_scope_info{otel_scope_name="{{MeterName}}"} 1 + # TYPE counter_double_total counter + counter_double_total{otel_scope_name="{{MeterName}}",otel_scope_version="{{MeterVersion}}",key1="value1",key2="value2"} 101.17 (\d+\.\d{3}) + # EOF + + """.ReplaceLineEndings() + : $$""" + # TYPE counter_double_total counter + counter_double_total{otel_scope_name="{{MeterName}}",otel_scope_version="{{MeterVersion}}",key1="value1",key2="value2"} 101.17 (\d+) + # EOF + + """.ReplaceLineEndings(); + + var matches = Regex.Matches(content, "^" + expected + "$"); + + Assert.True(matches.Count == 1, content); + + var timestamp = long.Parse(matches[0].Groups[1].Value.Replace(".", string.Empty)); + + Assert.True(beginTimestamp <= timestamp && timestamp <= endTimestamp, $"{beginTimestamp} {timestamp} {endTimestamp}"); + } + + private static Task StartTestHostAsync( + Action configure, + Action configureServices = null, + bool registerMeterProvider = true, + Action configureOptions = null) + { + return new HostBuilder() + .ConfigureWebHost(webBuilder => webBuilder + .UseTestServer() + .ConfigureServices(services => + { + if (registerMeterProvider) + { + services.AddOpenTelemetry().WithMetrics(builder => builder + .ConfigureResource(x => x.Clear().AddService("my_service", serviceInstanceId: "id1")) + .AddMeter(MeterName) + .AddPrometheusExporter(o => + { + configureOptions?.Invoke(o); + })); + } + + configureServices?.Invoke(services); + }) + .Configure(configure)) + .StartAsync(); } } #endif diff --git a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusCollectionManagerTests.cs b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusCollectionManagerTests.cs index f4cadff53b..fafbb19647 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusCollectionManagerTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusCollectionManagerTests.cs @@ -49,19 +49,20 @@ public async Task EnterExitCollectTest(int scrapeResponseCacheDurationMillisecon var counter = meter.CreateCounter("counter_int", description: "Prometheus help text goes here \n escaping."); counter.Add(100); - Task[] collectTasks = new Task[10]; + Task<(Response Response, bool FromCache)>[] collectTasks = new Task<(Response, bool)>[10]; for (int i = 0; i < collectTasks.Length; i++) { collectTasks[i] = Task.Run(async () => { - var response = await exporter.CollectionManager.EnterCollect(openMetricsRequested); + var result = exporter.CollectionManager.EnterCollect(); + var response = await result.Response; try { - return new Response + return (new Response { CollectionResponse = response, - ViewPayload = response.View.ToArray(), - }; + ViewPayload = openMetricsRequested ? response.OpenMetricsView.ToArray() : response.PlainTextView.ToArray(), + }, result.FromCache); } finally { @@ -76,33 +77,35 @@ public async Task EnterExitCollectTest(int scrapeResponseCacheDurationMillisecon var firstResponse = await collectTasks[0]; - Assert.False(firstResponse.CollectionResponse.FromCache); + Assert.False(firstResponse.FromCache); for (int i = 1; i < collectTasks.Length; i++) { - Assert.Equal(firstResponse.ViewPayload, (await collectTasks[i]).ViewPayload); - Assert.Equal(firstResponse.CollectionResponse.GeneratedAtUtc, (await collectTasks[i]).CollectionResponse.GeneratedAtUtc); + Assert.Equal(firstResponse.Response.ViewPayload, (await collectTasks[i]).Response.ViewPayload); + Assert.Equal(firstResponse.Response.CollectionResponse.GeneratedAtUtc, (await collectTasks[i]).Response.CollectionResponse.GeneratedAtUtc); } counter.Add(100); // This should use the cache and ignore the second counter update. - var task = exporter.CollectionManager.EnterCollect(openMetricsRequested); - Assert.True(task.IsCompleted); - var response = await task; + var result = exporter.CollectionManager.EnterCollect(); + Assert.Equal(cacheEnabled, result.Response.IsCompleted); + Assert.Equal(cacheEnabled, result.FromCache); + + var response = await result.Response; try { if (cacheEnabled) { Assert.Equal(1, runningCollectCount); - Assert.True(response.FromCache); - Assert.Equal(firstResponse.CollectionResponse.GeneratedAtUtc, response.GeneratedAtUtc); + Assert.True(result.FromCache); + Assert.Equal(firstResponse.Response.CollectionResponse.GeneratedAtUtc, response.GeneratedAtUtc); } else { Assert.Equal(2, runningCollectCount); - Assert.False(response.FromCache); - Assert.True(firstResponse.CollectionResponse.GeneratedAtUtc < response.GeneratedAtUtc); + Assert.False(result.FromCache); + Assert.True(firstResponse.Response.CollectionResponse.GeneratedAtUtc < response.GeneratedAtUtc); } } finally @@ -118,14 +121,15 @@ public async Task EnterExitCollectTest(int scrapeResponseCacheDurationMillisecon { collectTasks[i] = Task.Run(async () => { - var response = await exporter.CollectionManager.EnterCollect(openMetricsRequested); + var result = exporter.CollectionManager.EnterCollect(); + var response = await result.Response; try { - return new Response + return (new Response { CollectionResponse = response, - ViewPayload = response.View.ToArray(), - }; + ViewPayload = openMetricsRequested ? response.OpenMetricsView.ToArray() : response.PlainTextView.ToArray(), + }, result.FromCache); } finally { @@ -137,17 +141,17 @@ public async Task EnterExitCollectTest(int scrapeResponseCacheDurationMillisecon await Task.WhenAll(collectTasks); Assert.Equal(cacheEnabled ? 2 : 3, runningCollectCount); - Assert.NotEqual(firstResponse.ViewPayload, (await collectTasks[0]).ViewPayload); - Assert.NotEqual(firstResponse.CollectionResponse.GeneratedAtUtc, (await collectTasks[0]).CollectionResponse.GeneratedAtUtc); + Assert.NotEqual(firstResponse.Response.ViewPayload, (await collectTasks[0]).Response.ViewPayload); + Assert.NotEqual(firstResponse.Response.CollectionResponse.GeneratedAtUtc, (await collectTasks[0]).Response.CollectionResponse.GeneratedAtUtc); firstResponse = await collectTasks[0]; - Assert.False(firstResponse.CollectionResponse.FromCache); + Assert.False(firstResponse.FromCache); for (int i = 1; i < collectTasks.Length; i++) { - Assert.Equal(firstResponse.ViewPayload, (await collectTasks[i]).ViewPayload); - Assert.Equal(firstResponse.CollectionResponse.GeneratedAtUtc, (await collectTasks[i]).CollectionResponse.GeneratedAtUtc); + Assert.Equal(firstResponse.Response.ViewPayload, (await collectTasks[i]).Response.ViewPayload); + Assert.Equal(firstResponse.Response.CollectionResponse.GeneratedAtUtc, (await collectTasks[i]).Response.CollectionResponse.GeneratedAtUtc); } } }