diff --git a/.github/workflows/publish-nuget-packages.yaml b/.github/workflows/publish-nuget-packages.yaml index 00e1cb3..f9da684 100644 --- a/.github/workflows/publish-nuget-packages.yaml +++ b/.github/workflows/publish-nuget-packages.yaml @@ -4,24 +4,16 @@ on: release: types: [published, prereleased] -jobs: - publish-v2: +jobs: + publish: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - uses: actions/setup-dotnet@v1 with: dotnet-version: '5.0.100' - - run: dotnet pack src/prometheus-net.DotNetRuntime --include-symbols -c "ReleaseV2" --output "build/" - - run: dotnet nuget push "build/prometheus-net.DotNetRuntime.2.*.symbols.nupkg" -k ${{ secrets.NUGET_API_KEY }} -s "https://api.nuget.org/v3/index.json" -n true - - - publish-v3: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - uses: actions/setup-dotnet@v1 - with: - dotnet-version: '5.0.100' - - run: dotnet pack src/prometheus-net.DotNetRuntime --include-symbols -c "ReleaseV3" --output "build/" - - run: dotnet nuget push "build/prometheus-net.DotNetRuntime.3.*.symbols.nupkg" -k ${{ secrets.NUGET_API_KEY }} -s "https://api.nuget.org/v3/index.json" -n true + - run: arrTag=(${GITHUB_REF//\// }) + - run: VERSION="${arrTag[2]}" + - run: echo "Version is $VERSION" + - run: dotnet pack src/prometheus-net.DotNetRuntime --include-symbols -c "Release" -p:PackageVersion=$VERSION --output "build/" + - run: dotnet nuget push "build/prometheus-net.DotNetRuntime.*.symbols.nupkg" -k ${{ secrets.NUGET_API_KEY }} -s "https://api.nuget.org/v3/index.json" -n true diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 6628ace..63b7bf8 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -6,22 +6,17 @@ on: pull_request: jobs: - test-v2: + test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - uses: actions/setup-dotnet@v1 + - name: Setup .NET Core 3.1 + uses: actions/setup-dotnet@v1 + with: + dotnet-version: 3.1.x + - name: Setup .NET Core 5.0 + uses: actions/setup-dotnet@v1 with: - dotnet-version: '5.0.100' - # excluding When_IO_work_is_executed_on_the_thread_pool_then_the_number_of_io_threads_is_measured for now, for some reason we don't seem to be - # generating IO thread events in the github actions environment - - run: dotnet test -c "DebugV2" --filter Name!=When_IO_work_is_executed_on_the_thread_pool_then_the_number_of_io_threads_is_measured - - test-v3: - runs-on: ubuntu-latest - steps: + dotnet-version: 5.0.x - uses: actions/checkout@v1 - - uses: actions/setup-dotnet@v1 - with: - dotnet-version: '5.0.100' - - run: dotnet test -c "DebugV3" --filter Name!=When_IO_work_is_executed_on_the_thread_pool_then_the_number_of_io_threads_is_measured + # This test constantly passes localy (windows + linux) but fails in the test environment. Don't have the time/ inclination to figure out why this is right now.. + - run: dotnet test -c "Debug" --filter Name!=When_blocking_work_is_executed_on_the_thread_pool_then_thread_pool_delays_are_measured diff --git a/README.md b/README.md index 80bd1ba..57584ea 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # prometheus-net.DotNetMetrics -A plugin for the [prometheus-net](https://github.com/prometheus-net/prometheus-net) package, exposing .NET core runtime metrics including: +A plugin for the [prometheus-net](https://github.com/prometheus-net/prometheus-net) package, [exposing .NET core runtime metrics](docs/metrics-exposed.md) including: - Garbage collection collection frequencies and timings by generation/ type, pause timings and GC CPU consumption ratio - Heap size by generation - Bytes allocated by small/ large object heap @@ -8,22 +8,21 @@ A plugin for the [prometheus-net](https://github.com/prometheus-net/prometheus-n - Lock contention - Exceptions thrown, broken down by type -These metrics are essential for understanding the peformance of any non-trivial application. Even if your application is well instrumented, you're only getting half the story- what the runtime is doing completes the picture. +These metrics are essential for understanding the performance of any non-trivial application. Even if your application is well instrumented, you're only getting half the story- what the runtime is doing completes the picture. -## Installation -Supports .NET core v2.2+ but **.NET core v3.0+ is recommended**. There are a [number of bugs present in the .NET core 2.2 runtime](https://github.com/djluck/prometheus-net.DotNetRuntime/issues?q=is%3Aissue+is%3Aopen+label%3A".net+core+2.2+bug") -that can impact metric collection or runtime stability. +## Using this package +### Requirements +- .NET core 3.1 (runtime version 3.1.11+ is recommended)/ .NET 5.0 +- The [prometheus-net](https://github.com/prometheus-net/prometheus-net) package -Add the packge from [nuget](https://www.nuget.org/packages/prometheus-net.DotNetRuntime): +### Install it +The package can be installed from [nuget](https://www.nuget.org/packages/prometheus-net.DotNetRuntime): ```powershell -# If you're using v3.* of prometheus-net dotnet add package prometheus-net.DotNetRuntime - -# If you're using v2.* of prometheus-net -dotnet add package prometheus-net.DotNetRuntime --version 2.2.0 ``` -And then start the collector: +### Start collecting metrics +You can start metric collection with: ```csharp IDisposable collector = DotNetRuntimeStatsBuilder.Default().StartCollecting() ``` @@ -34,7 +33,6 @@ IDisposable collector = DotNetRuntimeStatsBuilder .Customize() .WithContentionStats() .WithJitStats() - .WithThreadPoolSchedulingStats() .WithThreadPoolStats() .WithGcStats() .WithExceptionStats() @@ -42,41 +40,42 @@ IDisposable collector = DotNetRuntimeStatsBuilder ``` Once the collector is registered, you should see metrics prefixed with `dotnet_` visible in your metric output (make sure you are [exporting your metrics](https://github.com/prometheus-net/prometheus-net#http-handler)). -## Sample Grafana dashboard -The metrics exposed can drive a rich dashboard, giving you a graphical insight into the performance of your application ( [exported dashboard available here](examples/NET_runtime_metrics_dashboard.json)): -![Grafana dashboard sample](docs/grafana-example.PNG) -## Performance impact +### Choosing a `CaptureLevel` +By default the library will default generate metrics based on [event counters](https://docs.microsoft.com/en-us/dotnet/core/diagnostics/event-counters). This allows for basic instrumentation of applications with very little performance overhead. + +You can enable higher-fidelity metrics by providing a custom `CaptureLevel`, e.g: +``` +DotNetRuntimeStatsBuilder + .Customize() + .WithGcStats(CaptureLevel.Informational) + .WithExceptionStats(CaptureLevel.Errors) + ... +``` + +Most builder methods allow the passing of a custom `CaptureLevel`- see the [documentation on exposed metrics](docs/metrics-exposed.md) for more information. + +### Performance impact of `CaptureLevel.Errors`+ The harder you work the .NET core runtime, the more events it generates. Event generation and processing costs can stack up, especially around these types of events: - **JIT stats**: each method compiled by the JIT compiler emits two events. Most JIT compilation is performed at startup and depending on the size of your application, this could impact your startup performance. -- **GC stats**: every 100KB of allocations, an event is emitted. If you are consistently allocating memory at a rate > 1GB/sec, you might like to disable GC stats. -- **.NET thread pool scheduling stats**: For every work item scheduled on the thread pool, two events are emitted. If you are scheduling thousands of items per second on the thread pool, you might like to disable scheduling events or decrease the sampling rate of these events. +- **GC stats with `CaptureLevel.Verbose`**: every 100KB of allocations, an event is emitted. If you are consistently allocating memory at a rate > 1GB/sec, you might like to disable GC stats. +- **Exception stats with `CaptureLevel.Errors`**: for every exception throw, an event is generated. -### Sampling -To counteract some of the performance impacts of measuring .NET core runtime events, sampling can be configured on supported collectors: -```csharp -IDisposable collector = DotNetRuntimeStatsBuilder.Customize() - // Only 1 in 10 contention events will be sampled - .WithContentionStats(sampleRate: SampleEvery.TenEvents) - // Only 1 in 100 JIT events will be sampled - .WithJitStats(sampleRate: SampleEvery.HundredEvents) - // Every event will be sampled (disables sampling) - .WithThreadPoolSchedulingStats(sampleRate: SampleEvery.OneEvent) - .StartCollecting(); -``` +There is also a [performance issue present in .NET core 3.1](https://github.com/dotnet/runtime/issues/43985#issuecomment-800629516) that will see CPU consumption grow over time when long-running trace sessions are used. -The default sample rates are listed below: +## Examples +An example `docker-compose` stack is available in the [`examples/`](examples/) folder. Start it with: -| Event collector | Default sample rate | -| ------------------------------ | ------------------------| -| `ThreadPoolSchedulingStats` | `SampleEvery.TenEvents` | -| `JitStats` | `SampleEvery.TenEvents` | -| `ContentionStats` | `SampleEvery.TwoEvents` | +``` +docker-compose up -d +``` + +You can then visit [`http://localhost:3000`](http://localhost:3000) to view metrics being generated by a sample application. -While the default sampling rates provide a decent balance between accuracy and resource consumption if you're concerned with the accuracy of metrics at all costs, -then feel free to change the sampling rate to `SampleEvery.OneEvent`. If minimal resource consumption (especially memory), is your goal you might like to -reduce the sampling rate. +### Grafana dashboard +The metrics exposed can drive a rich dashboard, giving you a graphical insight into the performance of your application ( [exported dashboard available here](examples/grafana/provisioning/dashboards/NET_runtime_metrics_dashboard.json)): +![Grafana dashboard sample](docs/grafana-example.PNG) ## Further reading - The mechanism for listening to runtime events is outlined in the [.NET core 2.2 release notes](https://docs.microsoft.com/en-us/dotnet/core/whats-new/dotnet-core-2-2#core). diff --git a/TODO.txt b/TODO.txt new file mode 100644 index 0000000..b1155d0 --- /dev/null +++ b/TODO.txt @@ -0,0 +1,247 @@ +# Summary for V4 +Main goals of next release: +- As this library is now in greater use, improve the currently poor baseline performance +- Fix longstanding issues (e.g. ThreadPoolStatsCollector doesn't work) +- Make use of all events currently being collected by current level of verbosity +- Keep compatability with .NET core v3 +- Remove support for v2 of prometheus-net +- Fix long-running perf bug + +### v4.0.0 release +- Figure out release process +- Fix build pipeline (no need for v2/ v3 anymore) +- Perf comparison when running default and all between v2 + v3 +- Long running perf test vs v3 +- Review PR +- Update README (note on performance, choosing levels, metrics exposed?, example docker-compose stack, recycling, compatibility with prior configuration) +- Perform metric diff vs v3 + - Use all vs default (to note what metrics are not present by default) + - Will be useful to note what metrics are missing now +- Write release notes (dropped support for prom v2) +- Publish prerelease version + +### v4.1+ release +- DNS events +- HTTP events +- So many events in .net 5! +- Counter-based metrics for contention + JIT +- Review all existing events and look for improvements/ updates made in .NET 5 +- Rethink sampling implmentation (consumption is actually single-threaded, what about circular buffer idea and time is either actual time or estimated max time) +- Review if V3 JIT/ Contention was as wrong as V4. If not, fix later. Otherwise Fix JIT + Contention CPU bug (measurements for time are way off, even with sampling off) + +## Splitting up work +Need to focus on getting event counters in and reducing overhead by default. +1. Reduce overhead + sane defaults +2. Improve documentation (generate documentation automatically?) +3. Improve data collected +4. Dynamic switching +5. Advanced collection? e.g. GCSampledObjectAllocation perhaps? + +## IObservable +- Separate out event producers and metric producers +- EventProducers vs EventConsumers +- EventListeners +- MetricProducers +- IEventProducers +- Metric producerss +- Different observables for different + +## Example GC metric producer +- Focusing on allocation rate +- will consume IObservable, GcEvents.Verbose +- Example impl. for GcEvents.Info: +``` +public class Info{ + public IObservable AlocTick { get; set; } +} +``` + +- Subscribe to both observables +- Enabled/ Disabled is good- allows us to set up metrics correctly + + + +## Improving performance +Main cost involves the .NET runtime producing events, not processing them so we need to be smarter about when we enable the more verbose event sources. Ideas include: +- Using event counters +- Dynamic switching of metric sources + +### Using event counters +Event counters should place a much lower stress on the runtime- using these could definitely help. +See https://docs.microsoft.com/en-us/dotnet/core/diagnostics/event-counters#sample-code. + +Counter implementation concerns: +- Rate is fixed ahead of time (min frequency = 1 sec) +- Counter values are collected separately from other events (need to provide mechanism for other profilers to consume counter values) +- Perhaps event listeners consume counters? + +### Dynamic switching of metric sources +Overall could be three levels of detail for most sources, ordered in terms of perf impact (low -> high): +1. Counters +2. Events (Warning) +2. Events (info) +3. Events (verbose) + +Levels are hierarchial- enabling a more detailed level implies the others are enabled. + +#### Ordering +Could start at a more verbose level and have rules to selectively enable less verbose levels. Or start at the least verbose and move towards more verbose. + +#### Changing verbosity +Reasons to change include: +- A period of time has elapsed +- A counter value has changed +- Number of events being processed has changed + +Depending on the information a user wishes to obtain, switching between these three verbosity levels could be useful. Scenarios include: +- Disabling more detailed collectors by default +- Enabling/ disabling collectors as conditions change (e.g. a lot of exceptions are thrown then disable high-impact collection, enabling detailed thread stats when thread pool queue times increase) + +``` +DotNetRuntimeStatsBuilder.LowestImpact.StartCollecting(); +DotNetRuntimeStatsBuilder.AllCollectors.StartCollecting(); + +// This should be good enough to start, right? +Gc.DefaultLevel(Info) + // Should we allow multiple conditions? This could help solve the problem of different evaluation timeframes + .Use.Verbose.While(x => x.allocRate > x, evalPeriod: 5 second) + .Use.Verbose.While(x => x.startTime < DateTime.Now.AddMinutes(2)) + .Use.Info.While(x => x.EventsSec > 100) + .Use.Counters.While(x => true) + +#### Goals of builder +- Good documentation of what metrics can be collected at each level +- Well-typed- should not be allowed to enable a level that offers no benefit +- Easy to use! + +// Why are we doing this? +// To control performance impact of collectors +// To enable more detail when needed +// TraceInfo, TraceVerbose +// Scenarios: +// JIT: on startup, when a lot of JIT is happening (e.g. num methods > x) +// Contention: when number of locks contended > 5, when number of locks isn't greater than 5 +// GC: when LOH > blah size (enable LOH allocs) +// ThreadPool: When queue length > x, num threads > Environment.ProcessorCount +// Exceptions: When num exceptions> blah + +// Profiles: +// Perf concious- I don't want performance to be destroyed by monitoring +// Detail oriented- I want to have more insight into why my application is degrading +// Balanced- I want more detail as long as it's not impacting the perf of my application + + + +// Levels: Info, Verbose +DotNetRuntimeStatsBuilder.Customize() + .With.Gc + .With.Jit + .With.ThreadPoolStats + .Use.Detailed.When(x => x.Blah) + .Use.Detailed.Always() + .Use.Level(Level.Info) + .Use.Normal.When(x => x.NumEventsSec > 100) + .With.ThreadPoolLatencyStats + +Use.Gc + .And.Jit + .Use.Level.Info.Always // same effect as And + .And.ThreadPoolStats + .EnableLevel.Detailed.When(x => x).DisableWhen(x => x) + .EnableLevel.Info.When(x => x).DisableWhen(x => x) + .And. + + +``` + +We are considering moving from low -> high, what about the other way around? +Start profiling with a lot of detail + +Context: +- Time since app start +- Time since level last enabled +- Rate of events (disable level only) +- Relevant counters +- + +How often will these be evaluated? +- Counter values are evaluated, enabling higher levels of detail +- Profilers are disabled when rate is too high or no interesting events are happening +- Need a control mechanism that says "after enabling/ disabling, do not disable/ enable for x period of time" +- Enable.When().While()For.AtLeast()/AtMost() +- .Default(level) + .Use(level).When(x => x.).Until(x => x.EventsSec > 100) + .Use(level).When(x => x.) + .Use(Counters).When(); +- What do conditions do? + - Check rate/ sec of events (this can only be known while the thingo is active) + - Check event counter values +- How often are they evaluated? +What is the long-term impact of starting and stopping? + + + +IDEAS: +- State change based on repetitive duration of time +- State change based on value of event counters (e.g. bytes jitted > x for y seconds) +- State change based on rate of events received (e.g. > 100/sec, then disable events for x seconds) +- Premade profiles (e.g. perf vs investigation) +- Need to track what collectors are enabled and at what level of verbosity +- Will need to completely redesign the construction of event listeners +- Evaluate every collection (perhaps this will be too long?) +- Counters have to have their refresh frequency specified up front (default to 1 sec?) + +- Counters will be updated at a fixed frequency, we can use this to inform judgments +- We need to take samples of the queue length via histogram +- E.g. thread pool, enable detail after we see a queue build up of y +- Collectors should be ignorant of verbosity changes (managed externally) +- Collectors need to expose additional information (e.g. counter values to base judgments on) + + + +## Collector improvements +Overall: +- Make full use of all events captured by current verbosity levels +- Upgrade to the latest version of events + +### GC +- Track finalizer processing times +- Track mark events (GcMarkWithType) to track the types of roots that hold memory +- Track pinned object heap size (heapstats v3) +- Track allocations more effeciently (don't use Verbose keyword). Can we support this in V3 of .NET core? +- GCHeapSurvivalAndMovementKeyword to track reserved sizes of heaps and positions +- Look into GCGlobalHeapHistory_V3? +- Track compactions? + +### Execeptions +- Track times spent throwing + time spent handling events +- Offer fallback to the count event counter +- No need to use Information by default- can track with Error Level + +### JIT +- _ilBytesJittedCounter to track bytes spent +- Offer to track greater verbosity? + +### RuntimeInformation? + +### [EventSource()] in coreclr +Possible ideas: +- HttpClient (time queued, connection count, etc) +- Dependency injection +- DNS lookups + +Ideas to reduce CPU consumpton: +- don't track JIT on startup +- don't track TP stats unless unhealthy (e.g. too many queued tasks) +- don't track contention stats unless lots of contention +- don't track exceptions unless count is + +For each source of info, offer options to: +- increase verbosity (more detailed log events, e.g. alloc by heap) +- upgrade from event counters -> event traces +- downgrade from event traces -> counters +- disable collectors entirely + +# Collector improvements +## GC +- Collect heap info diff --git a/docs/metrics-exposed.md b/docs/metrics-exposed.md new file mode 100644 index 0000000..2ae8593 --- /dev/null +++ b/docs/metrics-exposed.md @@ -0,0 +1,120 @@ +# Metrics exposed + +A breakdown of all the metrics exposed by this library. Each subheading details the metrics produced by calling builder methods with the specified `CaptureLevel`. + +## Default metrics + +Metrics that are included by default, regardless of what stats collectors are enabled. + +| Name | Type | Description | Labels | +| ------------------- | ------- | ------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | +| `dotnet_build_info` | `Gauge` | Build information about prometheus\-net.DotNetRuntime and the environment | `version`, `target_framework`, `runtime_version`, `os_version`, `process_architecture`, `gc_mode` | +| `process_cpu_count` | `Gauge` | The number of processor cores available to this process. | | + +## `.WithExceptionStats()` + +Include metrics that measure the number of exceptions thrown. + +### `CaptureLevel.Counters` + +| Name | Type | Description | Labels | +| ------------------------- | --------- | -------------------------- | ------ | +| `dotnet_exceptions_total` | `Counter` | Count of exceptions thrown | `type` | + +### `CaptureLevel.Errors` + +Includes metrics generated by `CaptureLevel.Counters` plus: + +| Name | Type | Description | Labels | +| ------------------------- | --------- | ----------------------------------------------- | ------ | +| `dotnet_exceptions_total` | `Counter` | Count of exceptions thrown, broken down by type | | + +## `.WithGcStats()` + +Include metrics recording the frequency and duration of garbage collections\/ pauses, heap sizes and + volume of allocations. + +### `CaptureLevel.Counters` + +| Name | Type | Description | Labels | +| ---------------------------------------- | --------- | ---------------------------------------------------------------------------------------------- | --------------- | +| `dotnet_gc_allocated_bytes_total` | `Counter` | The total number of bytes allocated on the managed heap | | +| `dotnet_gc_memory_total_available_bytes` | `Gauge` | The upper limit on the amount of physical memory .NET can allocate to | | +| `dotnet_gc_pause_ratio` | `Gauge` | The percentage of time the process spent paused for garbage collection | | +| `dotnet_gc_collection_count_total` | `Counter` | Counts the number of garbage collections that have occurred, broken down by generation number. | `gc_generation` | +| `dotnet_gc_heap_size_bytes` | `Gauge` | The current size of all heaps (only updated after a garbage collection) | `gc_generation` | + +### `CaptureLevel.Informational` + +Includes metrics generated by `CaptureLevel.Counters` plus: + +| Name | Type | Description | Labels | +| ------------------------------------- | ----------- | -------------------------------------------------------------------------------------------------------------------------------- | ---------------------------- | +| `dotnet_gc_pause_seconds` | `Histogram` | The amount of time execution was paused for garbage collection | | +| `dotnet_gc_pinned_objects` | `Gauge` | The number of pinned objects | | +| `dotnet_gc_collection_seconds` | `Histogram` | The amount of time spent running garbage collections | `gc_generation`, `gc_type` | +| `dotnet_gc_collection_count_total` | `Counter` | Counts the number of garbage collections that have occurred, broken down by generation number and the reason for the collection. | `gc_generation`, `gc_reason` | +| `dotnet_gc_cpu_ratio` | `Gauge` | The percentage of process CPU time spent running garbage collections | | +| `dotnet_gc_finalization_queue_length` | `Gauge` | The number of objects waiting to be finalized | | + +### `CaptureLevel.Verbose` + +Includes metrics generated by `CaptureLevel.Counters`, `CaptureLevel.Informational` plus: + +| Name | Type | Description | Labels | +| --------------------------------- | --------- | ------------------------------------------------------- | --------- | +| `dotnet_gc_allocated_bytes_total` | `Counter` | The total number of bytes allocated on the managed heap | `gc_heap` | + +## `.WithContentionStats()` + +Include metrics around volume of locks contended. + +### `CaptureLevel.Counters` + +| Name | Type | Description | Labels | +| ------------------------- | --------- | ----------------------------- | ------ | +| `dotnet_contention_total` | `Counter` | The number of locks contended | | + +### `CaptureLevel.Informational` + +Includes metrics generated by `CaptureLevel.Counters` plus: + +| Name | Type | Description | Labels | +| --------------------------------- | --------- | ----------------------------------------------- | ------ | +| `dotnet_contention_seconds_total` | `Counter` | The total amount of time spent contending locks | | + +## `.WithThreadPoolStats()` + +Include metrics around the size of the worker and IO thread pools and reasons + for worker thread pool changes. + +### `CaptureLevel.Counters` + +| Name | Type | Description | Labels | +| ------------------------------------ | ----------- | ----------------------------------------------------------------------------------------------------------------------------- | ------ | +| `dotnet_threadpool_queue_length` | `Histogram` | Measures the queue length of the thread pool. Values greater than 0 indicate a backlog of work for the threadpool to process. | | +| `dotnet_threadpool_throughput_total` | `Counter` | The total number of work items that have finished execution in the thread pool | | +| `dotnet_threadpool_timer_count` | `Gauge` | The number of timers active | | +| `dotnet_threadpool_num_threads` | `Gauge` | The number of active threads in the thread pool | | + +### `CaptureLevel.Informational` + +Includes metrics generated by `CaptureLevel.Counters` plus: + +| Name | Type | Description | Labels | +| ------------------------------------- | --------- | ------------------------------------------------------------------------------------------------- | ------------------- | +| `dotnet_threadpool_io_num_threads` | `Gauge` | The number of active threads in the IO thread pool | | +| `dotnet_threadpool_adjustments_total` | `Counter` | The total number of changes made to the size of the thread pool, labeled by the reason for change | `adjustment_reason` | + +## `.WithJitStats()` + +Include metrics summarizing the volume of methods being compiled + by the Just\-In\-Time compiler. + +### `CaptureLevel.Verbose` + +| Name | Type | Description | Labels | +| --------------------------------- | --------- | ---------------------------------------------------- | --------- | +| `dotnet_jit_method_seconds_total` | `Counter` | Total number of seconds spent in the JIT compiler | `dynamic` | +| `dotnet_jit_cpu_ratio` | `Gauge` | The amount of total CPU time consumed spent JIT'ing | | +| `dotnet_jit_method_total` | `Counter` | Total number of methods compiled by the JIT compiler | `dynamic` | diff --git a/examples/AspNetCoreExample/AspNetCoreExample.csproj b/examples/AspNetCoreExample/AspNetCoreExample.csproj index 65129ed..f51d0fc 100644 --- a/examples/AspNetCoreExample/AspNetCoreExample.csproj +++ b/examples/AspNetCoreExample/AspNetCoreExample.csproj @@ -1,18 +1,15 @@  - - netcoreapp3.0 + netcoreapp3.1 + 8 - - - - - - + + + diff --git a/examples/AspNetCoreExample/Controllers/CollectorController.cs b/examples/AspNetCoreExample/Controllers/CollectorController.cs new file mode 100644 index 0000000..44be734 --- /dev/null +++ b/examples/AspNetCoreExample/Controllers/CollectorController.cs @@ -0,0 +1,44 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using System.Net; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; + +namespace AspNetCoreExample.Controllers +{ + [Route("api/[controller]")] + [ApiController] + public class CollectorController : ControllerBase + { + // GET api/values + [HttpGet] + [Route("enable")] + public async Task Enable() + { + + if (Startup.Collector != null) + return new JsonResult(new { Status = "Failed - already enabled"}) { StatusCode = (int)HttpStatusCode.InternalServerError}; + + Startup.Collector = Startup.CreateCollector(); + + return new JsonResult(new { Status = "Ok- started and assigned collector"}); + } + + [HttpGet] + [Route("disable")] + public async Task Disable() + { + if (Startup.Collector == null) + return new JsonResult(new { Status = "Failed - already disable"}) { StatusCode = (int)HttpStatusCode.InternalServerError}; + + Startup.Collector.Dispose(); + Startup.Collector = null; + + return new JsonResult(new { Status = "Ok- stopped the collector"}); + } + } +} \ No newline at end of file diff --git a/examples/AspNetCoreExample/Controllers/SimulateController.cs b/examples/AspNetCoreExample/Controllers/SimulateController.cs new file mode 100644 index 0000000..eecb2a2 --- /dev/null +++ b/examples/AspNetCoreExample/Controllers/SimulateController.cs @@ -0,0 +1,66 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; + +namespace AspNetCoreExample.Controllers +{ + [Route("api/[controller]")] + [ApiController] + public class SimulateController : ControllerBase + { + [HttpGet] + public async Task>> Get( + bool simulateAlloc = true, + bool simulateJit = true, + bool simulateException = true, + bool simulateBlocking = false) + { + var r = new Random(); + if (simulateAlloc) + { + // assign some SOH memory + var x = new byte[r.Next(1024, 1024 * 64)]; + + // assign some LOH memory + x = new byte[r.Next(1024 * 90, 1024 * 100)]; + } + + // await a task (will result in a Task being scheduled on the thread pool) + await Task.Yield(); + + if (simulateJit) + { + var val = r.Next(); + CompileMe(() => val); + } + + if (simulateException) + { + try + { + var divide = 0; + var result = 1 / divide; + } + catch + { + } + } + + if (simulateBlocking) + { + Thread.Sleep(100); + } + + return new string[] {"value1" + r.Next(), "value2"+ r.Next()}; + } + + private void CompileMe(Expression> func) + { + func.Compile()(); + } + } +} \ No newline at end of file diff --git a/examples/AspNetCoreExample/Controllers/ValuesController.cs b/examples/AspNetCoreExample/Controllers/ValuesController.cs deleted file mode 100644 index 9733798..0000000 --- a/examples/AspNetCoreExample/Controllers/ValuesController.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Linq.Expressions; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Mvc; - -namespace AspNetCoreExample.Controllers -{ - [Route("api/[controller]")] - [ApiController] - public class ValuesController : ControllerBase - { - private Random r = new Random(); - // GET api/values - [HttpGet] - public async Task>> Get() - { - // assign some SOH memory - var x = new byte[1024]; - - // assign some LOH memory - x = new byte[1024 * 100]; - - // await a task (will result in a Task being scheduled on the thread pool) - await Task.Yield(); - - var val = this.r.Next(); - CompileMe(() => val); - - try - { - var divide = 0; - var result = 1 / divide; - } - catch { } - - return new string[] {"value1" + this.r.Next(), "value2"+ this.r.Next()}; - } - - private void CompileMe(Expression> func) - { - func.Compile()(); - } - } -} \ No newline at end of file diff --git a/examples/AspNetCoreExample/Dockerfile b/examples/AspNetCoreExample/Dockerfile new file mode 100644 index 0000000..9056391 --- /dev/null +++ b/examples/AspNetCoreExample/Dockerfile @@ -0,0 +1,13 @@ +# FROM mcr.microsoft.com/dotnet/core/sdk:3.1.201 AS build +FROM mcr.microsoft.com/dotnet/core/sdk:3.1.406 AS build +#FROM mcr.microsoft.com/dotnet/sdk:5.0 AS build +WORKDIR /src +COPY . . +RUN dotnet publish "examples/AspNetCoreExample" -c Release -o /app + +#FROM mcr.microsoft.com/dotnet/core/aspnet:3.1.3 AS final +FROM mcr.microsoft.com/dotnet/core/aspnet:3.1.10 AS final +#FROM mcr.microsoft.com/dotnet/aspnet:5.0 as final +WORKDIR /app +COPY --from=build /app /app +ENTRYPOINT ["dotnet", "AspNetCoreExample.dll"] diff --git a/examples/AspNetCoreExample/Options.cs b/examples/AspNetCoreExample/Options.cs new file mode 100644 index 0000000..7c584b1 --- /dev/null +++ b/examples/AspNetCoreExample/Options.cs @@ -0,0 +1,13 @@ +using System; + +namespace AspNetCoreExample +{ + public class Options + { + public bool EnableMetrics { get; set; } = true; + public bool UseDefaultMetrics { get; set; } = false; + public bool UseDebuggingMetrics { get; set; } = false; + public TimeSpan RecycleEvery { get; set; } = TimeSpan.FromDays(1); + public int? MinThreadPoolSize { get; set; } = null; + } +} \ No newline at end of file diff --git a/examples/AspNetCoreExample/Program.cs b/examples/AspNetCoreExample/Program.cs index a140702..1cf245f 100644 --- a/examples/AspNetCoreExample/Program.cs +++ b/examples/AspNetCoreExample/Program.cs @@ -1,14 +1,7 @@ using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Runtime; -using System.Threading.Tasks; using Microsoft.AspNetCore; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; -using Prometheus; using Prometheus.DotNetRuntime; namespace AspNetCoreExample @@ -17,26 +10,15 @@ public class Program { public static void Main(string[] args) { - if (Environment.GetEnvironmentVariable("NOMON") == null) - { - Console.WriteLine("Enabling prometheus-net.DotNetStats..."); - DotNetRuntimeStatsBuilder.Customize() - .WithThreadPoolSchedulingStats() - .WithContentionStats() - .WithGcStats() - .WithJitStats() - .WithThreadPoolStats() - .WithExceptionStats() - .WithErrorHandler(ex => Console.WriteLine("ERROR: " + ex.ToString())) - //.WithDebuggingMetrics(true); - .StartCollecting(); - } - CreateWebHostBuilder(args).Build().Run(); } - + public static IWebHostBuilder CreateWebHostBuilder(string[] args) => WebHost.CreateDefaultBuilder(args) + .ConfigureAppConfiguration(opts => + { + opts.AddEnvironmentVariables("Example"); + }) .ConfigureKestrel(opts => { opts.AllowSynchronousIO = true; diff --git a/examples/AspNetCoreExample/Properties/launchSettings.json b/examples/AspNetCoreExample/Properties/launchSettings.json index 6fae566..1277cb3 100644 --- a/examples/AspNetCoreExample/Properties/launchSettings.json +++ b/examples/AspNetCoreExample/Properties/launchSettings.json @@ -4,10 +4,12 @@ "AspNetCoreExample": { "commandName": "Project", "launchBrowser": true, - "launchUrl": "api/values", + "launchUrl": "metrics", "applicationUrl": "http://localhost:5000", "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" + "ASPNETCORE_ENVIRONMENT": "Development", + "Example__UseDebuggingMetrics": "true", + "Example__UseDefaultMetrics": "true" } } } diff --git a/examples/AspNetCoreExample/Startup.cs b/examples/AspNetCoreExample/Startup.cs index 0fce17f..350f73c 100644 --- a/examples/AspNetCoreExample/Startup.cs +++ b/examples/AspNetCoreExample/Startup.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; @@ -11,14 +12,36 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Prometheus; +using Prometheus.DotNetRuntime; namespace AspNetCoreExample { public class Startup { - public Startup(IConfiguration configuration) + private static Options _options; + public static IDisposable Collector; + private static ILogger _logger; + + public Startup(IConfiguration configuration, ILogger logger) { Configuration = configuration; + + _options = new Options(); + _logger = logger; + configuration.Bind("Example", _options); + + if (_options.EnableMetrics) + { + Collector = CreateCollector(); + } + else + logger.LogWarning($"prometheus-net.DotNetRuntime was NOT started- {_options.EnableMetrics} was set to false"); + + if (_options.MinThreadPoolSize.HasValue) + { + logger.LogInformation($"Setting minimum thread pool size of {_options.MinThreadPoolSize.Value}"); + ThreadPool.SetMinThreads(_options.MinThreadPoolSize.Value, 1); + } } public IConfiguration Configuration { get; } @@ -26,7 +49,7 @@ public Startup(IConfiguration configuration) // This method gets called by the runtime. Use this method to add services to the container. public void ConfigureServices(IServiceCollection services) { - services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_2); + services.AddMvc(); } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. @@ -36,13 +59,7 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env) { app.UseDeveloperExceptionPage(); } - else - { - // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts. - app.UseHsts(); - } - - app.UseHttpsRedirection(); + app.UseRouting(); app.UseEndpoints(endpoints => @@ -53,5 +70,39 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env) app.UseMetricServer(); } + + public static IDisposable CreateCollector() + { + _logger.LogInformation($"Configuring prometheus-net.DotNetRuntime: will recycle event listeners every {_options.RecycleEvery} ({_options.RecycleEvery.TotalSeconds:N0} seconds)."); + + var builder = DotNetRuntimeStatsBuilder.Default(); + + if (!_options.UseDefaultMetrics) + { + builder = DotNetRuntimeStatsBuilder.Customize() + .WithContentionStats(CaptureLevel.Informational) + .WithGcStats(CaptureLevel.Verbose) + .WithThreadPoolStats(CaptureLevel.Informational) + .WithExceptionStats(CaptureLevel.Errors) + .WithJitStats(); + } + + builder +#if NET5_0 + .RecycleCollectorsEvery(_options.RecycleEvery) +#endif + .WithErrorHandler(ex => _logger.LogError(ex, "Unexpected exception occurred in prometheus-net.DotNetRuntime")); + + if (_options.UseDebuggingMetrics) + { + _logger.LogInformation("Using debugging metrics."); + builder.WithDebuggingMetrics(true); + } + + _logger.LogInformation("Starting prometheus-net.DotNetRuntime..."); + + return builder + .StartCollecting(); + } } } \ No newline at end of file diff --git a/examples/AspNetCoreExample/appsettings.json b/examples/AspNetCoreExample/appsettings.json index def9159..e2590f5 100644 --- a/examples/AspNetCoreExample/appsettings.json +++ b/examples/AspNetCoreExample/appsettings.json @@ -1,7 +1,8 @@ { "Logging": { "LogLevel": { - "Default": "Warning" + "Default": "Warning", + "AspNetCoreExample" : "Information" } }, "AllowedHosts": "*" diff --git a/examples/docker-compose.yml b/examples/docker-compose.yml new file mode 100644 index 0000000..20db619 --- /dev/null +++ b/examples/docker-compose.yml @@ -0,0 +1,83 @@ +# With thanks to https://github.com/stefanprodan/dockprom for this excellent template for providing prom + grafana +version: '2.1' + +networks: + monitor-net: + driver: bridge + +volumes: + prometheus_data: {} + grafana_data: {} + +services: + + prometheus: + image: prom/prometheus:v2.22.0 + container_name: prometheus + volumes: + - ./prometheus:/etc/prometheus + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/etc/prometheus/console_libraries' + - '--web.console.templates=/etc/prometheus/consoles' + - '--storage.tsdb.retention.time=200h' + - '--web.enable-lifecycle' + restart: unless-stopped + expose: + - 9090 + ports: + - 9090:9090 + networks: + - monitor-net + labels: + org.label-schema.group: "monitoring" + + grafana: + image: grafana/grafana:7.3.1 + container_name: grafana + volumes: + - grafana_data:/var/lib/grafana + - ./grafana/provisioning:/etc/grafana/provisioning + environment: + - GF_SECURITY_ADMIN_USER=${ADMIN_USER:-admin} + - GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin} + - GF_USERS_ALLOW_SIGN_UP=false + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Editor + restart: unless-stopped + ports: + - 3000:3000 + networks: + - monitor-net + labels: + org.label-schema.group: "monitoring" + + aspexample: + build: + context: ../ + dockerfile: examples/AspNetCoreExample/Dockerfile + expose: + - 5000 + environment: + - ASPNETCORE_URLS=http://+:5000 + # Additional vars that can be set to tweak behaviour + #- Example__UseDefaultMetrics=true + #- Example__EnableMetrics=false + #- Example__UseDebuggingMetrics=true + #- Example__RecycleEvery=00:10:00 + #- Example__MinThreadPoolSize=100 + ports: + - 5001:5000 + mem_limit: "200M" + networks: + - monitor-net + + bombardier: + image: alpine/bombardier + command: -c 25 -d 1000h -r 100 -t 15s http://aspexample:5000/api/simulate + # High intensity + # command: -c 1000 -d 1000h -r 2000 -t 10s http://aspexample:5000/api/simulate + networks: + - monitor-net \ No newline at end of file diff --git a/examples/grafana/provisioning/dashboards/Debugging_metrics.json b/examples/grafana/provisioning/dashboards/Debugging_metrics.json new file mode 100644 index 0000000..e9f0582 --- /dev/null +++ b/examples/grafana/provisioning/dashboards/Debugging_metrics.json @@ -0,0 +1,952 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Displays internal debugging metrics generated by prometheus-net.DotNetRuntime when WithDebuggingMetrics() has been called at setup.", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 3, + "iteration": 1616199947834, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{instance=\"$instance\"}[$__rate_interval]) / process_cpu_count{instance=\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU use", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate(dotnet_debug_event_count_total{instance=\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{listener_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Events/ sec", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Events/ sec", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(dotnet_debug_event_seconds_total{instance=\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Processing time (s)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Event processing time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "dotnet_debug_publish_thread_count{instance=\"$instance\"}", + "interval": "", + "legendFormat": "{{listener_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Num threads producing events (by listener)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 9, + "panels": [], + "title": "Volume of events", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (listener_name) (rate(dotnet_debug_event_count_total{instance=\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{listener_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Events/ sec (by listener name)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Events/ sec", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (event_source_name) (rate(dotnet_debug_event_count_total{instance=\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{event_source_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Events/ sec (by event source name)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Events/ sec", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(dotnet_debug_event_count_total{instance=\"$instance\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{listener_name}}:{{event_source_name}}/{{event_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Events (full breakdown)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "events/ sec", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 11, + "title": "Event processing time", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (listener_name)(rate(dotnet_debug_event_seconds_total{instance=\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{listener_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Time consumed (by event listener)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (event_source_name)(rate(dotnet_debug_event_seconds_total{instance=\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{event_source_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Time consumed (by event source name)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "text": "host.docker.internal:5000", + "value": "host.docker.internal:5000" + }, + "datasource": "Prometheus", + "definition": "dotnet_build_info", + "error": null, + "hide": 0, + "includeAll": false, + "label": "instance", + "multi": false, + "name": "instance", + "options": [], + "query": "dotnet_build_info", + "refresh": 1, + "regex": "/instance=\"([^\"]+)\"/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Debugging metrics", + "uid": "py__-w8Mk", + "version": 6 +} \ No newline at end of file diff --git a/examples/NET_runtime_metrics_dashboard.json b/examples/grafana/provisioning/dashboards/NET_runtime_metrics_dashboard.json similarity index 66% rename from examples/NET_runtime_metrics_dashboard.json rename to examples/grafana/provisioning/dashboards/NET_runtime_metrics_dashboard.json index 02c3ca7..5e53563 100644 --- a/examples/NET_runtime_metrics_dashboard.json +++ b/examples/grafana/provisioning/dashboards/NET_runtime_metrics_dashboard.json @@ -1,46 +1,4 @@ { - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "6.1.6" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "panel", - "id": "heatmap", - "name": "Heatmap", - "version": "" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "" - } - ], "annotations": { "list": [ { @@ -56,13 +14,14 @@ }, "editable": true, "gnetId": null, - "graphTooltip": 0, - "id": null, - "iteration": 1565080722750, + "graphTooltip": 1, + "id": 1, + "iteration": 1616891650176, "links": [], "panels": [ { "collapsed": false, + "datasource": "Prometheus", "gridPos": { "h": 1, "w": 24, @@ -83,9 +42,15 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "${DS_PROMETHEUS}", + "datasource": "Prometheus", "decimals": 1, "description": "The percentage of available CPU resources consumed.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "format": "percentunit", "gauge": { "maxValue": 100, @@ -132,7 +97,7 @@ "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", - "show": false + "show": true }, "tableColumn": "", "targets": [ @@ -165,9 +130,15 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "${DS_PROMETHEUS}", + "datasource": "Prometheus", "decimals": 1, - "description": "The percentage of time that the application was paused to allow the garbage collector to run.", + "description": "The percentage of available memory resources consumed.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "format": "percentunit", "gauge": { "maxValue": 100, @@ -182,6 +153,96 @@ "x": 4, "y": 1 }, + "id": 49, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "process_working_set_bytes{instance=\"$instance\"} / dotnet_gc_memory_total_available_bytes{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "0.70,0.90", + "title": "% Memory consumption", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "decimals": 1, + "description": "The percentage of time that the application was paused to allow the garbage collector to run.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, "id": 28, "interval": null, "links": [], @@ -214,7 +275,7 @@ "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", - "show": false + "show": true }, "tableColumn": "", "targets": [ @@ -247,9 +308,15 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "${DS_PROMETHEUS}", + "datasource": "Prometheus", "decimals": 1, "description": "The percentage of total process CPU utilization that was dedicated to performing garbage collections.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "format": "percentunit", "gauge": { "maxValue": 100, @@ -261,7 +328,7 @@ "gridPos": { "h": 4, "w": 4, - "x": 8, + "x": 12, "y": 1 }, "id": 42, @@ -296,7 +363,7 @@ "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", - "show": false + "show": true }, "tableColumn": "", "targets": [ @@ -329,9 +396,105 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "${DS_PROMETHEUS}", + "datasource": "Prometheus", + "decimals": null, + "description": "The average length of the thread pool's work queue. Values greater than 0 indicate a problem.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 31, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(dotnet_threadpool_queue_length_sum{instance=\"$instance\"}[$__rate_interval])) / \nsum(rate(dotnet_threadpool_queue_length_count{instance=\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "1,100", + "title": "Thread pool queue length", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", "decimals": 1, "description": "The percentage of CPU utilization that was dedicated to compiling methods by the Just In Time (JIT) compiler.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "format": "percentunit", "gauge": { "maxValue": 100, @@ -343,7 +506,7 @@ "gridPos": { "h": 4, "w": 4, - "x": 12, + "x": 20, "y": 1 }, "id": 29, @@ -378,7 +541,7 @@ "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", - "show": false + "show": true }, "tableColumn": "", "targets": [ @@ -411,10 +574,16 @@ "rgba(237, 129, 40, 0.89)", "#d44a3a" ], - "datasource": "${DS_PROMETHEUS}", - "decimals": null, - "description": "The delay between when a job is scheduled for execution on the thread pool and when it starts execution. Measurement taken from the 90th percentile.", - "format": "s", + "datasource": "Prometheus", + "decimals": 1, + "description": "The rate of exceptions being thrown per second", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "short", "gauge": { "maxValue": 100, "minValue": 0, @@ -425,10 +594,10 @@ "gridPos": { "h": 4, "w": 4, - "x": 16, - "y": 1 + "x": 0, + "y": 5 }, - "id": 31, + "id": 51, "interval": null, "links": [], "mappingType": 1, @@ -460,19 +629,21 @@ "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", - "show": false + "show": true }, "tableColumn": "", "targets": [ { - "expr": "sum(rate(dotnet_threadpool_scheduling_delay_seconds_sum{instance=\"$instance\"}[$__interval])) / sum(rate(dotnet_threadpool_scheduling_delay_seconds_count{instance=\"$instance\"}[$__interval]))", + "expr": "sum(rate(dotnet_exceptions_total{instance=\"$instance\"}[$__rate_interval]))", "format": "time_series", + "interval": "", "intervalFactor": 1, + "legendFormat": "", "refId": "A" } ], - "thresholds": "0.001,0.5", - "title": "Thread pool scheduling latency", + "thresholds": "1,100", + "title": "Exceptions/ sec", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -486,11 +657,12 @@ }, { "collapsed": false, + "datasource": "Prometheus", "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 5 + "y": 9 }, "id": 33, "panels": [], @@ -498,18 +670,28 @@ "type": "row" }, { - "aliasColors": {}, + "aliasColors": { + "Total bytes available (memory limit)": "dark-red" + }, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS}", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, - "y": 6 + "y": 10 }, + "hiddenSeries": false, "id": 6, "legend": { "avg": false, @@ -524,28 +706,45 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.3.1", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "Total bytes available (memory limit)", + "dashes": true, + "lines": false + } + ], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "dotnet_gc_heap_size_bytes{instance=\"$instance\"}", + "expr": "process_working_set_bytes{instance=\"$instance\"}", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "gen {{gc_generation}}", + "legendFormat": "# bytes (in physical memory)", "refId": "A" + }, + { + "expr": "dotnet_gc_memory_total_available_bytes{instance=\"$instance\"}", + "interval": "", + "legendFormat": "Total bytes available (memory limit)", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Heap size", + "title": "Memory use", "tooltip": { "shared": true, "sort": 0, @@ -561,7 +760,7 @@ }, "yaxes": [ { - "format": "decbytes", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -584,19 +783,26 @@ }, { "aliasColors": {}, - "bars": true, + "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS}", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, - "y": 6 + "y": 10 }, - "id": 37, - "interval": "", + "hiddenSeries": false, + "id": 50, "legend": { "avg": false, "current": false, @@ -606,11 +812,15 @@ "total": false, "values": false }, - "lines": false, + "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.3.1", "pointradius": 5, "points": false, "renderer": "flot", @@ -620,25 +830,19 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(dotnet_gc_allocated_bytes_total{gc_heap=\"loh\", instance=\"$instance\"}[$__interval])) by (gc_heap)", + "expr": "dotnet_gc_heap_size_bytes{instance=\"$instance\"}", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "Large object heap (objects > 85KB)", + "legendFormat": "gen {{gc_generation}}", "refId": "A" - }, - { - "expr": "sum(rate(dotnet_gc_allocated_bytes_total{gc_heap=\"soh\", instance=\"$instance\"}[$__interval])) by (gc_heap)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Small object heap", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Bytes allocated (by heap)", + "title": "Heap size", "tooltip": { "shared": true, "sort": 0, @@ -680,15 +884,23 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS}", + "datasource": "Prometheus", "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, - "y": 6 + "y": 10 }, + "hiddenSeries": false, "id": 2, "interval": "", "legend": { @@ -704,7 +916,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.3.1", "pointradius": 5, "points": false, "renderer": "flot", @@ -776,14 +992,22 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS}", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, - "y": 14 + "y": 18 }, + "hiddenSeries": false, "id": 13, "interval": "", "legend": { @@ -800,7 +1024,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.3.1", "pointradius": 5, "points": false, "renderer": "flot", @@ -810,11 +1038,11 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(dotnet_gc_collection_seconds_count{instance=\"$instance\"}[$__interval])) by (gc_generation, gc_type)", + "expr": "sum(rate(dotnet_gc_collection_count_total{instance=\"$instance\"}[$__interval])) by (gc_generation, gc_reason)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "gen {{gc_generation}}: {{gc_type}}", + "legendFormat": "gen {{gc_generation}}: {{gc_reason}}", "refId": "A" } ], @@ -860,82 +1088,125 @@ } }, { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateBlues", - "exponent": 0.5, - "mode": "spectrum" + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] }, - "dataFormat": "tsbuckets", - "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, - "y": 14 + "y": 18 }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 11, + "hiddenSeries": false, + "id": 37, + "interval": "", "legend": { - "show": false + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, + "lines": false, + "linewidth": 1, "links": [], - "reverseYBuckets": false, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, "targets": [ { - "expr": "sum(increase(dotnet_gc_collection_seconds_bucket{instance=\"$instance\"}[$__interval])) by (le)", - "format": "heatmap", + "expr": "sum(rate(dotnet_gc_allocated_bytes_total{instance=\"$instance\"}[$__interval])) by (gc_heap)", + "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "{{le}}", + "legendFormat": "Alloc bytes {{gc_heap}}", "refId": "A" } ], + "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "GC duration", + "title": "Bytes allocated (by heap)", "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true + "shared": true, + "sort": 0, + "value_type": "individual" }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "s", - "logBase": 1, - "max": null, - "min": null, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, "show": true, - "splitFactor": null + "values": [] }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS}", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, - "y": 14 + "y": 18 }, + "hiddenSeries": false, "id": 35, "interval": "30s", "legend": { @@ -951,7 +1222,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null as zero", + "options": { + "alertThreshold": false + }, "percentage": false, + "pluginVersion": "7.3.1", "pointradius": 5, "points": false, "renderer": "flot", @@ -990,9 +1265,9 @@ { "format": "s", "label": null, - "logBase": 10, + "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -1009,13 +1284,84 @@ "alignLevel": null } }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateBlues", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 26 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 11, + "legend": { + "show": false + }, + "links": [], + "reverseYBuckets": false, + "targets": [ + { + "expr": "sum(increase(dotnet_gc_collection_seconds_bucket{instance=\"$instance\"}[$__interval])) by (le)", + "format": "heatmap", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "GC duration", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, { "collapsed": false, + "datasource": "Prometheus", "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 22 + "y": 34 }, "id": 23, "panels": [], @@ -1030,17 +1376,24 @@ "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", - "colorScheme": "interpolateOranges", + "colorScheme": "interpolateCool", "exponent": 0.5, "mode": "spectrum" }, "dataFormat": "tsbuckets", - "datasource": "${DS_PROMETHEUS}", + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 8, "x": 0, - "y": 23 + "y": 35 }, "heatmap": {}, "hideZeroBuckets": true, @@ -1053,8 +1406,9 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum by (le) (increase(dotnet_threadpool_scheduling_delay_seconds_bucket{instance=\"$instance\"}[$__interval]))", + "expr": "sum by (le) (increase(dotnet_threadpool_queue_length_bucket{instance=\"$instance\"}[$__rate_interval]))", "format": "heatmap", + "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "refId": "D" @@ -1062,7 +1416,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "Thread pool scheduling latency", + "title": "Thread pool queue length", "tooltip": { "show": true, "showHistogram": false @@ -1075,7 +1429,7 @@ "xBucketSize": null, "yAxis": { "decimals": null, - "format": "s", + "format": "short", "logBase": 1, "max": null, "min": null, @@ -1092,14 +1446,22 @@ "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS}", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, - "y": 23 + "y": 35 }, + "hiddenSeries": false, "id": 21, "legend": { "avg": false, @@ -1114,7 +1476,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.3.1", "pointradius": 2, "points": false, "renderer": "flot", @@ -1126,7 +1492,9 @@ { "expr": "dotnet_threadpool_num_threads{instance=\"$instance\"}", "format": "time_series", + "interval": "", "intervalFactor": 1, + "legendFormat": "{{instance}}", "refId": "A" } ], @@ -1174,16 +1542,123 @@ { "aliasColors": {}, "bars": false, + "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS}", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, - "y": 23 + "y": 35 + }, + "hiddenSeries": false, + "id": 56, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "dotnet_threadpool_timer_count{instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Num thread timers", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 43 }, + "hiddenSeries": false, "id": 19, "interval": "", "legend": { @@ -1199,7 +1674,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.3.1", "pointradius": 5, "points": false, "renderer": "flot", @@ -1259,11 +1738,12 @@ }, { "collapsed": false, + "datasource": "Prometheus", "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 31 + "y": 51 }, "id": 41, "panels": [], @@ -1275,14 +1755,22 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS}", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 52 }, + "hiddenSeries": false, "id": 15, "interval": "", "legend": { @@ -1298,7 +1786,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.3.1", "pointradius": 5, "points": false, "renderer": "flot", @@ -1368,14 +1860,22 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS}", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 32 + "y": 52 }, + "hiddenSeries": false, "id": 43, "interval": "", "legend": { @@ -1391,7 +1891,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.3.1", "pointradius": 5, "points": false, "renderer": "flot", @@ -1451,11 +1955,12 @@ }, { "collapsed": false, + "datasource": "Prometheus", "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 40 + "y": 60 }, "id": 39, "panels": [], @@ -1467,14 +1972,22 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS}", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 41 + "y": 61 }, + "hiddenSeries": false, "id": 44, "interval": "", "legend": { @@ -1490,7 +2003,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.3.1", "pointradius": 5, "points": false, "renderer": "flot", @@ -1553,14 +2070,22 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "${DS_PROMETHEUS}", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 41 + "y": 61 }, + "hiddenSeries": false, "id": 45, "interval": "", "legend": { @@ -1576,7 +2101,11 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, + "pluginVersion": "7.3.1", "pointradius": 5, "points": false, "renderer": "flot", @@ -1633,19 +2162,133 @@ "align": false, "alignLevel": null } + }, + { + "collapsed": false, + "datasource": "Prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 53, + "panels": [], + "title": "Exceptions", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 70 + }, + "hiddenSeries": false, + "id": 55, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(dotnet_exceptions_total{instance=\"$instance\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Exceptions/ sec", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "refresh": false, - "schemaVersion": 18, + "schemaVersion": 26, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, - "current": {}, - "datasource": "${DS_PROMETHEUS}", + "current": { + "selected": true, + "text": "aspexample:5000", + "value": "aspexample:5000" + }, + "datasource": "Prometheus", "definition": "dotnet_build_info", + "error": null, "hide": 0, "includeAll": false, "label": "instance", @@ -1666,8 +2309,8 @@ ] }, "time": { - "from": "2019-08-04T03:15:57.203Z", - "to": "2019-08-04T12:04:02.670Z" + "from": "now-15m", + "to": "now" }, "timepicker": { "refresh_intervals": [ @@ -1697,5 +2340,5 @@ "timezone": "", "title": ".NET runtime metrics", "uid": "RHbwEa8mz", - "version": 17 + "version": 8 } \ No newline at end of file diff --git a/examples/grafana/provisioning/dashboards/dashboard.yml b/examples/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 0000000..d83b43c --- /dev/null +++ b/examples/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: 'Prometheus' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + allowUiUpdates: true + options: + path: /etc/grafana/provisioning/dashboards \ No newline at end of file diff --git a/examples/grafana/provisioning/datasources/datasource.yml b/examples/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 0000000..bb37f13 --- /dev/null +++ b/examples/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 + basicAuth: false + isDefault: true + editable: true \ No newline at end of file diff --git a/examples/prometheus/prometheus.yml b/examples/prometheus/prometheus.yml new file mode 100644 index 0000000..2c17f38 --- /dev/null +++ b/examples/prometheus/prometheus.yml @@ -0,0 +1,18 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + + +# A scrape configuration containing exactly one endpoint to scrape. +scrape_configs: + - job_name: 'dotnetruntime' + scrape_interval: 5s + static_configs: + - targets: ['aspexample:5000', 'host.docker.internal:5000'] + + + - job_name: 'prometheus' + scrape_interval: 10s + static_configs: + - targets: ['localhost:9090'] + diff --git a/prometheus-net.DotNetRuntime.sln b/prometheus-net.DotNetRuntime.sln index d71ce8a..cc4796a 100644 --- a/prometheus-net.DotNetRuntime.sln +++ b/prometheus-net.DotNetRuntime.sln @@ -10,46 +10,39 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AspNetCoreExample", "exampl EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Benchmarks", "src\Benchmarks\Benchmarks.csproj", "{DD607E45-45AD-4F9D-9102-82BD99E49BEC}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{D8594A14-5AC8-40F3-B346-38A266B235E0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DocsGenerator", "tools\DocsGenerator\DocsGenerator.csproj", "{193B461A-49E4-4178-B88C-BA0EF6B4FC55}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution - ReleaseV2|Any CPU = ReleaseV2|Any CPU - DebugV2|Any CPU = DebugV2|Any CPU - DebugV3|Any CPU = DebugV3|Any CPU - ReleaseV3|Any CPU = ReleaseV3|Any CPU + Release|Any CPU = Release|Any CPU + Debug|Any CPU = Debug|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {A40AD08A-53CB-40F3-A6D8-6FFCEC024289}.ReleaseV2|Any CPU.ActiveCfg = ReleaseV2|Any CPU - {A40AD08A-53CB-40F3-A6D8-6FFCEC024289}.ReleaseV2|Any CPU.Build.0 = ReleaseV2|Any CPU - {A40AD08A-53CB-40F3-A6D8-6FFCEC024289}.DebugV2|Any CPU.ActiveCfg = DebugV2|Any CPU - {A40AD08A-53CB-40F3-A6D8-6FFCEC024289}.DebugV2|Any CPU.Build.0 = DebugV2|Any CPU - {A40AD08A-53CB-40F3-A6D8-6FFCEC024289}.DebugV3|Any CPU.ActiveCfg = DebugV3|Any CPU - {A40AD08A-53CB-40F3-A6D8-6FFCEC024289}.DebugV3|Any CPU.Build.0 = DebugV3|Any CPU - {A40AD08A-53CB-40F3-A6D8-6FFCEC024289}.ReleaseV3|Any CPU.ActiveCfg = ReleaseV3|Any CPU - {A40AD08A-53CB-40F3-A6D8-6FFCEC024289}.ReleaseV3|Any CPU.Build.0 = ReleaseV3|Any CPU - {7F4E2E72-5745-4312-B238-CD7B731957B0}.ReleaseV2|Any CPU.ActiveCfg = ReleaseV2|Any CPU - {7F4E2E72-5745-4312-B238-CD7B731957B0}.ReleaseV2|Any CPU.Build.0 = ReleaseV2|Any CPU - {7F4E2E72-5745-4312-B238-CD7B731957B0}.DebugV2|Any CPU.ActiveCfg = DebugV2|Any CPU - {7F4E2E72-5745-4312-B238-CD7B731957B0}.DebugV2|Any CPU.Build.0 = DebugV2|Any CPU - {7F4E2E72-5745-4312-B238-CD7B731957B0}.DebugV3|Any CPU.ActiveCfg = DebugV3|Any CPU - {7F4E2E72-5745-4312-B238-CD7B731957B0}.DebugV3|Any CPU.Build.0 = DebugV3|Any CPU - {7F4E2E72-5745-4312-B238-CD7B731957B0}.ReleaseV3|Any CPU.ActiveCfg = ReleaseV3|Any CPU - {7F4E2E72-5745-4312-B238-CD7B731957B0}.ReleaseV3|Any CPU.Build.0 = ReleaseV3|Any CPU - {D01E9ED3-E35C-4F44-A5AD-5350E43AA636}.DebugV2|Any CPU.ActiveCfg = DebugV2|Any CPU - {D01E9ED3-E35C-4F44-A5AD-5350E43AA636}.DebugV2|Any CPU.Build.0 = DebugV2|Any CPU - {D01E9ED3-E35C-4F44-A5AD-5350E43AA636}.DebugV3|Any CPU.ActiveCfg = DebugV3|Any CPU - {D01E9ED3-E35C-4F44-A5AD-5350E43AA636}.DebugV3|Any CPU.Build.0 = DebugV3|Any CPU - {D01E9ED3-E35C-4F44-A5AD-5350E43AA636}.ReleaseV2|Any CPU.ActiveCfg = ReleaseV2|Any CPU - {D01E9ED3-E35C-4F44-A5AD-5350E43AA636}.ReleaseV2|Any CPU.Build.0 = ReleaseV2|Any CPU - {D01E9ED3-E35C-4F44-A5AD-5350E43AA636}.ReleaseV3|Any CPU.ActiveCfg = ReleaseV3|Any CPU - {D01E9ED3-E35C-4F44-A5AD-5350E43AA636}.ReleaseV3|Any CPU.Build.0 = ReleaseV3|Any CPU - {DD607E45-45AD-4F9D-9102-82BD99E49BEC}.ReleaseV2|Any CPU.ActiveCfg = ReleaseV2|Any CPU - {DD607E45-45AD-4F9D-9102-82BD99E49BEC}.ReleaseV2|Any CPU.Build.0 = ReleaseV2|Any CPU - {DD607E45-45AD-4F9D-9102-82BD99E49BEC}.ReleaseV3|Any CPU.ActiveCfg = ReleaseV3|Any CPU - {DD607E45-45AD-4F9D-9102-82BD99E49BEC}.ReleaseV3|Any CPU.Build.0 = ReleaseV3|Any CPU - {DD607E45-45AD-4F9D-9102-82BD99E49BEC}.DebugV2|Any CPU.ActiveCfg = DebugV2|Any CPU - {DD607E45-45AD-4F9D-9102-82BD99E49BEC}.DebugV3|Any CPU.ActiveCfg = DebugV3|Any CPU + {D01E9ED3-E35C-4F44-A5AD-5350E43AA636}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D01E9ED3-E35C-4F44-A5AD-5350E43AA636}.Release|Any CPU.Build.0 = Release|Any CPU + {D01E9ED3-E35C-4F44-A5AD-5350E43AA636}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D01E9ED3-E35C-4F44-A5AD-5350E43AA636}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DD607E45-45AD-4F9D-9102-82BD99E49BEC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DD607E45-45AD-4F9D-9102-82BD99E49BEC}.Release|Any CPU.Build.0 = Release|Any CPU + {DD607E45-45AD-4F9D-9102-82BD99E49BEC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DD607E45-45AD-4F9D-9102-82BD99E49BEC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A40AD08A-53CB-40F3-A6D8-6FFCEC024289}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A40AD08A-53CB-40F3-A6D8-6FFCEC024289}.Release|Any CPU.Build.0 = Release|Any CPU + {A40AD08A-53CB-40F3-A6D8-6FFCEC024289}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A40AD08A-53CB-40F3-A6D8-6FFCEC024289}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7F4E2E72-5745-4312-B238-CD7B731957B0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7F4E2E72-5745-4312-B238-CD7B731957B0}.Release|Any CPU.Build.0 = Release|Any CPU + {7F4E2E72-5745-4312-B238-CD7B731957B0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7F4E2E72-5745-4312-B238-CD7B731957B0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {193B461A-49E4-4178-B88C-BA0EF6B4FC55}.Release|Any CPU.ActiveCfg = Release|Any CPU + {193B461A-49E4-4178-B88C-BA0EF6B4FC55}.Release|Any CPU.Build.0 = Release|Any CPU + {193B461A-49E4-4178-B88C-BA0EF6B4FC55}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {193B461A-49E4-4178-B88C-BA0EF6B4FC55}.Debug|Any CPU.Build.0 = Debug|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {D01E9ED3-E35C-4F44-A5AD-5350E43AA636} = {31AD912F-A1DC-434A-8C8D-049F4BBD67D4} + {193B461A-49E4-4178-B88C-BA0EF6B4FC55} = {D8594A14-5AC8-40F3-B346-38A266B235E0} EndGlobalSection EndGlobal diff --git a/src/Benchmarks/Benchmarks.csproj b/src/Benchmarks/Benchmarks.csproj index 432f090..853b58f 100644 --- a/src/Benchmarks/Benchmarks.csproj +++ b/src/Benchmarks/Benchmarks.csproj @@ -7,25 +7,27 @@ true + + true + + - - + + + + - + - - - - - + true diff --git a/src/Benchmarks/Benchmarks/DictBenchmark.cs b/src/Benchmarks/Benchmarks/DictBenchmark.cs new file mode 100644 index 0000000..4591f95 --- /dev/null +++ b/src/Benchmarks/Benchmarks/DictBenchmark.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Reactive; +using System.Reactive.Disposables; +using System.Reactive.Linq; +using System.Reactive.Subjects; +using System.Threading; +using System.Threading.Channels; +using BenchmarkDotNet.Attributes; + +namespace Benchmarks.Benchmarks +{ + public class DictBenchmark + { + private Dictionary _dict ; + private ConcurrentDictionary _connDict; + + public DictBenchmark() + { + } + + [IterationSetup] + public void Setup() + { + _dict = new Dictionary(12000); + _connDict = new (concurrencyLevel: Environment.ProcessorCount, 20000); + + } + + [Benchmark] + public void AddToDict() + { + for (int i = 0; i < 10000; i++) + _dict.Add(i, "test value"); + } + + [Benchmark] + public void AddToConcurrentDict() + { + for (int i = 0; i < 10000; i++) + _connDict.TryAdd(i, "test value"); + } + } +} \ No newline at end of file diff --git a/src/Benchmarks/Benchmarks/EventCounterParserBenchmark.cs b/src/Benchmarks/Benchmarks/EventCounterParserBenchmark.cs new file mode 100644 index 0000000..46bc1c9 --- /dev/null +++ b/src/Benchmarks/Benchmarks/EventCounterParserBenchmark.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.ObjectModel; +using System.Diagnostics.Tracing; +using System.Linq; +using BenchmarkDotNet.Attributes; +using Fasterflect; +using Prometheus.DotNetRuntime.EventListening; + +namespace Benchmarks.Benchmarks +{ + public class EventCounterParserBenchmark + { + private EventWrittenEventArgs _meanValue; + private EventWrittenEventArgs _incrCounter; + private DummyTypeEventCounterParser _parser; + + public EventCounterParserBenchmark() + { + _meanValue = CreateCounterEventWrittenEventArgs( + ("Name", "test-mean-counter"), + ("DisplayName", "some value"), + ("Mean", 5.0), + ("StandardDeviation", 1), + ("Count", 1), + ("Min", 1), + ("Max", 1), + ("IntervalSec", 1), + ("Series", 1), + ("CounterType", "Mean"), + ("Metadata", ""), + ("DisplayUnits", "") + ); + + _incrCounter = CreateCounterEventWrittenEventArgs( + ("Name", "test-incrementing-counter"), + ("DisplayName", "some value"), + ("DisplayRateTimeScale", TimeSpan.FromSeconds(10)), + ("Increment", 6.0), + ("IntervalSec", 1), + ("Series", 1), + ("CounterType", "Sum"), + ("Metadata", ""), + ("DisplayUnits", "") + ); + + _parser = new DummyTypeEventCounterParser(); + var total = 0.0; + _parser.TestIncrementingCounter += e => total += e.IncrementedBy; + + var last = 0.0; + _parser.TestMeanCounter += e => last = e.Mean; + } + + [Benchmark] + public void ParseIncrementingCounter() + { + _parser.ProcessEvent(_incrCounter); + } + + [Benchmark] + public void ParseMeanCounter() + { + _parser.ProcessEvent(_meanValue); + } + + public static EventWrittenEventArgs CreateEventWrittenEventArgs(int eventId, DateTime? timestamp = null, params object[] payload) + { + var args = (EventWrittenEventArgs)typeof(EventWrittenEventArgs).CreateInstance(new []{ typeof(EventSource)}, Flags.NonPublic | Flags.Instance, new object[] { null}); + args.SetPropertyValue("EventId", eventId); + args.SetPropertyValue("Payload", new ReadOnlyCollection(payload)); + + if (timestamp.HasValue) + { + args.SetPropertyValue("TimeStamp", timestamp.Value); + } + + return args; + } + + public static EventWrittenEventArgs CreateCounterEventWrittenEventArgs(params (string key, object val)[] payload) + { + var counterPayload = payload.ToDictionary(k => k.key, v => v.val); + + var e = CreateEventWrittenEventArgs(-1, DateTime.UtcNow, new[] { counterPayload }); + e.SetPropertyValue("EventName", "EventCounters"); + return e; + } + + public interface TestCounters : ICounterEvents + { +#pragma disable warning + public event Action TestIncrementingCounter; + public event Action TestMeanCounter; +#pragma warning restore + } + + public class DummyTypeEventCounterParser : EventCounterParserBase, TestCounters + { + public override Guid EventSourceGuid { get; } + public override EventKeywords Keywords { get; } + public override int RefreshIntervalSeconds { get; set; } + + [CounterName("test-incrementing-counter")] + public event Action TestIncrementingCounter; + [CounterName("test-mean-counter")] + public event Action TestMeanCounter; + } + } +} \ No newline at end of file diff --git a/src/Benchmarks/Benchmarks/NoSamplingBenchmark.cs b/src/Benchmarks/Benchmarks/NoSamplingBenchmark.cs deleted file mode 100644 index 24159ad..0000000 --- a/src/Benchmarks/Benchmarks/NoSamplingBenchmark.cs +++ /dev/null @@ -1,16 +0,0 @@ -using BenchmarkDotNet.Attributes; -using Prometheus.DotNetRuntime; - -namespace Benchmarks.Benchmarks -{ - public class NoSamplingBenchmark : DotNetRuntimeStatsBenchmarkBase - { - protected override DotNetRuntimeStatsBuilder.Builder GetStatsBuilder() - { - return DotNetRuntimeStatsBuilder.Default() - .WithThreadPoolSchedulingStats(sampleRate: SampleEvery.OneEvent) - .WithJitStats(SampleEvery.OneEvent) - .WithContentionStats(SampleEvery.OneEvent); - } - } -} \ No newline at end of file diff --git a/src/Benchmarks/Benchmarks/PrometheusMTBenchmark.cs b/src/Benchmarks/Benchmarks/PrometheusMTBenchmark.cs new file mode 100644 index 0000000..a0b5f23 --- /dev/null +++ b/src/Benchmarks/Benchmarks/PrometheusMTBenchmark.cs @@ -0,0 +1,111 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Reports; +using Prometheus; + +namespace Benchmarks.Benchmarks +{ + public class PrometheusMTBenchmark + { + private Counter _counter = Metrics.CreateCounter("test_counter", ""); + private Counter _counterLabelled1 = Metrics.CreateCounter("test_counter_labelled_1", "", "label1"); + private Counter _counterLabelled2 = Metrics.CreateCounter("test_counter_labelled_2", "", "label1", "label2"); + private Histogram _histogram = Metrics.CreateHistogram("test_histo", ""); + private Histogram _histogramLabelled1 = Metrics.CreateHistogram("test_histo_labeled1", "", "label1"); + private Histogram _histogramLabelled2 = Metrics.CreateHistogram("test_histo_labeled2", "", "label1", "label2"); + private CancellationTokenSource _ctSource; + private Task[] _tasks; + + [GlobalSetup] + public void GlobalSetup() + { + _ctSource = new CancellationTokenSource(); + _tasks = Enumerable.Range(1, Environment.ProcessorCount - 4) + .Select(async _ => + { + while (!_ctSource.IsCancellationRequested) + { + for (int i = 0; i < 500_000; i++) + { + IncrementUnlabeled(); + IncrementLabeled1(); + IncrementUnlabeled2(); + ObserveUnlabeled(); + ObserveLabeled1(); + ObserveUnlabeled2(); + ObserveHighUnlabeled(); + ObserveHighLabeled1(); + ObserveHighUnlabeled2(); + } + + await Task.Delay(1); + } + }) + .ToArray(); + } + + [GlobalCleanup] + public void GlobalCleanup() + { + _ctSource.Cancel(); + } + + [Benchmark] + public void IncrementUnlabeled() + { + _counter.Inc(1); + } + + [Benchmark] + public void IncrementLabeled1() + { + _counterLabelled1.WithLabels("test_label1").Inc(1); + } + + [Benchmark] + public void IncrementUnlabeled2() + { + _counterLabelled2.WithLabels("test_label1", "test_label2").Inc(1); + } + + [Benchmark] + public void ObserveUnlabeled() + { + _histogram.Observe(0); + } + + [Benchmark] + public void ObserveLabeled1() + { + _histogramLabelled1.WithLabels("test_label1").Observe(0); + } + + [Benchmark] + public void ObserveUnlabeled2() + { + _histogramLabelled2.WithLabels("test_label1", "test_label2").Observe(0); + } + + [Benchmark] + public void ObserveHighUnlabeled() + { + _histogram.Observe(int.MaxValue); + } + + [Benchmark] + public void ObserveHighLabeled1() + { + _histogramLabelled1.WithLabels("test_label1").Observe(int.MaxValue); + } + + [Benchmark] + public void ObserveHighUnlabeled2() + { + _histogramLabelled2.WithLabels("test_label1", "test_label2").Observe(int.MaxValue); + } + + } +} \ No newline at end of file diff --git a/src/Benchmarks/Benchmarks/PrometheusSTBenchmark.cs b/src/Benchmarks/Benchmarks/PrometheusSTBenchmark.cs new file mode 100644 index 0000000..70ee076 --- /dev/null +++ b/src/Benchmarks/Benchmarks/PrometheusSTBenchmark.cs @@ -0,0 +1,71 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Reports; +using Prometheus; + +namespace Benchmarks.Benchmarks +{ + public class PrometheusSTBenchmark + { + private Counter _counter = Metrics.CreateCounter("test_counter", ""); + private Counter _counterLabelled1 = Metrics.CreateCounter("test_counter_labelled_1", "", "label1"); + private Counter _counterLabelled2 = Metrics.CreateCounter("test_counter_labelled_2", "", "label1", "label2"); + private Histogram _histogram = Metrics.CreateHistogram("test_histo", ""); + private Histogram _histogramLabelled1 = Metrics.CreateHistogram("test_histo_labeled1", "", "label1"); + private Histogram _histogramLabelled2 = Metrics.CreateHistogram("test_histo_labeled2", "", "label1", "label2"); + + [Benchmark] + public void IncrementUnlabeled() + { + _counter.Inc(1); + } + + [Benchmark] + public void IncrementLabeled1() + { + _counterLabelled1.WithLabels("test_label1").Inc(1); + } + + [Benchmark] + public void IncrementUnlabeled2() + { + _counterLabelled2.WithLabels("test_label1", "test_label2").Inc(1); + } + + [Benchmark] + public void ObserveUnlabeled() + { + _histogram.Observe(0); + } + + [Benchmark] + public void ObserveLabeled1() + { + _histogramLabelled1.WithLabels("test_label1").Observe(0); + } + + [Benchmark] + public void ObserveUnlabeled2() + { + _histogramLabelled2.WithLabels("test_label1", "test_label2").Observe(0); + } + + [Benchmark] + public void ObserveHighUnlabeled() + { + _histogram.Observe(int.MaxValue); + } + + [Benchmark] + public void ObserveHighLabeled1() + { + _histogramLabelled1.WithLabels("test_label1").Observe(int.MaxValue); + } + + [Benchmark] + public void ObserveHighUnlabeled2() + { + _histogramLabelled2.WithLabels("test_label1", "test_label2").Observe(int.MaxValue); + } + + } +} \ No newline at end of file diff --git a/src/Benchmarks/Benchmarks/ReactiveBenchmark.cs b/src/Benchmarks/Benchmarks/ReactiveBenchmark.cs new file mode 100644 index 0000000..4bd090c --- /dev/null +++ b/src/Benchmarks/Benchmarks/ReactiveBenchmark.cs @@ -0,0 +1,374 @@ +using System; +using System.Reactive; +using System.Reactive.Disposables; +using System.Reactive.Linq; +using System.Reactive.Subjects; +using System.Threading; +using System.Threading.Channels; +using BenchmarkDotNet.Attributes; + +namespace Benchmarks.Benchmarks +{ + public class ReactiveBenchmark + { + private Subject _s; + public event Action MyTestEvent; + + public ReactiveBenchmark() + { + // channels for event producers -> metric producers + // + _s = new Subject(); + _s.AsObservable().Subscribe(new Subscriber()); + var i = 0; + //MyTestEvent += (e) => { i++; }; + _eventObs = Observable.FromEvent(h => MyTestEvent += h, r => {}); + _eventObs.Subscribe(new Subscriber()); + } + + public class Subscriber : IObserver + { + private int _i = 0; + public void OnCompleted() + { + + } + + public void OnError(Exception error) + { + + } + + public void OnNext(TestClass value) + { + _i++; + } + } + + [Benchmark] + public void TestPublishConsume() + { + _s.Publish(new TestClass()); + } + + private TestClass _fixed = new TestClass(); + private IObservable _eventObs; + + [Benchmark] + public void TestPublishConsumeReuse() + { + _s.Publish(_fixed); + } + + [Benchmark] + public void TestEvent() + { + MyTestEvent.Invoke(new TestClass()); + } + + [Benchmark] + public void TestEventReuse() + { + MyTestEvent.Invoke(_fixed); + } + + [Benchmark] + public void TestEventObservable() + { + MyTestEvent.Invoke(_fixed); + } + + + + public class TestClass + { + public int SampleValue1 { get; set; } + public string SampleValue2 { get; set; } + } + } + + public sealed class SubjectNoAlloc : SubjectBase + { + #region Fields + + private SubjectDisposable[] _observers; + private Exception? _exception; + private static readonly SubjectDisposable[] Terminated = new SubjectDisposable[0]; + private static readonly SubjectDisposable[] Disposed = new SubjectDisposable[0]; + + #endregion + + #region Constructors + // + // /// + // /// Creates a subject. + // /// + // public Subject() => _observers = Array.Empty(); + + #endregion + + #region Properties + + /// + /// Indicates whether the subject has observers subscribed to it. + /// + public override bool HasObservers => Volatile.Read(ref _observers).Length != 0; + + /// + /// Indicates whether the subject has been disposed. + /// + public override bool IsDisposed => Volatile.Read(ref _observers) == Disposed; + + #endregion + + #region Methods + + #region IObserver implementation + + private static void ThrowDisposed() => throw new ObjectDisposedException(string.Empty); + + /// + /// Notifies all subscribed observers about the end of the sequence. + /// + public override void OnCompleted() + { + for (; ; ) + { + var observers = Volatile.Read(ref _observers); + + if (observers == Disposed) + { + _exception = null; + ThrowDisposed(); + break; + } + + if (observers == Terminated) + { + break; + } + + if (Interlocked.CompareExchange(ref _observers, Terminated, observers) == observers) + { + foreach (var observer in observers) + { + observer.Observer?.OnCompleted(); + } + + break; + } + } + } + + /// + /// Notifies all subscribed observers about the specified exception. + /// + /// The exception to send to all currently subscribed observers. + /// is null. + public override void OnError(Exception error) + { + if (error == null) + { + throw new ArgumentNullException(nameof(error)); + } + + for (; ; ) + { + var observers = Volatile.Read(ref _observers); + + if (observers == Disposed) + { + _exception = null; + ThrowDisposed(); + break; + } + + if (observers == Terminated) + { + break; + } + + _exception = error; + + if (Interlocked.CompareExchange(ref _observers, Terminated, observers) == observers) + { + foreach (var observer in observers) + { + observer.Observer?.OnError(error); + } + + break; + } + } + } + + /// + /// Notifies all subscribed observers about the arrival of the specified element in the sequence. + /// + /// The value to send to all currently subscribed observers. + public override void OnNext(T value) + { + var observers = Volatile.Read(ref _observers); + + if (observers == Disposed) + { + _exception = null; + ThrowDisposed(); + return; + } + + foreach (var observer in observers) + { + observer.Observer?.OnNext(value); + } + } + + #endregion + + #region IObservable implementation + + /// + /// Subscribes an observer to the subject. + /// + /// Observer to subscribe to the subject. + /// Disposable object that can be used to unsubscribe the observer from the subject. + /// is null. + public override IDisposable Subscribe(IObserver observer) + { + if (observer == null) + { + throw new ArgumentNullException(nameof(observer)); + } + + var disposable = default(SubjectDisposable); + for (; ; ) + { + var observers = Volatile.Read(ref _observers); + + if (observers == Disposed) + { + _exception = null; + ThrowDisposed(); + + break; + } + + if (observers == Terminated) + { + var ex = _exception; + + if (ex != null) + { + observer.OnError(ex); + } + else + { + observer.OnCompleted(); + } + + break; + } + + disposable ??= new SubjectDisposable(this, observer); + + var n = observers.Length; + var b = new SubjectDisposable[n + 1]; + + Array.Copy(observers, 0, b, 0, n); + + b[n] = disposable; + + if (Interlocked.CompareExchange(ref _observers, b, observers) == observers) + { + return disposable; + } + } + + return Disposable.Empty; + } + + private void Unsubscribe(SubjectDisposable observer) + { + for (; ; ) + { + var a = Volatile.Read(ref _observers); + var n = a.Length; + + if (n == 0) + { + break; + } + + var j = Array.IndexOf(a, observer); + + if (j < 0) + { + break; + } + + SubjectDisposable[] b; + + if (n == 1) + { + b = Array.Empty(); + } + else + { + b = new SubjectDisposable[n - 1]; + + Array.Copy(a, 0, b, 0, j); + Array.Copy(a, j + 1, b, j, n - j - 1); + } + + if (Interlocked.CompareExchange(ref _observers, b, a) == a) + { + break; + } + } + } + + private sealed class SubjectDisposable : IDisposable + { + private SubjectNoAlloc _subject; + private volatile IObserver? _observer; + + public SubjectDisposable(SubjectNoAlloc subject, IObserver observer) + { + _subject = subject; + _observer = observer; + } + + public IObserver? Observer => _observer; + + public void Dispose() + { + var observer = Interlocked.Exchange(ref _observer, null); + if (observer == null) + { + return; + } + + _subject.Unsubscribe(this); + _subject = null!; + } + } + + #endregion + + #region IDisposable implementation + + /// + /// Releases all resources used by the current instance of the class and unsubscribes all observers. + /// + public override void Dispose() + { + Interlocked.Exchange(ref _observers, Disposed); + _exception = null; + + } + + #endregion + + #endregion + } +} \ No newline at end of file diff --git a/src/Benchmarks/Program.cs b/src/Benchmarks/Program.cs index 6657cea..191890e 100644 --- a/src/Benchmarks/Program.cs +++ b/src/Benchmarks/Program.cs @@ -11,7 +11,6 @@ using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Horology; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Order; @@ -21,6 +20,8 @@ using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; +using Perfolizer.Horology; +using Perfolizer.Mathematics.OutlierDetection; using Prometheus.DotNetRuntime; namespace Benchmarks @@ -29,20 +30,33 @@ public class Program { public static void Main(string[] args) { - BenchmarkSwitcher.FromTypes(new []{typeof(BaselineBenchmark), typeof(NoSamplingBenchmark), typeof(DefaultBenchmark)}).RunAllJoined( + BenchmarkRunner.Run( DefaultConfig.Instance .With( new Job() - .With(RunStrategy.Monitoring) - .WithLaunchCount(3) + .With(RunStrategy.Throughput) .WithWarmupCount(1) - .WithIterationTime(TimeInterval.FromSeconds(10)) - .WithCustomBuildConfiguration("ReleaseV3") + .WithIterationTime(TimeInterval.FromMilliseconds(300)) + .WithMaxIterationCount(30) + .WithCustomBuildConfiguration("Release") .WithOutlierMode(OutlierMode.DontRemove) ) .With(MemoryDiagnoser.Default) - .With(HardwareCounter.TotalCycles) ); + // BenchmarkSwitcher.FromTypes(new []{typeof(BaselineBenchmark), typeof(NoSamplingBenchmark), typeof(DefaultBenchmark)}).RunAllJoined( + // DefaultConfig.Instance + // .With( + // new Job() + // .With(RunStrategy.Monitoring) + // .WithLaunchCount(3) + // .WithWarmupCount(1) + // .WithIterationTime(TimeInterval.FromSeconds(10)) + // .WithCustomBuildConfiguration("Release") + // .WithOutlierMode(OutlierMode.DontRemove) + // ) + // .With(MemoryDiagnoser.Default) + // .With(HardwareCounter.TotalCycles) + // ); } } } \ No newline at end of file diff --git a/src/Common.csproj b/src/Common.csproj index a07633f..08b33fc 100644 --- a/src/Common.csproj +++ b/src/Common.csproj @@ -1,19 +1,8 @@ - - - 2 - $(DefineConstants);PROMV2 - - - - 3 - $(DefineConstants);PROMV3 - - + - 3 - ReleaseV2;DebugV2;DebugV3;ReleaseV3 + Release;Debug; AnyCPU - 8 + 9 @@ -25,11 +14,7 @@ false - + - - - - diff --git a/src/prometheus-net.DotNetRuntime.Tests/DotNetRuntimeStatsBuilderTests.cs b/src/prometheus-net.DotNetRuntime.Tests/DotNetRuntimeStatsBuilderTests.cs index 3abca9d..4033768 100644 --- a/src/prometheus-net.DotNetRuntime.Tests/DotNetRuntimeStatsBuilderTests.cs +++ b/src/prometheus-net.DotNetRuntime.Tests/DotNetRuntimeStatsBuilderTests.cs @@ -1,13 +1,10 @@ using System; -using System.Collections.Generic; using System.Net.Http; +using System.Threading; using System.Threading.Tasks; -using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; +using Microsoft.Extensions.DependencyInjection; using NUnit.Framework; -#if PROMV2 -using Prometheus.Advanced; -#endif -using Prometheus.DotNetRuntime.StatsCollectors; +using Prometheus.DotNetRuntime.EventListening.Parsers; namespace Prometheus.DotNetRuntime.Tests { @@ -25,7 +22,7 @@ public async Task Default_registers_all_expected_stats() // arrange using (DotNetRuntimeStatsBuilder.Default().StartCollecting()) { - await Assert_Expected_Stats_Are_Present_In_Registry(GetDefaultRegistry()); + await Assert_Expected_Stats_Are_Present_In_Registry(Prometheus.Metrics.DefaultRegistry); } } @@ -39,20 +36,6 @@ public async Task Default_registers_all_expected_stats_to_a_custom_registry() await Assert_Expected_Stats_Are_Present_In_Registry(registry); } } - - [Test] - public void WithCustomCollector_will_not_register_the_same_collector_twice() - { - var expectedCollector = new GcStatsCollector(); - var builder = DotNetRuntimeStatsBuilder - .Customize() - .WithGcStats() - .WithCustomCollector(expectedCollector); - - Assert.That(builder.StatsCollectors.Count, Is.EqualTo(1)); - Assert.That(builder.StatsCollectors.TryGetValue(new GcStatsCollector(), out var actualColector), Is.True); - Assert.That(actualColector, Is.SameAs(expectedCollector)); - } [Test] public void StartCollecting_Does_Not_Allow_Two_Collectors_To_Run_Simultaneously() @@ -66,14 +49,15 @@ public void StartCollecting_Does_Not_Allow_Two_Collectors_To_Run_Simultaneously( [Test] public async Task StartCollecting_Allows_A_New_Collector_To_Run_After_Disposing_A_Previous_Collector() { - using (DotNetRuntimeStatsBuilder.Customize().StartCollecting()) + var registry = NewRegistry(); + using (DotNetRuntimeStatsBuilder.Default().StartCollecting(registry)) { - await Assert_Expected_Stats_Are_Present_In_Registry(GetDefaultRegistry()); + await Assert_Expected_Stats_Are_Present_In_Registry(registry); } - using (DotNetRuntimeStatsBuilder.Customize().StartCollecting()) + using (DotNetRuntimeStatsBuilder.Default().StartCollecting(registry)) { - await Assert_Expected_Stats_Are_Present_In_Registry(GetDefaultRegistry()); + await Assert_Expected_Stats_Are_Present_In_Registry(registry); } } @@ -134,12 +118,53 @@ public void StartCollecting_Allows_A_New_Collector_To_Run_After_Disposing_Previo } } + [Test] + public void RegisterDefaultConsumers_Can_Register_Default_Consumers_For_All_Parsers() + { + // arrange + var services = new ServiceCollection(); + + // act + DotNetRuntimeStatsBuilder.Builder.RegisterDefaultConsumers(services); + var sp = services.BuildServiceProvider(); + + // assert + var infoConsumer = sp.GetService>(); + Assert.That(infoConsumer.Enabled, Is.False); + Assert.That(infoConsumer.Events, Is.Null); + + Assert.That(sp.GetService>, Is.Not.Null); + Assert.That(sp.GetService>, Is.Not.Null); + Assert.That(sp.GetService>, Is.Not.Null); + } + + [Test] + public void Cannot_Register_Tasks_At_Unsupported_Levels() + { + var ex = Assert.Throws(() => DotNetRuntimeStatsBuilder.Customize().WithGcStats(CaptureLevel.Errors)); + Assert.That(ex.SpecifiedLevel, Is.EqualTo(CaptureLevel.Errors)); + Assert.That(ex.SupportedLevels, Is.EquivalentTo(new []{ CaptureLevel.Verbose, CaptureLevel.Informational})); + + ex = Assert.Throws(() => DotNetRuntimeStatsBuilder.Customize().WithThreadPoolStats(CaptureLevel.Verbose)); + Assert.That(ex.SpecifiedLevel, Is.EqualTo(CaptureLevel.Verbose)); + Assert.That(ex.SupportedLevels, Is.EquivalentTo(new []{ CaptureLevel.Counters, CaptureLevel.Informational})); + } + + [Test] + public async Task Debugging_Metrics_Works_Correctly() + { + // arrange + var registry = NewRegistry(); + + using (DotNetRuntimeStatsBuilder.Default().WithDebuggingMetrics(true).StartCollecting(registry)) + { + await Assert_Expected_Stats_Are_Present_In_Registry(registry, shouldContainDebug: true); + } + } + private async Task Assert_Expected_Stats_Are_Present_In_Registry( -#if PROMV2 - DefaultCollectorRegistry registry -#else - CollectorRegistry registry -#endif + CollectorRegistry registry, + bool shouldContainDebug = false ) { // arrange @@ -156,37 +181,24 @@ CollectorRegistry registry // Some basic assertions to check that the output of our stats collectors is present Assert.That(content, Contains.Substring("dotnet_threadpool")); - Assert.That(content, Contains.Substring("dotnet_jit")); Assert.That(content, Contains.Substring("dotnet_gc")); Assert.That(content, Contains.Substring("dotnet_contention")); Assert.That(content, Contains.Substring("dotnet_build_info")); Assert.That(content, Contains.Substring("process_cpu_count")); + + if (shouldContainDebug) + { + Assert.That(content, Contains.Substring("dotnet_debug_event")); + } + else + StringAssert.DoesNotContain("dotnet_debug", content); } } } - -#if PROMV2 - private DefaultCollectorRegistry NewRegistry() - { - return new DefaultCollectorRegistry(); - } - private DefaultCollectorRegistry GetDefaultRegistry() - { - return DefaultCollectorRegistry.Instance; - } - -#elif PROMV3 private CollectorRegistry NewRegistry() { - return Metrics.NewCustomRegistry(); + return Prometheus.Metrics.NewCustomRegistry(); } - - private CollectorRegistry GetDefaultRegistry() - { - return Metrics.DefaultRegistry; - } -#endif - } } \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/DotNetRuntimeStatsCollectorTests.cs b/src/prometheus-net.DotNetRuntime.Tests/DotNetRuntimeStatsCollectorTests.cs new file mode 100644 index 0000000..331816b --- /dev/null +++ b/src/prometheus-net.DotNetRuntime.Tests/DotNetRuntimeStatsCollectorTests.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Tracing; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Prometheus.DotNetRuntime.EventListening; +using Prometheus.DotNetRuntime.EventListening.Parsers; + +namespace Prometheus.DotNetRuntime.Tests.EventListening +{ + [TestFixture] + public class DotNetRuntimeStatsCollectorTests + { + [Test] + [Timeout(20_000)] + public async Task After_Recycling_Then_Events_Can_Still_Be_Processed_Correctly() + { + // arrange + var parser = new RuntimeEventParser() { RefreshIntervalSeconds = 1}; + var eventAssertion = TestHelpers.ArrangeEventAssertion(e => parser.ExceptionCount += e); + var services = new ServiceCollection(); + var parserRego = ListenerRegistration.Create(CaptureLevel.Counters, _ => parser); + parserRego.RegisterServices(services); + services.AddSingleton, HashSet>(_ => new[] { parserRego }.ToHashSet()); + + // act + using var l = new DotNetRuntimeStatsCollector(services.BuildServiceProvider(), new CollectorRegistry(), new DotNetRuntimeStatsCollector.Options() { RecycleListenersEvery = TimeSpan.FromSeconds(3)}); + Assert.That(() => eventAssertion.Fired, Is.True.After(2000, 10)); + await Task.Delay(TimeSpan.FromSeconds(10)); + + // Why do we expected this value of events? Although we are waiting for 10 seconds for events, recycles may cause a counter period + // to not fire. As counter events fire each second, as long as this value is greater than the recycle period this test can veryify + // recycling is working correctly. + const int expectedCounterEvents = 6; + Assert.That(eventAssertion.History.Count, Is.GreaterThanOrEqualTo(expectedCounterEvents)); + Assert.That(l.EventListenerRecycles.Value, Is.InRange(3, 5)); + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/EventListening/EventCounterParserBaseTests.cs b/src/prometheus-net.DotNetRuntime.Tests/EventListening/EventCounterParserBaseTests.cs new file mode 100644 index 0000000..e124e69 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime.Tests/EventListening/EventCounterParserBaseTests.cs @@ -0,0 +1,135 @@ +using System; +using System.Diagnostics.Tracing; +using NUnit.Framework; +using Prometheus.DotNetRuntime.EventListening; + +namespace Prometheus.DotNetRuntime.Tests.EventListening +{ + [TestFixture] + public class Given_An_Implementation_Of_EventCounterParserBaseTest_That_Has_No_Decorated_Events + { + [Test] + public void When_Attributes_Are_Missing_From_Interface_Events_Then_Throw_Exception() + { + var ex = Assert.Throws(() => new NoAttributesEventCounterParser()); + Assert.That(ex, Has.Message.Contains("All events part of an ICounterEvents interface require a [CounterNameAttribute] attribute. Events without attribute: TestIncrementingCounter, TestMeanCounter.")); + } + + public class NoAttributesEventCounterParser : EventCounterParserBase, TestCounters + { + public override Guid EventSourceGuid { get; } + public override EventKeywords Keywords { get; } + public override int RefreshIntervalSeconds { get; set; } + + public event Action TestIncrementingCounter; + public event Action TestMeanCounter; + public event Action UnrelatedEvent; + } + } + + [TestFixture] + public class Given_An_Implementation_Of_EventCounterParserBaseTest_That_Has_Decorated_Events + { + private DummyTypeEventCounterParser _parser; + + [SetUp] + public void SetUp() + { + _parser = new DummyTypeEventCounterParser(); + } + + [Test] + public void When_An_Empty_Event_Is_Passed_Then_No_Exception_Occurs_And_No_Event_Is_Raised() + { + // arrange + var eventAssertionMean = TestHelpers.ArrangeEventAssertion(h => _parser.TestMeanCounter += h); + var eventAssertionIncr = TestHelpers.ArrangeEventAssertion(h => _parser.TestIncrementingCounter += h); + + // act + _parser.ProcessEvent(TestHelpers.CreateCounterEventWrittenEventArgs()); + + // assert + Assert.IsFalse(eventAssertionIncr.Fired); + Assert.IsFalse(eventAssertionMean.Fired); + } + + [Test] + public void When_A_MeanCounter_Is_Passed_For_A_Matching_MeanCounterValue_Event_Then_The_Event_Is_Fired() + { + // arrange + var eventAssertion = TestHelpers.ArrangeEventAssertion(h => _parser.TestMeanCounter += h); + var e = TestHelpers.CreateCounterEventWrittenEventArgs( + ("CounterType", "Mean"), + ("Name", "test-mean-counter"), + ("Mean", 5.0), + ("Count", 1) + ); + + // act + _parser.ProcessEvent(e); + + // assert + Assert.That(eventAssertion.Fired, Is.True); + Assert.That(eventAssertion.LastEvent.Mean, Is.EqualTo(5.0)); + Assert.That(eventAssertion.LastEvent.Count, Is.EqualTo(1)); + } + + [Test] + public void When_A_IncrementingCounter_Is_Passed_For_A_Matching_IncrementingCounterValue_Event_Then_The_Event_Is_Fired() + { + // arrange + var eventAssertion = TestHelpers.ArrangeEventAssertion(h => _parser.TestIncrementingCounter += h); + var e = TestHelpers.CreateCounterEventWrittenEventArgs( + ("CounterType", "Sum"), + ("Name", "test-incrementing-counter"), + ("Increment", 10.0) + ); + + // act + _parser.ProcessEvent(e); + + // assert + Assert.That(eventAssertion.Fired, Is.True); + Assert.That(eventAssertion.LastEvent.IncrementedBy, Is.EqualTo(10.0)); + } + + [Test] + public void When_A_IncrementingCounter_Is_Passed_For_A_Mismatching_MeanCounterValue_Event_Then_An_Exception_Is_Thrown() + { + // arrange + var meanEventAssertion = TestHelpers.ArrangeEventAssertion(h => _parser.TestMeanCounter += h); + var incrEventAssertion = TestHelpers.ArrangeEventAssertion(h => _parser.TestIncrementingCounter += h); + + var e = TestHelpers.CreateCounterEventWrittenEventArgs( + ("CounterType", "Mean"), + // refers to the incrementing counter + ("Name", "test-incrementing-counter"), + ("Mean", 5.0), + ("Count", 1) + ); + + // act + Assert.Throws(() => _parser.ProcessEvent(e)); + } + + public class DummyTypeEventCounterParser : EventCounterParserBase, TestCounters + { + public override Guid EventSourceGuid { get; } + public override EventKeywords Keywords { get; } + public override int RefreshIntervalSeconds { get; set; } + + [CounterName("test-incrementing-counter")] + public event Action TestIncrementingCounter; + [CounterName("test-mean-counter")] + public event Action TestMeanCounter; + } + } + + public interface TestCounters : ICounterEvents + { +#pragma disable warning + public event Action TestIncrementingCounter; + public event Action TestMeanCounter; +#pragma warning restore + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/EventListening/EventParserTypes.cs b/src/prometheus-net.DotNetRuntime.Tests/EventListening/EventParserTypes.cs new file mode 100644 index 0000000..0a010f5 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime.Tests/EventListening/EventParserTypes.cs @@ -0,0 +1,75 @@ +using System.Diagnostics.Tracing; +using NUnit.Framework; +using NUnit.Framework.Internal.Execution; +using Prometheus.DotNetRuntime.EventListening; +using Prometheus.DotNetRuntime.EventListening.Parsers; + +namespace Prometheus.DotNetRuntime.Tests.EventListening +{ + [TestFixture] + public class EventParserTypesTests + { + [Test] + public void Given_A_Type_That_Implements_All_IEvent_Interfaces_When_Calling_GetEventInterfaces_Then_Returns_All_Interfaces_Except_IEvents() + { + var interfaces = EventParserTypes.GetEventInterfaces(typeof(AllEvents)); + + Assert.That(interfaces, Is.EquivalentTo(new[] + { + typeof(AllEvents.Events.Verbose), + typeof(AllEvents.Events.Info), + typeof(AllEvents.Events.Warning), + typeof(AllEvents.Events.Error), + typeof(AllEvents.Events.Always), + typeof(AllEvents.Events.Critical), + typeof(AllEvents.Events.Counters) + })); + } + + [Test] + public void Given_A_Type_That_Implements_All_IEvent_Interfaces_When_Calling_GetLevelsFromType_Then_Returns_All_Interfaces_Except_IEvents() + { + var levels = EventParserTypes.GetLevelsFromParser(typeof(AllEvents)); + + Assert.That(levels, Is.EquivalentTo(new[] + { + EventLevel.LogAlways, + EventLevel.Verbose, + EventLevel.Informational, + EventLevel.Warning, + EventLevel.Error, + EventLevel.Critical + })); + } + + [Test] + public void When_Calling_GetEventParsers_Then_Returns_All_Event_Parsers_Defined_In_The_DotNetRuntime_Library() + { + var parsers = EventParserTypes.GetEventParsers(); + + Assert.That(parsers, Is.SupersetOf(new[] + { + typeof(GcEventParser), + typeof(JitEventParser), + typeof(ThreadPoolEventParser), + typeof(RuntimeEventParser), + typeof(ContentionEventParser), + typeof(ExceptionEventParser) + })); + } + + public class AllEvents : AllEvents.Events.Verbose, AllEvents.Events.Info, AllEvents.Events.Warning, AllEvents.Events.Error, AllEvents.Events.Always, AllEvents.Events.Counters, AllEvents.Events.Critical, IEvents + { + public static class Events + { + public interface Verbose : IVerboseEvents{} + public interface Info : IInfoEvents{} + public interface Warning : IWarningEvents {} + public interface Error : IErrorEvents{} + public interface Always : IAlwaysEvents{} + public interface Critical : ICriticalEvents{} + public interface Counters : ICounterEvents{} + } + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/EventListening/IEventParserTests.cs b/src/prometheus-net.DotNetRuntime.Tests/EventListening/IEventParserTests.cs new file mode 100644 index 0000000..f4aa396 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime.Tests/EventListening/IEventParserTests.cs @@ -0,0 +1,43 @@ +using System; +using System.Diagnostics.Tracing; +using NUnit.Framework; +using Prometheus.DotNetRuntime.EventListening; +using Prometheus.DotNetRuntime.EventListening.Parsers; + +namespace Prometheus.DotNetRuntime.Tests.EventListening +{ + [TestFixture] + public class IEventParserTests + { + [Test] + public void Given_A_Parser_Implements_One_Or_More_IEvents_Interface_Then_Can_Get_Appropriate_Levels() + { + IEventListener gcParser = new GcEventParser(); + Assert.That(gcParser.SupportedLevels, Is.EquivalentTo(new[] { EventLevel.Informational, EventLevel.Verbose })); + } + + [Test] + public void Given_A_Parser_Implements_ICounterEvents_Then_Can_Get_Appropriate_Levels() + { + IEventListener runtimeEventParser = new RuntimeEventParser(); + Assert.That(runtimeEventParser.SupportedLevels, Is.EquivalentTo(new[] { EventLevel.LogAlways })); + } + + [Test] + public void Given_A_Parser_Only_Implements_IEvents_Returns_No_Levels() + { + IEventListener noEventsParser = new TestParserNoEvents(); + Assert.That(noEventsParser.SupportedLevels, Is.Empty); + } + + private class TestParserNoEvents : IEventParser, IEvents + { + public Guid EventSourceGuid { get; } + public EventKeywords Keywords { get; } + public void ProcessEvent(EventWrittenEventArgs e) + { + throw new NotImplementedException(); + } + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/EventListening/Parsers/EventListenerIntegrationTestBase.cs b/src/prometheus-net.DotNetRuntime.Tests/EventListening/Parsers/EventListenerIntegrationTestBase.cs new file mode 100644 index 0000000..1d3ae17 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime.Tests/EventListening/Parsers/EventListenerIntegrationTestBase.cs @@ -0,0 +1,41 @@ +using System; +using System.Diagnostics.Tracing; +using System.Threading; +using NUnit.Framework; +using Prometheus.DotNetRuntime.EventListening; + +namespace Prometheus.DotNetRuntime.Tests.EventListening.Parsers +{ + [TestFixture] + public abstract class EventListenerIntegrationTestBase + where TEventListener : IEventListener + { + private DotNetEventListener _eventListener; + protected TEventListener Parser { get; private set; } + + [SetUp] + public void SetUp() + { + Parser = CreateListener(); + _eventListener = new DotNetEventListener(Parser, EventLevel.LogAlways, new DotNetEventListener.GlobalOptions{ ErrorHandler = ex => Assert.Fail($"Unexpected exception occurred: {ex}")}); + + // wait for event listener thread to spin up + while (!_eventListener.StartedReceivingEvents) + { + Thread.Sleep(10); + Console.Write("Waiting.. "); + + } + Console.WriteLine("EventListener should be active now."); + } + + [TearDown] + public void TearDown() + { + Console.WriteLine("Disposing event listener.."); + _eventListener.Dispose(); + } + + protected abstract TEventListener CreateListener(); + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/EventListening/Parsers/SystemRuntimeCounterTests.cs b/src/prometheus-net.DotNetRuntime.Tests/EventListening/Parsers/SystemRuntimeCounterTests.cs new file mode 100644 index 0000000..28d861b --- /dev/null +++ b/src/prometheus-net.DotNetRuntime.Tests/EventListening/Parsers/SystemRuntimeCounterTests.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using NUnit.Framework; +using Prometheus.DotNetRuntime.EventListening.Parsers; + +namespace Prometheus.DotNetRuntime.Tests.EventListening.Parsers +{ + [TestFixture] + public class SystemRuntimeCounterTests : EventListenerIntegrationTestBase + { + [Test] + public void TestEvent() + { + var resetEvent = new AutoResetEvent(false); + Parser.AllocRate += e => + { + resetEvent.Set(); + Assert.That(e.IncrementedBy, Is.GreaterThan(0)); + }; + + Assert.IsTrue(resetEvent.WaitOne(TimeSpan.FromSeconds(10))); + } + + protected override RuntimeEventParser CreateListener() + { + return new (); + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/Util/Given_An_EventPairTimer_That_Samples_Every_Event.cs b/src/prometheus-net.DotNetRuntime.Tests/EventListening/Parsers/Util/EventPairTimerTests.cs similarity index 66% rename from src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/Util/Given_An_EventPairTimer_That_Samples_Every_Event.cs rename to src/prometheus-net.DotNetRuntime.Tests/EventListening/Parsers/Util/EventPairTimerTests.cs index 1736ffc..3035d28 100644 --- a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/Util/Given_An_EventPairTimer_That_Samples_Every_Event.cs +++ b/src/prometheus-net.DotNetRuntime.Tests/EventListening/Parsers/Util/EventPairTimerTests.cs @@ -1,11 +1,11 @@ using System; using System.Collections.ObjectModel; using System.Diagnostics.Tracing; -using NUnit.Framework; -using Prometheus.DotNetRuntime.StatsCollectors.Util; using Fasterflect; +using NUnit.Framework; +using Prometheus.DotNetRuntime.EventListening.Parsers.Util; -namespace Prometheus.DotNetRuntime.Tests.StatsCollectors.Util +namespace Prometheus.DotNetRuntime.Tests.EventListening.Parsers.Util { [TestFixture] public class Given_An_EventPairTimer_That_Samples_Every_Event : EventPairTimerBaseClass @@ -21,7 +21,7 @@ public void SetUp() [Test] public void TryGetEventPairDuration_ignores_events_that_its_not_configured_to_look_for() { - var nonMonitoredEvent = CreateEventWrittenEventArgs(3); + var nonMonitoredEvent = TestHelpers.CreateEventWrittenEventArgs(3); Assert.That(_eventPairTimer.TryGetDuration(nonMonitoredEvent, out var duration), Is.EqualTo(DurationResult.Unrecognized)); Assert.That(duration, Is.EqualTo(TimeSpan.Zero)); } @@ -29,7 +29,7 @@ public void TryGetEventPairDuration_ignores_events_that_its_not_configured_to_lo [Test] public void TryGetEventPairDuration_ignores_end_events_if_it_never_saw_the_start_event() { - var nonMonitoredEvent = CreateEventWrittenEventArgs(EventIdEnd, payload: 1L); + var nonMonitoredEvent = TestHelpers.CreateEventWrittenEventArgs(EventIdEnd, payload: 1L); Assert.That(_eventPairTimer.TryGetDuration(nonMonitoredEvent, out var duration),Is.EqualTo(DurationResult.FinalWithoutDuration)); Assert.That(duration, Is.EqualTo(TimeSpan.Zero)); } @@ -39,9 +39,9 @@ public void TryGetEventPairDuration_calculates_duration_between_configured_event { // arrange var now = DateTime.UtcNow; - var startEvent = CreateEventWrittenEventArgs(EventIdStart, now, payload: 1L); + var startEvent = TestHelpers.CreateEventWrittenEventArgs(EventIdStart, now, payload: 1L); Assert.That(_eventPairTimer.TryGetDuration(startEvent, out var _), Is.EqualTo(DurationResult.Start)); - var endEvent = CreateEventWrittenEventArgs(EventIdEnd, now.AddMilliseconds(100), payload: 1L); + var endEvent = TestHelpers.CreateEventWrittenEventArgs(EventIdEnd, now.AddMilliseconds(100), payload: 1L); // act Assert.That(_eventPairTimer.TryGetDuration(endEvent, out var duration), Is.EqualTo(DurationResult.FinalWithDuration)); @@ -53,9 +53,9 @@ public void TryGetEventPairDuration_calculates_duration_between_configured_event { // arrange var now = DateTime.UtcNow; - var startEvent = CreateEventWrittenEventArgs(EventIdStart, now, payload: 1L); + var startEvent = TestHelpers.CreateEventWrittenEventArgs(EventIdStart, now, payload: 1L); Assert.That(_eventPairTimer.TryGetDuration(startEvent, out var _), Is.EqualTo(DurationResult.Start)); - var endEvent = CreateEventWrittenEventArgs(EventIdEnd, now, payload: 1L); + var endEvent = TestHelpers.CreateEventWrittenEventArgs(EventIdEnd, now, payload: 1L); // act Assert.That(_eventPairTimer.TryGetDuration(endEvent, out var duration), Is.EqualTo(DurationResult.FinalWithDuration)); @@ -67,12 +67,12 @@ public void TryGetEventPairDuration_calculates_duration_between_multiple_out_of_ { // arrange var now = DateTime.UtcNow; - var startEvent1 = CreateEventWrittenEventArgs(EventIdStart, now, payload: 1L); - var endEvent1 = CreateEventWrittenEventArgs(EventIdEnd, now.AddMilliseconds(300), payload: 1L); - var startEvent2 = CreateEventWrittenEventArgs(EventIdStart, now, payload: 2L); - var endEvent2 = CreateEventWrittenEventArgs(EventIdEnd, now.AddMilliseconds(200), payload: 2L); - var startEvent3 = CreateEventWrittenEventArgs(EventIdStart, now, payload: 3L); - var endEvent3 = CreateEventWrittenEventArgs(EventIdEnd, now.AddMilliseconds(100), payload: 3L); + var startEvent1 = TestHelpers.CreateEventWrittenEventArgs(EventIdStart, now, payload: 1L); + var endEvent1 = TestHelpers.CreateEventWrittenEventArgs(EventIdEnd, now.AddMilliseconds(300), payload: 1L); + var startEvent2 = TestHelpers.CreateEventWrittenEventArgs(EventIdStart, now, payload: 2L); + var endEvent2 = TestHelpers.CreateEventWrittenEventArgs(EventIdEnd, now.AddMilliseconds(200), payload: 2L); + var startEvent3 = TestHelpers.CreateEventWrittenEventArgs(EventIdStart, now, payload: 3L); + var endEvent3 = TestHelpers.CreateEventWrittenEventArgs(EventIdEnd, now.AddMilliseconds(100), payload: 3L); _eventPairTimer.TryGetDuration(startEvent1, out var _); _eventPairTimer.TryGetDuration(startEvent2, out var _); @@ -105,7 +105,7 @@ public void SetUp() [Test] public void TryGetEventPairDuration_recognizes_start_events_that_will_be_discarded() { - var startEvent1 = CreateEventWrittenEventArgs(EventIdStart, DateTime.UtcNow, payload: 1L); + var startEvent1 = TestHelpers.CreateEventWrittenEventArgs(EventIdStart, DateTime.UtcNow, payload: 1L); Assert.That(_eventPairTimer.TryGetDuration(startEvent1, out var duration),Is.EqualTo(DurationResult.Start)); Assert.That(duration, Is.EqualTo(TimeSpan.Zero)); } @@ -115,10 +115,10 @@ public void TryGetEventPairDuration_will_discard_1_event_and_calculate_duration_ { // arrange var now = DateTime.UtcNow; - var startEvent1 = CreateEventWrittenEventArgs(EventIdStart, now, payload: 1L); - var endEvent1 = CreateEventWrittenEventArgs(EventIdEnd, now.AddMilliseconds(300), payload: 1L); - var startEvent2 = CreateEventWrittenEventArgs(EventIdStart, now, payload: 2L); - var endEvent2 = CreateEventWrittenEventArgs(EventIdEnd, now.AddMilliseconds(200), payload: 2L); + var startEvent1 = TestHelpers.CreateEventWrittenEventArgs(EventIdStart, now, payload: 1L); + var endEvent1 = TestHelpers.CreateEventWrittenEventArgs(EventIdEnd, now.AddMilliseconds(300), payload: 1L); + var startEvent2 = TestHelpers.CreateEventWrittenEventArgs(EventIdStart, now, payload: 2L); + var endEvent2 = TestHelpers.CreateEventWrittenEventArgs(EventIdEnd, now.AddMilliseconds(200), payload: 2L); _eventPairTimer.TryGetDuration(startEvent1, out var _); _eventPairTimer.TryGetDuration(startEvent2, out var _); @@ -134,18 +134,6 @@ public class EventPairTimerBaseClass { protected const int EventIdStart = 1, EventIdEnd = 2; - protected EventWrittenEventArgs CreateEventWrittenEventArgs(int eventId, DateTime? timestamp = null, params object[] payload) - { - var args = (EventWrittenEventArgs)typeof(EventWrittenEventArgs).CreateInstance(new []{ typeof(EventSource)}, Flags.NonPublic | Flags.Instance, new object[] { null}); - args.SetPropertyValue("EventId", eventId); - args.SetPropertyValue("Payload", new ReadOnlyCollection(payload)); - - if (timestamp.HasValue) - { - args.SetPropertyValue("TimeStamp", timestamp.Value); - } - - return args; - } + } } \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/Util/SamplingRateTests.cs b/src/prometheus-net.DotNetRuntime.Tests/EventListening/Parsers/Util/SamplingRateTests.cs similarity index 92% rename from src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/Util/SamplingRateTests.cs rename to src/prometheus-net.DotNetRuntime.Tests/EventListening/Parsers/Util/SamplingRateTests.cs index fee34ba..12ca927 100644 --- a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/Util/SamplingRateTests.cs +++ b/src/prometheus-net.DotNetRuntime.Tests/EventListening/Parsers/Util/SamplingRateTests.cs @@ -1,8 +1,7 @@ -using System; using NUnit.Framework; -using Prometheus.DotNetRuntime.StatsCollectors.Util; +using Prometheus.DotNetRuntime.EventListening.Parsers.Util; -namespace Prometheus.DotNetRuntime.Tests.StatsCollectors.Util +namespace Prometheus.DotNetRuntime.Tests.EventListening.Parsers.Util { [TestFixture] public class SamplingRateTests diff --git a/src/prometheus-net.DotNetRuntime.Tests/EventListening/TestHelpers.cs b/src/prometheus-net.DotNetRuntime.Tests/EventListening/TestHelpers.cs new file mode 100644 index 0000000..f63abeb --- /dev/null +++ b/src/prometheus-net.DotNetRuntime.Tests/EventListening/TestHelpers.cs @@ -0,0 +1,61 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics.Tracing; +using System.Linq; +using Fasterflect; +using NUnit.Framework; + +namespace Prometheus.DotNetRuntime.Tests.EventListening +{ + public class TestHelpers + { + public static EventWrittenEventArgs CreateEventWrittenEventArgs(int eventId, DateTime? timestamp = null, params object[] payload) + { + var args = (EventWrittenEventArgs)typeof(EventWrittenEventArgs).CreateInstance(new []{ typeof(EventSource)}, Flags.NonPublic | Flags.Instance, new object[] { null}); + args.SetPropertyValue("EventId", eventId); + args.SetPropertyValue("Payload", new ReadOnlyCollection(payload)); + + if (timestamp.HasValue) + { + args.SetPropertyValue("TimeStamp", timestamp.Value); + } + + return args; + } + + public static EventWrittenEventArgs CreateCounterEventWrittenEventArgs(params (string key, object val)[] payload) + { + var counterPayload = payload.ToDictionary(k => k.key, v => v.val); + + var e = CreateEventWrittenEventArgs(-1, DateTime.UtcNow, new[] { counterPayload }); + e.SetPropertyValue("EventName", "EventCounters"); + return e; + } + + public static EventAssertion ArrangeEventAssertion(Action> wireUp) + { + return new EventAssertion(wireUp); + } + + public class EventAssertion + { + private Action _handler; + + public EventAssertion(Action> wireUp) + { + _handler = e => + { + History.Add(e); + }; + + wireUp(_handler); + } + + public bool Fired => History.Count > 0; + public List History { get; } = new List(); + public T LastEvent => History.Last(); + + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/ExtensionTests.cs b/src/prometheus-net.DotNetRuntime.Tests/ExtensionTests.cs index 09fef49..60ee0eb 100644 --- a/src/prometheus-net.DotNetRuntime.Tests/ExtensionTests.cs +++ b/src/prometheus-net.DotNetRuntime.Tests/ExtensionTests.cs @@ -1,5 +1,6 @@ using System.Linq; using NUnit.Framework; +using Prometheus.DotNetRuntime.Metrics; namespace Prometheus.DotNetRuntime.Tests { @@ -10,13 +11,13 @@ public class ExtensionTests public void CollectAllValues_Extracts_All_Labeled_And_Unlabeled_Values_From_A_Counter() { // arrange - var counter = Metrics.CreateCounter("test_counter", "", "label1", "label2"); + var counter = Prometheus.Metrics.CreateCounter("test_counter", "", "label1", "label2"); counter.Inc(); // unlabeled counter.Labels("1", "2").Inc(); counter.Labels("1", "3").Inc(2); // act - var values = counter.CollectAllValues(); + var values = MetricExtensions.CollectAllValues(counter); // assert Assert.That(values.Count(), Is.EqualTo(3)); @@ -27,13 +28,13 @@ public void CollectAllValues_Extracts_All_Labeled_And_Unlabeled_Values_From_A_Co public void CollectAllValues_Extracts_All_Labeled_Values_From_A_Counter_When_excludeUnlabeled_Is_True() { // arrange - var counter = Metrics.CreateCounter("test_counter2", "", "label1", "label2"); + var counter = Prometheus.Metrics.CreateCounter("test_counter2", "", "label1", "label2"); counter.Inc(); // unlabeled counter.Labels("1", "2").Inc(); counter.Labels("1", "3").Inc(2); // act - var values = counter.CollectAllValues(excludeUnlabeled: true); + var values = MetricExtensions.CollectAllValues(counter, excludeUnlabeled: true); // assert Assert.That(values.Count(), Is.EqualTo(2)); @@ -44,14 +45,14 @@ public void CollectAllValues_Extracts_All_Labeled_Values_From_A_Counter_When_exc public void CollectAllValues_Extracts_All_Labeled_And_Unlabeled_Summed_Values_From_A_Histogram() { // arrange - var histo = Metrics.CreateHistogram("test_histo", "", labelNames: new [] {"label1", "label2"}); + var histo = Prometheus.Metrics.CreateHistogram("test_histo", "", labelNames: new [] {"label1", "label2"}); histo.Observe(1); // unlabeled histo.Labels("1", "2").Observe(2); histo.Labels("1", "2").Observe(3); histo.Labels("1", "3").Observe(4); // act - var values = histo.CollectAllSumValues(); + var values = MetricExtensions.CollectAllSumValues(histo); // assert Assert.That(values.Count(), Is.EqualTo(3)); @@ -62,14 +63,14 @@ public void CollectAllValues_Extracts_All_Labeled_And_Unlabeled_Summed_Values_Fr public void CollectAllValues_Extracts_Labeled_Summed_Values_From_A_Histogram_When_excludeUnlabeled_Is_True() { // arrange - var histo = Metrics.CreateHistogram("test_histo2", "", labelNames: new [] {"label1", "label2"}); + var histo = Prometheus.Metrics.CreateHistogram("test_histo2", "", labelNames: new [] {"label1", "label2"}); histo.Observe(1); // unlabeled histo.Labels("1", "2").Observe(2); histo.Labels("1", "2").Observe(3); histo.Labels("1", "3").Observe(4); // act - var values = histo.CollectAllSumValues(excludeUnlabeled: true); + var values = MetricExtensions.CollectAllSumValues(histo, excludeUnlabeled: true); // assert Assert.That(values.Count(), Is.EqualTo(2)); @@ -80,14 +81,14 @@ public void CollectAllValues_Extracts_Labeled_Summed_Values_From_A_Histogram_Whe public void CollectAllValues_Extracts_All_Labeled_And_Unlabeled_Count_Values_From_A_Histogram() { // arrange - var histo = Metrics.CreateHistogram("test_histo3", "", labelNames: new []{ "label1", "label2"}); + var histo = Prometheus.Metrics.CreateHistogram("test_histo3", "", labelNames: new []{ "label1", "label2"}); histo.Observe(1); // unlabeled histo.Labels("1", "2").Observe(2); histo.Labels("1", "2").Observe(3); histo.Labels("1", "3").Observe(4); // act - var values = histo.CollectAllCountValues(); + var values = MetricExtensions.CollectAllCountValues(histo); // assert Assert.That(values.Count(), Is.EqualTo(3)); diff --git a/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/ContentionTests.cs b/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/ContentionTests.cs new file mode 100644 index 0000000..a9ef204 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/ContentionTests.cs @@ -0,0 +1,116 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Prometheus.DotNetRuntime.Metrics.Producers; + +namespace Prometheus.DotNetRuntime.Tests.IntegrationTests +{ + [TestFixture] + internal class Given_Contention_Events_Are_Enabled_For_Contention_Stats : IntegrationTestBase + { + protected override DotNetRuntimeStatsBuilder.Builder ConfigureBuilder(DotNetRuntimeStatsBuilder.Builder toConfigure) + { + return toConfigure.WithContentionStats(CaptureLevel.Informational, SampleEvery.OneEvent); + } + + [Test] + public void Will_measure_no_contention_on_an_uncontested_lock() + { + // arrange + var key = new Object(); + + // act + lock (key) + { + } + + // assert + Assert.That(MetricProducer.ContentionTotal.Value, Is.EqualTo(0)); + Assert.That(MetricProducer.ContentionSecondsTotal.Value, Is.EqualTo(0)); + } + + /// + /// This test has the potential to be flaky (due to attempting to simulate lock contention across multiple threads in the thread pool), + /// may have to revisit this in the future.. + /// + /// + [Test] + [Repeat(3)] + public async Task Will_measure_contention_on_a_contested_lock() + { + // arrange + const int numThreads = 10; + const int sleepForMs = 50; + var key = new object(); + // Increase the min. thread pool size so that when we use Thread.Sleep, we don't run into scheduling delays + ThreadPool.SetMinThreads(numThreads * 2, 1); + + // act + var tasks = Enumerable.Range(1, numThreads) + .Select(_ => Task.Run(() => + { + lock (key) + { + Thread.Sleep(sleepForMs); + } + })); + + await Task.WhenAll(tasks); + + // assert + + // Why -1? The first thread will not contend the lock + const int numLocksContended = numThreads - 1; + Assert.That(() => MetricProducer.ContentionTotal.Value, Is.GreaterThanOrEqualTo(numLocksContended).After(3000, 10)); + + // Pattern of expected contention times is: 50ms, 100ms, 150ms, etc. + var expectedDelay = TimeSpan.FromMilliseconds(Enumerable.Range(1, numLocksContended).Aggregate(sleepForMs, (acc, next) => acc + (sleepForMs * next))); + Assert.That(MetricProducer.ContentionSecondsTotal.Value, Is.EqualTo(expectedDelay.TotalSeconds).Within(sleepForMs)); + } + } + + [TestFixture] + internal class Given_Only_Counters_Are_Enabled_For_Contention_Stats : IntegrationTestBase + { + protected override DotNetRuntimeStatsBuilder.Builder ConfigureBuilder(DotNetRuntimeStatsBuilder.Builder toConfigure) + { + return toConfigure.WithContentionStats(CaptureLevel.Counters, SampleEvery.OneEvent); + } + + /// + /// This test has the potential to be flaky (due to attempting to simulate lock contention across multiple threads in the thread pool), + /// may have to revisit this in the future.. + /// + /// + [Test] + public async Task Will_measure_contention_on_a_contested_lock() + { + // arrange + const int numThreads = 10; + const int sleepForMs = 50; + var key = new object(); + // Increase the min. thread pool size so that when we use Thread.Sleep, we don't run into scheduling delays + ThreadPool.SetMinThreads(numThreads * 2, 1); + + // act + var tasks = Enumerable.Range(1, numThreads) + .Select(_ => Task.Run(() => + { + lock (key) + { + Thread.Sleep(sleepForMs); + } + })); + + await Task.WhenAll(tasks); + + // assert + + // Why -1? The first thread will not contend the lock + const int numLocksContended = numThreads - 1; + Assert.That(() => MetricProducer.ContentionTotal.Value, Is.GreaterThanOrEqualTo(numLocksContended).After(3000, 10)); + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/ExceptionTests.cs b/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/ExceptionTests.cs new file mode 100644 index 0000000..b752e21 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/ExceptionTests.cs @@ -0,0 +1,70 @@ +using NUnit.Framework; +using System; +using Prometheus.DotNetRuntime.Metrics.Producers; + +namespace Prometheus.DotNetRuntime.Tests.IntegrationTests +{ + [TestFixture] + internal class Given_Exception_Events_Are_Enabled_For_Exception_Stats : IntegrationTestBase + { + protected override DotNetRuntimeStatsBuilder.Builder ConfigureBuilder(DotNetRuntimeStatsBuilder.Builder toConfigure) + { + return toConfigure.WithExceptionStats(CaptureLevel.Errors); + } + + [Test] + [MaxTime(10_000)] + public void Will_measure_when_occurring_an_exception() + { + // act + var divider = 0; + const int numToThrow = 10; + + for (int i = 0; i < numToThrow; i++) + { + try + { + _ = 1 / divider; + } + catch (DivideByZeroException) + { + } + } + + // assert + Assert.That(() => MetricProducer.ExceptionCount.Labels("System.DivideByZeroException").Value, Is.GreaterThanOrEqualTo(numToThrow).After(100, 1000)); + } + } + + [TestFixture] + internal class Given_Only_Runtime_Counters_Are_Enabled_For_Exception_Stats : IntegrationTestBase + { + protected override DotNetRuntimeStatsBuilder.Builder ConfigureBuilder(DotNetRuntimeStatsBuilder.Builder toConfigure) + { + return toConfigure.WithExceptionStats(CaptureLevel.Counters); + } + + [Test] + [MaxTime(10_000)] + public void Will_measure_when_occurring_an_exception() + { + // act + var divider = 0; + const int numToThrow = 10; + + for (int i = 0; i < numToThrow; i++) + { + try + { + _ = 1 / divider; + } + catch (DivideByZeroException) + { + } + } + + // assert + Assert.That(() => MetricProducer.ExceptionCount.Value, Is.GreaterThanOrEqualTo(numToThrow).After(3_000, 100)); + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/GcTests.cs b/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/GcTests.cs new file mode 100644 index 0000000..ec0b49e --- /dev/null +++ b/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/GcTests.cs @@ -0,0 +1,241 @@ +using System; +using System.Linq; +using System.Runtime.InteropServices; +using System.Threading; +using NUnit.Framework; +using Prometheus.DotNetRuntime.Metrics; +using Prometheus.DotNetRuntime.Metrics.Producers; + +namespace Prometheus.DotNetRuntime.Tests.IntegrationTests +{ + internal class Given_Only_Counters_Are_Available_For_GcStats : IntegrationTestBase + { + protected override DotNetRuntimeStatsBuilder.Builder ConfigureBuilder(DotNetRuntimeStatsBuilder.Builder toConfigure) + { + return toConfigure.WithGcStats(CaptureLevel.Counters); + } + + [Test] + public void When_objects_are_allocated_then_the_allocated_bytes_counter_is_increased() + { + Assert.That(MetricProducer.AllocatedBytes.LabelNames, Is.Empty); + var previousValue = MetricProducer.AllocatedBytes.Value; + + // allocate roughly 100kb+ of small objects + for (int i = 0; i < 11; i++) + { + var b = new byte[10_000]; + } + + Assert.That(() => MetricProducer.AllocatedBytes.Value, Is.GreaterThanOrEqualTo(previousValue + 100_000).After(2_000, 10)); + } + + [Test] + public void When_collections_happen_then_the_collection_count_is_increased([Values(0, 1, 2)] int generation) + { + Assert.That(MetricProducer.GcCollections.LabelNames, Is.EquivalentTo(new []{ "gc_generation" })); + var previousValue = MetricProducer.GcCollections.Labels(generation.ToString()).Value; + const int numCollectionsToRun = 10; + + // run collections + for (int i = 0; i < numCollectionsToRun; i++) + { + GC.Collect(generation, GCCollectionMode.Forced); + } + + Assert.That(() => MetricProducer.GcCollections.Labels(generation.ToString()).Value, Is.GreaterThanOrEqualTo(previousValue + numCollectionsToRun).After(2_000, 10)); + } + + [Test] + public void When_a_garbage_collection_is_performed_then_the_heap_sizes_are_updated() + { + Assert.That(() => MetricProducer.GcHeapSizeBytes.Labels("0").Value, Is.GreaterThan(0).After(2000, 10)); + Assert.That(() => MetricProducer.GcHeapSizeBytes.Labels("1").Value, Is.GreaterThan(0).After(2000, 10)); + Assert.That(() => MetricProducer.GcHeapSizeBytes.Labels("2").Value, Is.GreaterThan(0).After(2000, 10)); + Assert.That(() => MetricProducer.GcHeapSizeBytes.Labels("loh").Value, Is.GreaterThan(0).After(2000, 10)); + } + + [Test] + public void When_a_garbage_collection_is_performed_then_the_pause_ratios_can_be_calculated() + { + // arrange + for (int i = 0; i < 5; i++) + GC.Collect(2, GCCollectionMode.Forced, true, true); + + // assert + Assert.That(() => MetricProducer.GcPauseRatio.Value, Is.GreaterThan(0.0).After(2000, 10)); + } + } + + internal class Given_Gc_Info_Events_Are_Available_For_GcStats : IntegrationTestBase + { + protected override DotNetRuntimeStatsBuilder.Builder ConfigureBuilder(DotNetRuntimeStatsBuilder.Builder toConfigure) + { + return toConfigure.WithGcStats(CaptureLevel.Informational); + } + + [Test] + public void When_objects_are_allocated_then_the_allocated_bytes_counter_is_increased() + { + Assert.That(MetricProducer.AllocatedBytes.LabelNames, Is.Empty); + var previousValue = MetricProducer.AllocatedBytes.Value; + + // allocate roughly 100kb+ of small objects + for (int i = 0; i < 11; i++) + { + var b = new byte[10_000]; + } + + Assert.That(() => MetricProducer.AllocatedBytes.Value, Is.GreaterThanOrEqualTo(previousValue + 100_000).After(2_000, 10)); + } + + [Test] + public void When_a_garbage_collection_is_performed_then_the_heap_sizes_are_updated() + { + unsafe + { + // arrange (fix a variable to ensure the pinned objects counter is incremented + var b = new byte[1]; + fixed (byte* p = b) + { + // act + GC.Collect(0); + } + + Assert.That(() => MetricProducer.GcHeapSizeBytes.Labels("0").Value, Is.GreaterThan(0).After(200, 10)); + Assert.That(() => MetricProducer.GcHeapSizeBytes.Labels("1").Value, Is.GreaterThan(0).After(200, 10)); + Assert.That(() => MetricProducer.GcHeapSizeBytes.Labels("2").Value, Is.GreaterThan(0).After(200, 10)); + Assert.That(() => MetricProducer.GcHeapSizeBytes.Labels("loh").Value, Is.GreaterThan(0).After(200, 10)); + Assert.That(() => MetricProducer.GcNumPinnedObjects.Value, Is.GreaterThan(0).After(200, 10)); + } + } + + [Test] + public void When_collections_happen_then_the_collection_count_is_increased([Values(0, 1, 2)] int generation) + { + double GetCollectionCount() + { + // Sum all the generation values (we cannot reliably know the reasons upfront) + return MetricProducer.GcCollections.GetAllLabelValues() + .Where(l => l[0] == generation.ToString()) + .Sum(l => MetricProducer.GcCollections.Labels(l).Value); + } + + Assert.That(MetricProducer.GcCollections.LabelNames, Is.EquivalentTo(new []{ "gc_generation", "gc_reason" })); + var previousValue = GetCollectionCount(); + const int numCollectionsToRun = 10; + + Thread.Sleep(2000); + + // run collections + for (int i = 0; i < numCollectionsToRun; i++) + { + GC.Collect(generation, GCCollectionMode.Forced); + } + + // assert + + // For some reason, the full number of gen0 collections are not being collected. I expect this is because .NET will not always force + // a gen 0 collection to occur. + const int minExpectedCollections = numCollectionsToRun / 2; + Assert.That( + del: GetCollectionCount, + Is.GreaterThanOrEqualTo(previousValue + minExpectedCollections).After(2_000, 10) + ); + } + + [Test] + public void When_a_garbage_collection_is_performed_then_the_finalization_queue_is_updated() + { + // arrange + { + var finalizable = new FinalizableTest(); + finalizable = null; + } + GC.Collect(0); + + // assert + Assert.That(() => MetricProducer.GcFinalizationQueueLength.Value, Is.GreaterThan(0).After(200, 10)); + } + + [Test] + public void When_a_garbage_collection_is_performed_then_the_collection_and_pause_stats_and_reasons_are_updated() + { + // arrange + GC.Collect(1, GCCollectionMode.Forced); + GC.Collect(2, GCCollectionMode.Forced, true, true); + + // assert + Assert.That(() => MetricProducer.GcCollectionSeconds.CollectAllCountValues().Count(), Is.GreaterThanOrEqualTo(1).After(500, 10)); // at least 3 generations + Assert.That(() => MetricProducer.GcCollectionSeconds.CollectAllSumValues(excludeUnlabeled: true), Is.All.GreaterThan(0)); + Assert.That(() => MetricProducer.GcCollections.CollectAllValues(excludeUnlabeled: true), Is.All.GreaterThan(0)); + Assert.That(() => MetricProducer.GcPauseSeconds.CollectAllSumValues().Single(), Is.GreaterThan(0).After(500, 10)); + } + + [Test] + public void When_a_garbage_collection_is_performed_then_the_gc_cpu_and_pause_ratios_can_be_calculated() + { + // arrange + GC.Collect(2, GCCollectionMode.Forced, true, true); + + Assert.That(() => MetricProducer.GcPauseSeconds.CollectAllCountValues().First(), Is.GreaterThan(0).After(2000, 10)); + Assert.That(()=> MetricProducer.GcCollectionSeconds.CollectAllSumValues().Sum(x => x), Is.GreaterThan(0).After(2000, 10)); + + // To improve the reliability of the test, do some CPU busy work + call UpdateMetrics here. + // Why? Process.TotalProcessorTime isn't very precise (it's not updated after every small bit of CPU consumption) + // and this can lead to CpuRatio believing that no CPU has been consumed + long i = 2_000_000_000; + while (i > 0) + i--; + + // act + MetricProducer.UpdateMetrics(); + + // assert + Assert.That(MetricProducer.GcPauseRatio.Value, Is.GreaterThan(0.0).After(1000, 1), "GcPauseRatio"); + Assert.That(MetricProducer.GcCpuRatio.Value, Is.GreaterThan(0.0).After(1000, 1), "GcCpuRatio"); + } + + public class FinalizableTest + { + ~FinalizableTest() + { + // Sleep for a bit so our object won't exit the finalization queue immediately + Thread.Sleep(1000); + } + } + } + + internal class Given_Gc_Verbose_Events_Are_Available_For_GcStats : IntegrationTestBase + { + protected override DotNetRuntimeStatsBuilder.Builder ConfigureBuilder(DotNetRuntimeStatsBuilder.Builder toConfigure) + { + return toConfigure.WithGcStats(CaptureLevel.Verbose); + } + + [Test] + public void When_100kb_of_small_objects_are_allocated_then_the_allocated_bytes_counter_is_increased() + { + var previousValue = MetricProducer.AllocatedBytes.Labels("soh").Value; + + // allocate roughly 100kb+ of small objects + for (int i = 0; i < 11; i++) + { + var b = new byte[10_000]; + } + + Assert.That(() => MetricProducer.AllocatedBytes.Labels("soh").Value, Is.GreaterThanOrEqualTo(previousValue + 100_000).After(500, 10)); + } + + [Test] + public void When_a_100kb_large_object_is_allocated_then_the_allocated_bytes_counter_is_increased() + { + var previousValue = MetricProducer.AllocatedBytes.Labels("loh").Value; + + // allocate roughly 100kb+ of large objects + var b = new byte[110_000]; + + Assert.That(() => MetricProducer.AllocatedBytes.Labels("loh").Value, Is.GreaterThanOrEqualTo(previousValue + 100_000).After(500, 10)); + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/IntegrationTestBase.cs b/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/IntegrationTestBase.cs new file mode 100644 index 0000000..07e9df5 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/IntegrationTestBase.cs @@ -0,0 +1,53 @@ +using System; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Prometheus.DotNetRuntime.Metrics; + +namespace Prometheus.DotNetRuntime.Tests.IntegrationTests +{ + [TestFixture] + internal abstract class IntegrationTestBase + where TMetricProducer : IMetricProducer + { + private DotNetRuntimeStatsCollector _collector; + protected TMetricProducer MetricProducer { get; private set; } + + [SetUp] + public void SetUp() + { + _collector = (DotNetRuntimeStatsCollector) ConfigureBuilder(DotNetRuntimeStatsBuilder.Customize()) + .StartCollecting(Prometheus.Metrics.NewCustomRegistry()); + + MetricProducer = (TMetricProducer)_collector.ServiceProvider.GetServices().Single(x => x is TMetricProducer); + + // wait for event listener thread to spin up + var waitingFor = Stopwatch.StartNew(); + var waitFor = TimeSpan.FromSeconds(10); + + while (!_collector.EventListeners.All(x => x.StartedReceivingEvents)) + { + Thread.Sleep(10); + Console.Write("Waiting for event listeners to be active.. "); + + if (waitingFor.Elapsed > waitFor) + { + Assert.Fail($"Waited {waitFor} and still not all event listeners were ready! Event listeners not ready: {string.Join(", ", _collector.EventListeners.Where(x => !x.StartedReceivingEvents))}"); + return; + } + } + + Console.WriteLine("All event listeners should be active now."); + } + + [TearDown] + public void TearDown() + { + _collector.Dispose(); + } + + protected abstract DotNetRuntimeStatsBuilder.Builder ConfigureBuilder(DotNetRuntimeStatsBuilder.Builder toConfigure); + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/JitStatsCollectorTests.cs b/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/JitCompilerTests.cs similarity index 64% rename from src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/JitStatsCollectorTests.cs rename to src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/JitCompilerTests.cs index 1e16caa..56f1d5c 100644 --- a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/JitStatsCollectorTests.cs +++ b/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/JitCompilerTests.cs @@ -2,34 +2,32 @@ using System.Diagnostics; using System.Linq.Expressions; using System.Runtime.CompilerServices; -using System.Threading; using NUnit.Framework; -using Prometheus.DotNetRuntime; -using Prometheus.DotNetRuntime.StatsCollectors; +using Prometheus.DotNetRuntime.Metrics.Producers; -namespace Prometheus.DotNetRuntime.Tests.StatsCollectors.IntegrationTests +namespace Prometheus.DotNetRuntime.Tests.IntegrationTests { - internal class Given_A_JitStatsCollector_That_Samples_Every_Jit_Event : StatsCollectorIntegrationTestBase + internal class Given_A_JitStatsCollector_That_Samples_Every_Jit_Event : IntegrationTestBase { - protected override JitStatsCollector CreateStatsCollector() + protected override DotNetRuntimeStatsBuilder.Builder ConfigureBuilder(DotNetRuntimeStatsBuilder.Builder toConfigure) { - return new JitStatsCollector(SampleEvery.OneEvent); + return toConfigure.WithJitStats(SampleEvery.OneEvent); } [Test] public void When_a_method_is_jitted_then_its_compilation_is_measured() { // arrange - var methodsJitted = StatsCollector.MethodsJittedTotal.Labels("false").Value; - var methodsJittedSeconds = StatsCollector.MethodsJittedSecondsTotal.Labels("false").Value; + var methodsJitted = MetricProducer.MethodsJittedTotal.Labels("false").Value; + var methodsJittedSeconds = MetricProducer.MethodsJittedSecondsTotal.Labels("false").Value; // act (call a method, JIT'ing it) ToJit(); // assert - Assert.That(() => StatsCollector.MethodsJittedTotal.Labels("false").Value, Is.GreaterThanOrEqualTo(methodsJitted + 1).After(100, 10)); - Assert.That(StatsCollector.MethodsJittedSecondsTotal.Labels("false").Value, Is.GreaterThan(methodsJittedSeconds )); + Assert.That(() => MetricProducer.MethodsJittedTotal.Labels("false").Value, Is.GreaterThanOrEqualTo(methodsJitted + 1).After(100, 10)); + Assert.That(MetricProducer.MethodsJittedSecondsTotal.Labels("false").Value, Is.GreaterThan(methodsJittedSeconds )); } [Test] @@ -37,25 +35,25 @@ public void When_a_method_is_jitted_then_the_CPU_ratio_can_be_measured() { // act (call a method, JIT'ing it) ToJit(); - StatsCollector.UpdateMetrics(); + MetricProducer.UpdateMetrics(); // assert - Assert.That(() => StatsCollector.CpuRatio.Value, Is.GreaterThanOrEqualTo(0.0).After(100, 10)); + Assert.That(() => MetricProducer.CpuRatio.Value, Is.GreaterThanOrEqualTo(0.0).After(100, 10)); } [Test] public void When_a_dynamic_method_is_jitted_then_its_compilation_is_measured() { // arrange - var dynamicMethodsJitted = StatsCollector.MethodsJittedTotal.Labels("true").Value; - var dynamicMethodsJittedSeconds = StatsCollector.MethodsJittedSecondsTotal.Labels("true").Value; + var dynamicMethodsJitted = MetricProducer.MethodsJittedTotal.Labels("true").Value; + var dynamicMethodsJittedSeconds = MetricProducer.MethodsJittedSecondsTotal.Labels("true").Value; // act (call a method, JIT'ing it) ToJitDynamic(); // assert - Assert.That(() => StatsCollector.MethodsJittedTotal.Labels("true").Value, Is.GreaterThanOrEqualTo(dynamicMethodsJitted + 1).After(100, 10)); - Assert.That(StatsCollector.MethodsJittedSecondsTotal.Labels("true").Value, Is.GreaterThan(dynamicMethodsJittedSeconds )); + Assert.That(() => MetricProducer.MethodsJittedTotal.Labels("true").Value, Is.GreaterThanOrEqualTo(dynamicMethodsJitted + 1).After(100, 10)); + Assert.That(MetricProducer.MethodsJittedSecondsTotal.Labels("true").Value, Is.GreaterThan(dynamicMethodsJittedSeconds )); } [MethodImpl(MethodImplOptions.NoInlining)] @@ -72,19 +70,19 @@ private int ToJitDynamic() } } - internal class Given_A_JitStatsCollector_That_Samples_Every_Fifth_Jit_Event : StatsCollectorIntegrationTestBase + internal class Given_A_JitStatsCollector_That_Samples_Every_Fifth_Jit_Event : IntegrationTestBase { - protected override JitStatsCollector CreateStatsCollector() + protected override DotNetRuntimeStatsBuilder.Builder ConfigureBuilder(DotNetRuntimeStatsBuilder.Builder toConfigure) { - return new JitStatsCollector(SampleEvery.FiveEvents); + return toConfigure.WithJitStats(SampleEvery.FiveEvents); } [Test] public void When_many_methods_are_jitted_then_their_compilation_is_measured() { // arrange - var methodsJitted = StatsCollector.MethodsJittedTotal.Labels("true").Value; - var methodsJittedSeconds = StatsCollector.MethodsJittedSecondsTotal.Labels("true").Value; + var methodsJitted = MetricProducer.MethodsJittedTotal.Labels("true").Value; + var methodsJittedSeconds = MetricProducer.MethodsJittedSecondsTotal.Labels("true").Value; // act var sp = Stopwatch.StartNew(); @@ -92,8 +90,8 @@ public void When_many_methods_are_jitted_then_their_compilation_is_measured() sp.Stop(); // assert - Assert.That(() => StatsCollector.MethodsJittedTotal.Labels("true").Value, Is.GreaterThanOrEqualTo(methodsJitted + 20).After(100, 10)); - Assert.That(StatsCollector.MethodsJittedSecondsTotal.Labels("true").Value, Is.GreaterThan(methodsJittedSeconds + sp.Elapsed.TotalSeconds).Within(0.1)); + Assert.That(() => MetricProducer.MethodsJittedTotal.Labels("true").Value, Is.GreaterThanOrEqualTo(methodsJitted + 20).After(100, 10)); + Assert.That(MetricProducer.MethodsJittedSecondsTotal.Labels("true").Value, Is.GreaterThan(methodsJittedSeconds + sp.Elapsed.TotalSeconds).Within(0.1)); } private void Compile100Methods(Expression> toCompile) diff --git a/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/ThreadPoolTests.cs b/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/ThreadPoolTests.cs new file mode 100644 index 0000000..490d8a8 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/ThreadPoolTests.cs @@ -0,0 +1,107 @@ +using System; +using System.Linq; +using System.Net.Http; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Prometheus.DotNetRuntime.Metrics; +using Prometheus.DotNetRuntime.Metrics.Producers; + +namespace Prometheus.DotNetRuntime.Tests.IntegrationTests +{ + internal class Given_Only_Runtime_Counters_Are_Enabled_For_ThreadPoolStats : IntegrationTestBase + { + protected override DotNetRuntimeStatsBuilder.Builder ConfigureBuilder(DotNetRuntimeStatsBuilder.Builder toConfigure) + { + return toConfigure.WithThreadPoolStats(CaptureLevel.Counters); + } + + [Test] + public async Task When_work_is_executed_on_the_thread_pool_then_executed_work_is_measured() + { + var startingThroughput = MetricProducer.Throughput.Value; + const int numTasksToSchedule = 100; + // schedule a bunch of tasks + var tasks = Enumerable.Range(1, numTasksToSchedule) + .Select(_ => Task.Run(() => { })); + + await Task.WhenAll(tasks); + + Assert.That(() => MetricProducer.NumThreads.Value, Is.GreaterThanOrEqualTo(Environment.ProcessorCount).After(2_000, 10)); + Assert.That(() => MetricProducer.Throughput.Value, Is.GreaterThanOrEqualTo(startingThroughput + numTasksToSchedule).After(2_000, 10)); + } + + [Test] + public async Task When_timers_are_active_then_they_are_measured() + { + var startingTimers = MetricProducer.NumTimers.Value; + const int numTimersToSchedule = 100; + // schedule a bunch of timers + var tasks = Enumerable.Range(1, numTimersToSchedule) + .Select(n => Task.Delay(3000 + n)) + .ToArray(); + + Assert.That(() => MetricProducer.NumTimers.Value, Is.GreaterThanOrEqualTo(startingTimers + numTimersToSchedule).After(2_000, 10)); + } + + [Test] + public async Task When_blocking_work_is_executed_on_the_thread_pool_then_thread_pool_delays_are_measured() + { + var startingQueueLength = MetricProducer.QueueLength.Sum; + var sleepDelay = TimeSpan.FromMilliseconds(250); + int desiredSecondsToBlock = 5; + int numTasksToSchedule = (int)(Environment.ProcessorCount / sleepDelay.TotalSeconds) * desiredSecondsToBlock; + + Console.WriteLine($"Scheduling {numTasksToSchedule} blocking tasks..."); + // schedule a bunch of blocking tasks that will make the thread pool will grow + var tasks = Enumerable.Range(1, numTasksToSchedule) + .Select(_ => Task.Run(() => Thread.Sleep(sleepDelay))) + .ToArray(); + + // dont' wait for the tasks to complete- we want to see stats present during a period of thread pool starvation + + Assert.That(() => MetricProducer.NumThreads.Value, Is.GreaterThan(Environment.ProcessorCount).After(desiredSecondsToBlock * 1000, 10)); + Assert.That(() => MetricProducer.QueueLength.Sum, Is.GreaterThan(startingQueueLength).After(desiredSecondsToBlock * 1000, 10)); + } + } + + internal class Given_Runtime_Counters_And_ThreadPool_Info_Events_Are_Enabled_For_ThreadPoolStats : IntegrationTestBase + { + protected override DotNetRuntimeStatsBuilder.Builder ConfigureBuilder(DotNetRuntimeStatsBuilder.Builder toConfigure) + { + return toConfigure.WithThreadPoolStats(CaptureLevel.Informational); + } + + [Test] + public async Task When_work_is_executed_on_the_thread_pool_then_executed_work_is_measured() + { + // schedule a bunch of blocking tasks that will make the thread pool will grow + var tasks = Enumerable.Range(1, 1000) + .Select(_ => Task.Run(() => Thread.Sleep(20))); + + await Task.WhenAll(tasks); + + Assert.That(() => MetricProducer.NumThreads.Value, Is.GreaterThanOrEqualTo(Environment.ProcessorCount).After(2000, 10)); + Assert.That(MetricProducer.AdjustmentsTotal.CollectAllValues().Sum(), Is.GreaterThanOrEqualTo(1)); + } + + [Test] + public async Task When_IO_work_is_executed_on_the_thread_pool_then_the_number_of_io_threads_is_measured() + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + Assert.Inconclusive("Cannot run this test on non-windows platforms."); + + // need to schedule a bunch of IO work to make the IO pool grow + using (var client = new HttpClient()) + { + var httpTasks = Enumerable.Range(1, 50) + .Select(_ => client.GetAsync("http://google.com")); + + await Task.WhenAll(httpTasks); + } + + Assert.That(() => MetricProducer.NumIocThreads.Value, Is.GreaterThanOrEqualTo(1).After(2000, 10)); + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/Util/LabelGeneratorTests.cs b/src/prometheus-net.DotNetRuntime.Tests/Metrics/Producers/Util/LabelGeneratorTests.cs similarity index 75% rename from src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/Util/LabelGeneratorTests.cs rename to src/prometheus-net.DotNetRuntime.Tests/Metrics/Producers/Util/LabelGeneratorTests.cs index 054bd17..0f30e95 100644 --- a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/Util/LabelGeneratorTests.cs +++ b/src/prometheus-net.DotNetRuntime.Tests/Metrics/Producers/Util/LabelGeneratorTests.cs @@ -1,8 +1,8 @@ using NUnit.Framework; -using Prometheus.DotNetRuntime.EventSources; -using Prometheus.DotNetRuntime.StatsCollectors.Util; +using Prometheus.DotNetRuntime.EventListening.EventSources; +using Prometheus.DotNetRuntime.Metrics.Producers.Util; -namespace Prometheus.DotNetRuntime.Tests.StatsCollectors.Util +namespace Prometheus.DotNetRuntime.Tests.Metrics.Producers.Util { [TestFixture] public class LabelGeneratorTests diff --git a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/Util/RatioTests.cs b/src/prometheus-net.DotNetRuntime.Tests/Metrics/Producers/Util/RatioTests.cs similarity index 94% rename from src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/Util/RatioTests.cs rename to src/prometheus-net.DotNetRuntime.Tests/Metrics/Producers/Util/RatioTests.cs index 4682bbc..dc317c0 100644 --- a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/Util/RatioTests.cs +++ b/src/prometheus-net.DotNetRuntime.Tests/Metrics/Producers/Util/RatioTests.cs @@ -3,12 +3,9 @@ using System.Linq; using System.Threading; using NUnit.Framework; -#if PROMV2 -using Prometheus.Advanced; -#endif -using Prometheus.DotNetRuntime.StatsCollectors.Util; +using Prometheus.DotNetRuntime.Metrics.Producers.Util; -namespace Prometheus.DotNetRuntime.Tests.StatsCollectors.Util +namespace Prometheus.DotNetRuntime.Tests.Metrics.Producers.Util { [TestFixture] public class RatioTests @@ -18,11 +15,7 @@ public class RatioTests [SetUp] public void SetUp() { -#if PROMV2 - _metricFactory = new MetricFactory(new DefaultCollectorRegistry()); -#elif PROMV3 - _metricFactory = Metrics.WithCustomRegistry(Metrics.NewCustomRegistry()); -#endif + _metricFactory = Prometheus.Metrics.WithCustomRegistry(Prometheus.Metrics.NewCustomRegistry()); } [Test] diff --git a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/Util/StringExtensionsTests.cs b/src/prometheus-net.DotNetRuntime.Tests/Metrics/Producers/Util/StringExtensionsTests.cs similarity index 80% rename from src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/Util/StringExtensionsTests.cs rename to src/prometheus-net.DotNetRuntime.Tests/Metrics/Producers/Util/StringExtensionsTests.cs index 15d7b19..94761c3 100644 --- a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/Util/StringExtensionsTests.cs +++ b/src/prometheus-net.DotNetRuntime.Tests/Metrics/Producers/Util/StringExtensionsTests.cs @@ -1,7 +1,7 @@ using NUnit.Framework; -using Prometheus.DotNetRuntime.StatsCollectors.Util; +using Prometheus.DotNetRuntime.Metrics.Producers.Util; -namespace Prometheus.DotNetRuntime.Tests.StatsCollectors.Util +namespace Prometheus.DotNetRuntime.Tests.Metrics.Producers.Util { public class StringExtensionsTests { diff --git a/src/prometheus-net.DotNetRuntime.Tests/Properties.cs b/src/prometheus-net.DotNetRuntime.Tests/Properties.cs new file mode 100644 index 0000000..98c73c1 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime.Tests/Properties.cs @@ -0,0 +1,3 @@ +using NUnit.Framework; + +[assembly: Timeout(30_000)] \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/ContentionStatsCollectorTests.cs b/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/ContentionStatsCollectorTests.cs deleted file mode 100644 index 5d17fee..0000000 --- a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/ContentionStatsCollectorTests.cs +++ /dev/null @@ -1,73 +0,0 @@ -using System; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using NUnit.Framework; -using Prometheus.DotNetRuntime.StatsCollectors; - -namespace Prometheus.DotNetRuntime.Tests.StatsCollectors.IntegrationTests -{ - [TestFixture] - internal class ContentionStatsCollectorTests : StatsCollectorIntegrationTestBase - { - protected override ContentionStatsCollector CreateStatsCollector() - { - return new ContentionStatsCollector(SampleEvery.OneEvent); - } - - [Test] - public void Will_measure_no_contention_on_an_uncontested_lock() - { - // arrange - var key = new Object(); - - // act - lock (key) - { - } - - // assert - Assert.That(StatsCollector.ContentionTotal.Value, Is.EqualTo(0)); - Assert.That(StatsCollector.ContentionSecondsTotal.Value, Is.EqualTo(0)); - } - - /// - /// This test has the potential to be flaky (due to attempting to simulate lock contention across multiple threads in the thread pool), - /// may have to revisit this in the future.. - /// - /// - [Test] - [Repeat(5)] - public async Task Will_measure_contention_on_a_contested_lock() - { - // arrange - const int numThreads = 10; - const int sleepForMs = 50; - var key = new object(); - // Increase the min. thread pool size so that when we use Thread.Sleep, we don't run into scheduling delays - ThreadPool.SetMinThreads(numThreads * 2, 1); - - // act - var tasks = Enumerable.Range(1, numThreads) - .Select(_ => Task.Run(() => - { - lock (key) - { - Thread.Sleep(sleepForMs); - } - })); - - await Task.WhenAll(tasks); - - // assert - - // Why -1? The first thread will not contend the lock - const int numLocksContended = numThreads - 1; - Assert.That(() => StatsCollector.ContentionTotal.Value, Is.GreaterThanOrEqualTo(numLocksContended).After(200, 10)); - - // Pattern of expected contention times is: 50ms, 100ms, 150ms, etc. - var expectedDelay = TimeSpan.FromMilliseconds(Enumerable.Range(1, numLocksContended).Aggregate(sleepForMs, (acc, next) => acc + (sleepForMs * next))); - Assert.That(StatsCollector.ContentionSecondsTotal.Value, Is.EqualTo(expectedDelay.TotalSeconds).Within(sleepForMs)); - } - } -} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/ExceptionStatsCollectorTests.cs b/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/ExceptionStatsCollectorTests.cs deleted file mode 100644 index 359e2c1..0000000 --- a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/ExceptionStatsCollectorTests.cs +++ /dev/null @@ -1,33 +0,0 @@ -using NUnit.Framework; -using Prometheus.DotNetRuntime.StatsCollectors; -using System; - -namespace Prometheus.DotNetRuntime.Tests.StatsCollectors.IntegrationTests -{ - [TestFixture] - internal class ExceptionStatsCollectorTests : StatsCollectorIntegrationTestBase - { - protected override ExceptionStatsCollector CreateStatsCollector() - { - return new ExceptionStatsCollector(); - } - - [Test] - public void Will_measure_when_occurring_an_exception() - { - // act - var divider = 0; - - try - { - _ = 1 / divider; - } - catch (DivideByZeroException) - { - } - - // assert - Assert.That(() => StatsCollector.ExceptionCount.Labels("System.DivideByZeroException").Value, Is.EqualTo(1).After(100, 1000)); - } - } -} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/GcStatsCollectorTests.cs b/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/GcStatsCollectorTests.cs deleted file mode 100644 index 2a2f1d3..0000000 --- a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/GcStatsCollectorTests.cs +++ /dev/null @@ -1,124 +0,0 @@ -using System; -using System.Linq; -using System.Runtime.InteropServices; -using System.Threading; -using NUnit.Framework; -using Prometheus.DotNetRuntime.StatsCollectors; - -namespace Prometheus.DotNetRuntime.Tests.StatsCollectors.IntegrationTests -{ - internal class GcStatsCollectorTests : StatsCollectorIntegrationTestBase - { - protected override GcStatsCollector CreateStatsCollector() - { - return new GcStatsCollector(); - } - - [Test] - public void When_100kb_of_small_objects_are_allocated_then_the_allocated_bytes_counter_is_increased() - { - var previousValue = StatsCollector.AllocatedBytes.Labels("soh").Value; - - // allocate roughly 100kb+ of small objects - for (int i = 0; i < 11; i++) - { - var b = new byte[10_000]; - } - - Assert.That(() => StatsCollector.AllocatedBytes.Labels("soh").Value, Is.GreaterThanOrEqualTo(previousValue + 100_000).After(500, 10)); - } - - [Test] - public void When_a_100kb_large_object_is_allocated_then_the_allocated_bytes_counter_is_increased() - { - var previousValue = StatsCollector.AllocatedBytes.Labels("loh").Value; - - // allocate roughly 100kb+ of large objects - var b = new byte[110_000]; - - Assert.That(() => StatsCollector.AllocatedBytes.Labels("loh").Value, Is.GreaterThanOrEqualTo(previousValue + 100_000).After(500, 10)); - } - - [Test] - public void When_a_garbage_collection_is_performed_then_the_heap_sizes_are_updated() - { - unsafe - { - // arrange (fix a variable to ensure the pinned objects counter is incremented - var b = new byte[1]; - fixed (byte* p = b) - { - // act - GC.Collect(0); - } - - Assert.That(() => StatsCollector.GcHeapSizeBytes.Labels("0").Value, Is.GreaterThan(0).After(200, 10)); - Assert.That(() => StatsCollector.GcHeapSizeBytes.Labels("1").Value, Is.GreaterThan(0).After(200, 10)); - Assert.That(() => StatsCollector.GcHeapSizeBytes.Labels("2").Value, Is.GreaterThan(0).After(200, 10)); - Assert.That(() => StatsCollector.GcHeapSizeBytes.Labels("loh").Value, Is.GreaterThan(0).After(200, 10)); - Assert.That(() => StatsCollector.GcNumPinnedObjects.Value, Is.GreaterThan(0).After(200, 10)); - } - } - - [Test] - public void When_a_garbage_collection_is_performed_then_the_finalization_queue_is_updated() - { - // arrange - { - var finalizable = new FinalizableTest(); - finalizable = null; - } - GC.Collect(0); - - // assert - Assert.That(() => StatsCollector.GcFinalizationQueueLength.Value, Is.GreaterThan(0).After(200, 10)); - } - - [Test] - public void When_a_garbage_collection_is_performed_then_the_collection_and_pause_stats_and_reasons_are_updated() - { - // arrange - GC.Collect(1, GCCollectionMode.Forced); - GC.Collect(2, GCCollectionMode.Forced, true, true); - - // assert - Assert.That(() => StatsCollector.GcCollectionSeconds.CollectAllCountValues().Count(), Is.GreaterThanOrEqualTo(1).After(500, 10)); // at least 3 generations - Assert.That(() => StatsCollector.GcCollectionSeconds.CollectAllSumValues(excludeUnlabeled: true), Is.All.GreaterThan(0)); - Assert.That(() => StatsCollector.GcCollectionReasons.CollectAllValues(excludeUnlabeled: true), Is.All.GreaterThan(0)); - Assert.That(() => StatsCollector.GcPauseSeconds.CollectAllSumValues().Single(), Is.GreaterThan(0).After(500, 10)); - } - - [Test] - public void When_a_garbage_collection_is_performed_then_the_gc_cpu_and_pause_ratios_can_be_calculated() - { - // arrange - GC.Collect(2, GCCollectionMode.Forced, true, true); - - Assert.That(() => StatsCollector.GcPauseSeconds.CollectAllCountValues().First(), Is.GreaterThan(0).After(2000, 10)); - Assert.That(()=> StatsCollector.GcCollectionSeconds.CollectAllSumValues().Sum(x => x), Is.GreaterThan(0).After(2000, 10)); - - // To improve the reliability of the test, do some CPU busy work + call UpdateMetrics here. - // Why? Process.TotalProcessorTime isn't very precise (it's not updated after every small bit of CPU consumption) - // and this can lead to CpuRatio believing that no CPU has been consumed - long i = 2_000_000_000; - while (i > 0) - i--; - - // act - StatsCollector.UpdateMetrics(); - - // assert - Assert.That(StatsCollector.GcPauseRatio.Value, Is.GreaterThan(0.0).After(1000, 1), "GcPauseRatio"); - Assert.That(StatsCollector.GcCpuRatio.Value, Is.GreaterThan(0.0).After(1000, 1), "GcCpuRatio"); - } - - public class FinalizableTest - { - ~FinalizableTest() - { - // Sleep for a bit so our object won't exit the finalization queue immediately - Thread.Sleep(1000); - } - } - } -} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/StatsCollectorIntegrationTestBase.cs b/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/StatsCollectorIntegrationTestBase.cs deleted file mode 100644 index 1fb1027..0000000 --- a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/StatsCollectorIntegrationTestBase.cs +++ /dev/null @@ -1,49 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using NUnit.Framework; -#if PROMV2 -using Prometheus.Advanced; -#endif -using Prometheus.DotNetRuntime; -using Prometheus.DotNetRuntime.StatsCollectors; - -namespace Prometheus.DotNetRuntime.Tests.StatsCollectors.IntegrationTests -{ - [TestFixture] - internal abstract class StatsCollectorIntegrationTestBase - where TStatsCollector : IEventSourceStatsCollector - { - private DotNetEventListener _eventListener; - protected TStatsCollector StatsCollector { get; private set; } - - [SetUp] - public void SetUp() - { - StatsCollector = CreateStatsCollector(); -#if PROMV2 - StatsCollector.RegisterMetrics(new MetricFactory(new DefaultCollectorRegistry())); -#elif PROMV3 - StatsCollector.RegisterMetrics(Metrics.WithCustomRegistry(Metrics.NewCustomRegistry())); -#endif - _eventListener = new DotNetEventListener(StatsCollector, null, false); - - // wait for event listener thread to spin up - while (!_eventListener.StartedReceivingEvents) - { - Thread.Sleep(10); - Console.Write("Waiting.. "); - - } - Console.WriteLine("EventListener should be active now."); - } - - [TearDown] - public void TearDown() - { - _eventListener.Dispose(); - } - - protected abstract TStatsCollector CreateStatsCollector(); - } -} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/ThreadPoolSchedulingStatsCollectorTests.cs b/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/ThreadPoolSchedulingStatsCollectorTests.cs deleted file mode 100644 index b35ce31..0000000 --- a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/ThreadPoolSchedulingStatsCollectorTests.cs +++ /dev/null @@ -1,69 +0,0 @@ -using System.Diagnostics; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using NUnit.Framework; -using Prometheus.DotNetRuntime.StatsCollectors; -using Prometheus.DotNetRuntime.StatsCollectors.Util; - -namespace Prometheus.DotNetRuntime.Tests.StatsCollectors.IntegrationTests -{ - [TestFixture] - internal class Given_A_ThreadPoolSchedulingStatsCollector_That_Samples_Every_Event : StatsCollectorIntegrationTestBase - { - protected override ThreadPoolSchedulingStatsCollector CreateStatsCollector() - { - return new ThreadPoolSchedulingStatsCollector(); - } - - [Test] - [Repeat(5)] - public async Task When_work_is_queued_on_the_thread_pool_then_the_queued_and_scheduled_work_is_measured() - { - Assert.That(StatsCollector.ScheduledCount.Value, Is.EqualTo(0)); - - // act (Task.Run will execute the function on the thread pool) - // There seems to be either a bug in the implementation of .NET core or a bug in my understanding... - // First call to Task.Run triggers a queued event but not a queue event. For now, call twice - await Task.Run(() => 1 ); - var sp = Stopwatch.StartNew(); - await Task.Run(() => sp.Stop()); - sp.Stop(); - - Assert.That(() => StatsCollector.ScheduledCount.Value, Is.GreaterThanOrEqualTo(1).After(100, 10)); - Assert.That(StatsCollector.ScheduleDelay.CollectAllCountValues().Single(), Is.GreaterThanOrEqualTo(1)); - Assert.That(StatsCollector.ScheduleDelay.CollectAllSumValues().Single(), Is.EqualTo(sp.Elapsed.TotalSeconds).Within(0.01)); - } - } - - [TestFixture] - internal class Given_A_ThreadPoolSchedulingStatsCollector_That_Samples_Fifth_Event : StatsCollectorIntegrationTestBase - { - protected override ThreadPoolSchedulingStatsCollector CreateStatsCollector() - { - return new ThreadPoolSchedulingStatsCollector(Constants.DefaultHistogramBuckets, SampleEvery.FiveEvents); - } - - [Test] - public async Task When_many_items_of_work_is_queued_on_the_thread_pool_then_the_queued_and_scheduled_work_is_measured() - { - Assert.That(StatsCollector.ScheduledCount.Value, Is.EqualTo(0)); - - // act (Task.Run will execute the function on the thread pool) - // There seems to be either a bug in the implementation of .NET core or a bug in my understanding... - // First call to Task.Run triggers a queued event but not a queue event. For now, call twice - await Task.Run(() => 1 ); - - var sp = Stopwatch.StartNew(); - for (int i = 0; i < 100; i++) - { - sp.Start(); - await Task.Run(() => sp.Stop()); - } - - Assert.That(() => StatsCollector.ScheduledCount.Value, Is.GreaterThanOrEqualTo(100).After(100, 10)); - Assert.That(StatsCollector.ScheduleDelay.CollectAllCountValues().Single(), Is.GreaterThanOrEqualTo(100)); - Assert.That(StatsCollector.ScheduleDelay.CollectAllSumValues().Single(), Is.EqualTo(sp.Elapsed.TotalSeconds).Within(0.01)); - } - } -} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/ThreadPoolStatsCollectorTests.cs b/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/ThreadPoolStatsCollectorTests.cs deleted file mode 100644 index 1164916..0000000 --- a/src/prometheus-net.DotNetRuntime.Tests/StatsCollectors/IntegrationTests/ThreadPoolStatsCollectorTests.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System; -using System.Diagnostics; -using System.Linq; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using NUnit.Framework; -using Prometheus.DotNetRuntime.StatsCollectors; - -namespace Prometheus.DotNetRuntime.Tests.StatsCollectors.IntegrationTests -{ - internal class ThreadPoolStatsCollectorTests : StatsCollectorIntegrationTestBase - { - protected override ThreadPoolStatsCollector CreateStatsCollector() - { - return new ThreadPoolStatsCollector(); - } - - [Test] - public async Task When_work_is_executed_on_the_thread_pool_then_executed_work_is_measured() - { - // schedule a bunch of blocking tasks that will make the thread pool will grow - var tasks = Enumerable.Range(1, 1000) - .Select(_ => Task.Run(() => Thread.Sleep(20))); - - await Task.WhenAll(tasks); - - Assert.That(() => StatsCollector.NumThreads.Value, Is.GreaterThanOrEqualTo(Environment.ProcessorCount).After(2000, 10)); - Assert.That(StatsCollector.AdjustmentsTotal.CollectAllValues().Sum(), Is.GreaterThanOrEqualTo(1)); - } - - [Test] - public async Task When_IO_work_is_executed_on_the_thread_pool_then_the_number_of_io_threads_is_measured() - { - // need to schedule a bunch of IO work to make the IO pool grow - using (var client = new HttpClient()) - { - var httpTasks = Enumerable.Range(1, 50) - .Select(_ => client.GetAsync("http://google.com")); - - await Task.WhenAll(httpTasks); - } - - Assert.That(() => StatsCollector.NumIocThreads.Value, Is.GreaterThanOrEqualTo(1).After(2000, 10)); - } - } -} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime.Tests/prometheus-net.DotNetRuntime.Tests.csproj b/src/prometheus-net.DotNetRuntime.Tests/prometheus-net.DotNetRuntime.Tests.csproj index 750859b..83c6b6a 100644 --- a/src/prometheus-net.DotNetRuntime.Tests/prometheus-net.DotNetRuntime.Tests.csproj +++ b/src/prometheus-net.DotNetRuntime.Tests/prometheus-net.DotNetRuntime.Tests.csproj @@ -1,8 +1,8 @@  - + - net5.0 + netcoreapp3.1;net5.0 false Prometheus.DotNetRuntime.Tests AnyCPU @@ -10,13 +10,17 @@ - - - + + + + + + + diff --git a/src/prometheus-net.DotNetRuntime/CaptureLevel.cs b/src/prometheus-net.DotNetRuntime/CaptureLevel.cs new file mode 100644 index 0000000..f273a5c --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/CaptureLevel.cs @@ -0,0 +1,26 @@ +using System.Diagnostics.Tracing; + +namespace Prometheus.DotNetRuntime +{ + /// + /// Specifies the fidelity of events captured. + /// + /// + /// In order to produce metrics this library collects events from the .NET runtime. The level chosen impacts both the performance of + /// your application (the more detailed events .NET produces the more CPU it consumes to produce them) and the level of detail present in the metrics + /// produced by this library (the more detailed events prometheus-net.DotNetRuntime captures, the more analysis it can perform). + /// + public enum CaptureLevel + { + /// + /// Collect event counters only- limited metrics will be available. + /// + Counters = EventLevel.LogAlways, + Errors = EventLevel.Error, + Informational = EventLevel.Informational, + /// + /// Collects events at level Verbose and all other levels- produces the highest level of metric detail. + /// + Verbose = EventLevel.Verbose, + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/DotNetEventListener.cs b/src/prometheus-net.DotNetRuntime/DotNetEventListener.cs deleted file mode 100644 index c9fe4dd..0000000 --- a/src/prometheus-net.DotNetRuntime/DotNetEventListener.cs +++ /dev/null @@ -1,76 +0,0 @@ -using Prometheus.DotNetRuntime.StatsCollectors.Util; -using System; -using System.Diagnostics; -using System.Diagnostics.Tracing; - -namespace Prometheus.DotNetRuntime -{ - internal sealed class DotNetEventListener : EventListener - { - private static Counter _eventTypeCounts; - private static Counter _cpuConsumed; - - private readonly IEventSourceStatsCollector _collector; - private readonly Action _errorHandler; - private readonly bool _enableDebugging; - private readonly string _nameSnakeCase; - - internal DotNetEventListener(IEventSourceStatsCollector collector, Action errorHandler, bool enableDebugging) : base() - { - _collector = collector; - _errorHandler = errorHandler; - _enableDebugging = enableDebugging; - - if (_enableDebugging) - { - _eventTypeCounts ??= Metrics.CreateCounter($"dotnet_debug_events_total", "The total number of .NET diagnostic events processed", "collector_name", "event_source_name", "event_name"); - _cpuConsumed ??= Metrics.CreateCounter("dotnet_debug_cpu_seconds_total", "The total CPU time consumed by processing .NET diagnostic events (does not include the CPU cost to generate the events)", "collector_name", "event_source_name", "event_name"); - _nameSnakeCase = collector.GetType().Name.ToSnakeCase(); - } - EventSourceCreated += OnEventSourceCreated; - } - - internal bool StartedReceivingEvents { get; private set; } - - private void OnEventSourceCreated(object sender, EventSourceCreatedEventArgs e) - { - var es = e.EventSource; - if (es.Guid == _collector.EventSourceGuid) - { - EnableEvents(es, _collector.Level, _collector.Keywords); - StartedReceivingEvents = true; - } - } - - protected override void OnEventWritten(EventWrittenEventArgs eventData) - { - var sp = new Stopwatch(); - try - { - if (_enableDebugging) - { - _eventTypeCounts.Labels(_nameSnakeCase, eventData.EventSource.Name, eventData.EventName).Inc(); - sp.Restart(); - } - - _collector.ProcessEvent(eventData); - - if (_enableDebugging) - { - sp.Stop(); - _cpuConsumed.Labels(_nameSnakeCase, eventData.EventSource.Name, eventData.EventName).Inc(sp.Elapsed.TotalSeconds); - } - } - catch (Exception e) - { - _errorHandler(e); - } - } - - public override void Dispose() - { - EventSourceCreated -= OnEventSourceCreated; - base.Dispose(); - } - } -} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/DotNetRuntimeStatsBuilder.cs b/src/prometheus-net.DotNetRuntime/DotNetRuntimeStatsBuilder.cs index 76e3963..9667821 100644 --- a/src/prometheus-net.DotNetRuntime/DotNetRuntimeStatsBuilder.cs +++ b/src/prometheus-net.DotNetRuntime/DotNetRuntimeStatsBuilder.cs @@ -1,13 +1,14 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; -using Prometheus.DotNetRuntime.StatsCollectors; -using Prometheus.DotNetRuntime.StatsCollectors.Util; -#if PROMV2 -using TCollectorRegistry = Prometheus.Advanced.DefaultCollectorRegistry; -#elif PROMV3 -using TCollectorRegistry = Prometheus.CollectorRegistry; -#endif +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Prometheus.DotNetRuntime.EventListening; +using Prometheus.DotNetRuntime.EventListening.Parsers; +using Prometheus.DotNetRuntime.Metrics; +using Prometheus.DotNetRuntime.Metrics.Producers; +using Prometheus.DotNetRuntime.Metrics.Producers.Util; namespace Prometheus.DotNetRuntime { @@ -17,16 +18,14 @@ namespace Prometheus.DotNetRuntime public static class DotNetRuntimeStatsBuilder { /// - /// Includes all available .NET runtime metrics by default. Call - /// to begin collecting metrics. + /// Includes all .NET runtime metrics that can be collected at the capture level, + /// ensuring minimal impact on performance. Call to begin collecting metrics. /// /// public static Builder Default() { return Customize() .WithContentionStats() - .WithJitStats() - .WithThreadPoolSchedulingStats() .WithThreadPoolStats() .WithGcStats() .WithExceptionStats(); @@ -46,9 +45,17 @@ public static Builder Customize() public class Builder { - private Action _errorHandler; - private bool _debugMetrics; - internal HashSet StatsCollectors { get; } = new HashSet(new TypeEquality()); + private readonly DotNetRuntimeStatsCollector.Options _options = new(); + internal HashSet ListenerRegistrations { get; } = new(); + private readonly IServiceCollection _services = new ServiceCollection(); + + public Builder() + { + // For now, we include runtime events by default. May make this customizable in the future. + ListenerRegistrations.Add(ListenerRegistration.Create(CaptureLevel.Counters, sp => new RuntimeEventParser() { RefreshIntervalSeconds = 1})); + + // TODO what if we want to support extensibility? e.g. add your own custom metric generator + } /// /// Finishes configuration and starts collecting .NET runtime metrics. Returns a that @@ -57,11 +64,7 @@ public class Builder /// public IDisposable StartCollecting() { -#if PROMV2 - return StartCollecting(TCollectorRegistry.Instance); -#elif PROMV3 - return StartCollecting(Metrics.DefaultRegistry); -#endif + return StartCollecting(Prometheus.Metrics.DefaultRegistry); } /// @@ -70,56 +73,57 @@ public IDisposable StartCollecting() /// /// Registry where metrics will be collected /// - public IDisposable StartCollecting(TCollectorRegistry registry) + public IDisposable StartCollecting(CollectorRegistry registry) { - var runtimeStatsCollector = new DotNetRuntimeStatsCollector(StatsCollectors.ToImmutableHashSet(), _errorHandler, _debugMetrics, registry); -#if PROMV2 - registry.RegisterOnDemandCollectors(runtimeStatsCollector); -#elif PROMV3 - runtimeStatsCollector.RegisterMetrics(registry); - registry.AddBeforeCollectCallback(runtimeStatsCollector.UpdateMetrics); -#endif - + var serviceProvider = BuildServiceProvider(); + var runtimeStatsCollector = new DotNetRuntimeStatsCollector(serviceProvider, registry, _options); return runtimeStatsCollector; } - /// - /// Include metrics around the volume of work scheduled on the worker thread pool - /// and the scheduling delays. - /// - /// Buckets for the scheduling delay histogram - /// - /// The sampling rate for thread pool scheduling events. A lower sampling rate reduces memory use - /// but reduces the accuracy of metrics produced (as a percentage of events are discarded). - /// If your application achieves a high level of throughput (thousands of work items scheduled per second on - /// the thread pool), it's recommend to reduce the sampling rate even further. - /// - public Builder WithThreadPoolSchedulingStats(double[] histogramBuckets = null, SampleEvery sampleRate = SampleEvery.TenEvents) - { - StatsCollectors.AddOrReplace(new ThreadPoolSchedulingStatsCollector(histogramBuckets ?? Constants.DefaultHistogramBuckets, sampleRate)); - return this; - } - /// /// Include metrics around the size of the worker and IO thread pools and reasons /// for worker thread pool changes. /// - public Builder WithThreadPoolStats() + public Builder WithThreadPoolStats(CaptureLevel level = CaptureLevel.Counters, ThreadPoolMetricsProducer.Options options = null) { - StatsCollectors.AddOrReplace(new ThreadPoolStatsCollector()); + try + { + if (level != CaptureLevel.Counters) + ListenerRegistrations.AddOrReplace(ListenerRegistration.Create(level, sp => new ThreadPoolEventParser())); + } + catch (UnsupportedEventParserLevelException ex) + { + throw UnsupportedCaptureLevelException.CreateWithCounterSupport(ex); + } + + _services.TryAddSingletonEnumerable(); + _services.AddSingleton(options ?? new ThreadPoolMetricsProducer.Options()); + return this; } /// /// Include metrics around volume of locks contended. /// + /// /// /// The sampling rate for contention events (defaults to 100%). A lower sampling rate reduces memory use /// but reduces the accuracy of metrics produced (as a percentage of events are discarded). /// - public Builder WithContentionStats(SampleEvery sampleRate = SampleEvery.TwoEvents) + public Builder WithContentionStats(CaptureLevel level = CaptureLevel.Counters, SampleEvery sampleRate = SampleEvery.TwoEvents) { - StatsCollectors.AddOrReplace(new ContentionStatsCollector(sampleRate)); + try + { + if (level != CaptureLevel.Counters) + ListenerRegistrations.AddOrReplace(ListenerRegistration.Create(CaptureLevel.Informational, sp => new ContentionEventParser(sampleRate))); + } + catch (UnsupportedEventParserLevelException ex) + { + throw new UnsupportedCaptureLevelException(ex); + } + + _services.TryAddSingletonEnumerable(); + return this; } @@ -135,7 +139,9 @@ public Builder WithContentionStats(SampleEvery sampleRate = SampleEvery.TwoEvent /// public Builder WithJitStats(SampleEvery sampleRate = SampleEvery.TenEvents) { - StatsCollectors.AddOrReplace(new JitStatsCollector(sampleRate)); + ListenerRegistrations.AddOrReplace(ListenerRegistration.Create(CaptureLevel.Verbose, sp => new JitEventParser(sampleRate))); + _services.TryAddSingletonEnumerable(); + return this; } @@ -143,25 +149,46 @@ public Builder WithJitStats(SampleEvery sampleRate = SampleEvery.TenEvents) /// Include metrics recording the frequency and duration of garbage collections/ pauses, heap sizes and /// volume of allocations. /// + /// /// Buckets for the GC collection and pause histograms - public Builder WithGcStats(double[] histogramBuckets = null) + public Builder WithGcStats(CaptureLevel atLevel = CaptureLevel.Counters, double[] histogramBuckets = null) { - StatsCollectors.AddOrReplace(new GcStatsCollector(histogramBuckets ?? Constants.DefaultHistogramBuckets)); + try + { + if (atLevel != CaptureLevel.Counters) + ListenerRegistrations.AddOrReplace(ListenerRegistration.Create(atLevel, sp => new GcEventParser())); + } + catch (UnsupportedEventParserLevelException ex) + { + throw new UnsupportedCaptureLevelException(ex); + } + + _services.TryAddSingletonEnumerable(); + + var opts = new GcMetricsProducer.Options(); + opts.HistogramBuckets ??= histogramBuckets; + + _services.AddSingleton(opts); + return this; } /// - /// Includes a breakdown of exceptions thrown labeled by type. + /// Include metrics that measure the number of exceptions thrown. /// - public Builder WithExceptionStats() + public Builder WithExceptionStats(CaptureLevel captureLevel = CaptureLevel.Counters) { - StatsCollectors.AddOrReplace(new ExceptionStatsCollector()); - return this; - } + try + { + if (captureLevel != CaptureLevel.Counters) + ListenerRegistrations.AddOrReplace(ListenerRegistration.Create(captureLevel, sp => new ExceptionEventParser())); + } + catch (UnsupportedEventParserLevelException ex) + { + throw new UnsupportedCaptureLevelException(ex); + } - public Builder WithCustomCollector(IEventSourceStatsCollector statsCollector) - { - StatsCollectors.AddOrReplace(statsCollector); + _services.TryAddSingletonEnumerable(); return this; } @@ -173,10 +200,40 @@ public Builder WithCustomCollector(IEventSourceStatsCollector statsCollector) /// public Builder WithErrorHandler(Action handler) { - _errorHandler = handler; + if (handler == null) + throw new ArgumentNullException(nameof(handler)); + + _options.ErrorHandler = handler; return this; } +#if NET5_0 + /// + /// Specifies a custom interval to recycle collectors. Defaults to 1 day. + /// + /// + /// The event collector mechanism in .NET core can in some circumstances degrade performance over time (gradual increased CPU consumption over many hours/ days). + /// Recycling the event collectors is a workaround, preventing CPU exhaustion (see https://github.com/dotnet/runtime/issues/43985#issuecomment-793187345 for more info). + /// During a recycle, existing metrics will not disappear/ reset but will not be updated for a short period (should be at most a couple of seconds). + /// + /// + /// + public Builder RecycleCollectorsEvery(TimeSpan interval) + { +#if DEBUG + // In debug mode, allow more aggressive recycling times to verify recycling works correctly + var min = TimeSpan.FromSeconds(10); +#else + var min = TimeSpan.FromMinutes(10); +#endif + if (interval < min) + throw new ArgumentOutOfRangeException(nameof(interval), $"Interval must be greater than {min}. If collectors are recycled too frequently, metrics cannot be collected accurately."); + + _options.RecycleListenersEvery = interval; + return this; + } +#endif + /// /// Include additional debugging metrics. Should NOT be used in production unless debugging /// perf issues. @@ -190,20 +247,35 @@ public Builder WithErrorHandler(Action handler) /// public Builder WithDebuggingMetrics(bool generateDebugMetrics) { - _debugMetrics = generateDebugMetrics; + _options.EnabledDebuggingMetrics = generateDebugMetrics; return this; } + + private ServiceProvider BuildServiceProvider() + { + RegisterDefaultConsumers(_services); - internal class TypeEquality : IEqualityComparer + // Add the set of event listeners configured.. + _services.AddSingleton, HashSet>(_ => ListenerRegistrations); + + // ..and register the instance of each listener + foreach (var r in ListenerRegistrations) + r.RegisterServices(_services); + + return _services.BuildServiceProvider(); + } + + internal static void RegisterDefaultConsumers(IServiceCollection services) { - public bool Equals(T x, T y) - { - return x.GetType() == y.GetType(); - } + var interfaceType = typeof(Consumes<>); + var concreteType = typeof(EventConsumer<>); + + var eventTypes = EventParserTypes.GetEventParsers() + .SelectMany(EventParserTypes.GetEventInterfaces); - public int GetHashCode(T obj) + foreach (var t in eventTypes) { - return obj.GetType().GetHashCode(); + services.AddSingleton(interfaceType.MakeGenericType(t), concreteType.MakeGenericType(t)); } } } diff --git a/src/prometheus-net.DotNetRuntime/DotNetRuntimeStatsCollector.cs b/src/prometheus-net.DotNetRuntime/DotNetRuntimeStatsCollector.cs index 28f7667..a81a97a 100644 --- a/src/prometheus-net.DotNetRuntime/DotNetRuntimeStatsCollector.cs +++ b/src/prometheus-net.DotNetRuntime/DotNetRuntimeStatsCollector.cs @@ -1,84 +1,127 @@ using System; using System.Collections.Generic; -using System.Collections.Immutable; using System.Linq; using System.Reflection; using System.Runtime; using System.Runtime.InteropServices; using System.Runtime.Versioning; -#if PROMV2 -using Prometheus.Advanced; -using TCollectorRegistry = Prometheus.Advanced.ICollectorRegistry; -#elif PROMV3 -using TCollectorRegistry = Prometheus.CollectorRegistry; -#endif +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Prometheus.DotNetRuntime.EventListening; +using Prometheus.DotNetRuntime.Metrics; namespace Prometheus.DotNetRuntime { internal sealed class DotNetRuntimeStatsCollector : IDisposable -#if PROMV2 - , IOnDemandCollector -#endif { - private static readonly Dictionary Instances = new Dictionary(); + private static readonly Dictionary Instances = new(); - private DotNetEventListener[] _eventListeners; - private readonly ImmutableHashSet _statsCollectors; - private readonly bool _enabledDebugging; - private readonly Action _errorHandler; - private readonly TCollectorRegistry _registry; - private readonly object _lockInstance = new object(); - - internal DotNetRuntimeStatsCollector(ImmutableHashSet statsCollectors, Action errorHandler, bool enabledDebugging, TCollectorRegistry registry) + private readonly CollectorRegistry _metricRegistry; + private readonly Options _options; + private readonly object _lockInstance = new (); + private readonly CancellationTokenSource _ctSource = new(); + private readonly Task _recycleTask; + private bool _disposed = false; + private DotNetEventListener.GlobalOptions _listenerGlobalOpts; + + internal DotNetRuntimeStatsCollector(ServiceProvider serviceProvider, CollectorRegistry metricRegistry, Options options) { - _statsCollectors = statsCollectors; - _enabledDebugging = enabledDebugging; - _errorHandler = errorHandler ?? (e => { }); - _registry = registry; + _metricRegistry = metricRegistry; + _options = options; + ServiceProvider = serviceProvider; + var metrics = Prometheus.Metrics.WithCustomRegistry(_metricRegistry); + _listenerGlobalOpts = DotNetEventListener.GlobalOptions.CreateFrom(_options, metrics); + lock (_lockInstance) { - if (Instances.ContainsKey(registry)) + if (Instances.ContainsKey(_metricRegistry)) { throw new InvalidOperationException(".NET runtime metrics are already being collected. Dispose() of your previous collector before calling this method again."); } - Instances.Add(registry, this); + Instances.Add(_metricRegistry, this); } + + RegisterMetrics(metrics); + EventListeners = CreateEventListeners(); + if (options.RecycleListenersEvery != null) + _recycleTask = Task.Factory.StartNew(() => RestartListeningEvery(options.RecycleListenersEvery.Value), TaskCreationOptions.LongRunning).Unwrap(); } - public void RegisterMetrics(TCollectorRegistry registry) - { -#if PROMV2 - var metrics = new MetricFactory(registry); -#elif PROMV3 - var metrics = Metrics.WithCustomRegistry(registry); -#endif - - foreach (var sc in _statsCollectors) - { - sc.RegisterMetrics(metrics); - } - - // Metrics have been registered, start the event listeners - _eventListeners = _statsCollectors - .Select(sc => new DotNetEventListener(sc, _errorHandler, _enabledDebugging)) + private DotNetEventListener[] CreateEventListeners() + { + return ServiceProvider + .GetService>() + .Select(r => new DotNetEventListener((IEventListener) ServiceProvider.GetService(r.Type), r.Level, _listenerGlobalOpts)) .ToArray(); + } + + internal DotNetEventListener[] EventListeners { get; private set; } + internal ServiceProvider ServiceProvider { get; } + internal Counter EventListenerRecycles { get; private set; } + + public void RegisterMetrics(MetricFactory metrics) + { + foreach (var mp in ServiceProvider.GetServices()) + mp.RegisterMetrics(metrics); + + _metricRegistry.AddBeforeCollectCallback(UpdateMetrics); + if (_options.RecycleListenersEvery.HasValue) + EventListenerRecycles = metrics.CreateCounter("dotnet_internal_recycle_count", "prometheus-net.DotNetRuntime internal metric. Counts the number of times the underlying event listeners have been recycled"); + SetupConstantMetrics(metrics); } public void UpdateMetrics() { - foreach (var sc in _statsCollectors) + // prometheus-net currently offers no mechanism to unregister collection callbacks added by AddBeforeCollectCallback. + // Once disposed to avoid errors, just exit immediately. + if (_disposed) + return; + + foreach (var mp in ServiceProvider.GetServices()) { try { - sc.UpdateMetrics(); + mp.UpdateMetrics(); } catch (Exception e) { - _errorHandler(e); + _options.ErrorHandler(e); + } + } + } + + private async Task RestartListeningEvery(TimeSpan recycleEvery) + { + while (!_ctSource.IsCancellationRequested) + { + try + { + await Task.Delay(recycleEvery, _ctSource.Token); + + // While it's slightly misleading to record a recycle as having taken place before completing, there is a known + // race condition in https://github.com/dotnet/runtime/issues/40190 that can occur if listeners are disabled/ re-enabled in quick succession. + // Record this now so if this happens in the wild, people will be able to spot the issue. + EventListenerRecycles.Inc(); + + foreach (var el in EventListeners) + { + el.Dispose(); + } + + EventListeners = CreateEventListeners(); + } + catch (OperationCanceledException) when (_ctSource.IsCancellationRequested) + { + // swallow, expected on dispose + } + catch (Exception ex) + { + _options.ErrorHandler(ex); } } } @@ -87,18 +130,25 @@ public void Dispose() { try { - if (_eventListeners == null) - return; + _ctSource.Cancel(); + _recycleTask?.Wait(TimeSpan.FromSeconds(1)); - foreach (var listener in _eventListeners) - listener?.Dispose(); + if (EventListeners != null) + { + foreach (var listener in EventListeners) + listener?.Dispose(); + } + + ServiceProvider.Dispose(); } finally { lock (_lockInstance) { - Instances.Remove(_registry); + Instances.Remove(_metricRegistry); } + + _disposed = true; } } @@ -131,7 +181,7 @@ private void SetupConstantMetrics(MetricFactory metrics) } catch (Exception e) { - _errorHandler(e); + _options.ErrorHandler(e); } try @@ -141,8 +191,22 @@ private void SetupConstantMetrics(MetricFactory metrics) } catch (Exception e) { - _errorHandler(e); + _options.ErrorHandler(e); } } + + public class Options + { + public Action ErrorHandler { get; set; } = (e => { }); + public bool EnabledDebuggingMetrics { get; set; } = false; + + public TimeSpan? RecycleListenersEvery { get; set; } = +#if NET5_0 + TimeSpan.FromDays(1); +#else + null; +#endif + + } } } \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/EventConsumption.cs b/src/prometheus-net.DotNetRuntime/EventConsumption.cs new file mode 100644 index 0000000..36364cb --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/EventConsumption.cs @@ -0,0 +1,42 @@ +using Prometheus.DotNetRuntime.EventListening; +using Prometheus.DotNetRuntime.Metrics; + +namespace Prometheus.DotNetRuntime +{ + /// + /// Used to communicate that a depends on the events generated by an or . + /// + /// + public interface Consumes + where TEvents : IEvents + { + public TEvents Events { get; } + + /// + /// Indicates if the events of will be produced and can be listened to. + /// + /// + /// As event parsers may or may not be enabled (or enabled at lower event levels), we need a mechanism to indicate if + /// events are available or not to generate metrics from. + /// + public bool Enabled { get; } + } + + internal class EventConsumer : Consumes + where T : IEvents + { + public EventConsumer() + { + Enabled = false; + } + + public EventConsumer(T events) + { + Events = events; + Enabled = true; + } + + public T Events { get; } + public bool Enabled { get; set; } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/EventListening/CounterNameAttribute.cs b/src/prometheus-net.DotNetRuntime/EventListening/CounterNameAttribute.cs new file mode 100644 index 0000000..5e11fdc --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/EventListening/CounterNameAttribute.cs @@ -0,0 +1,15 @@ +using System; + +namespace Prometheus.DotNetRuntime.EventListening +{ + [AttributeUsage(AttributeTargets.Event)] + public class CounterNameAttribute : Attribute + { + public CounterNameAttribute(string name) + { + Name = name; + } + + public string Name { get; } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/EventListening/Counters.cs b/src/prometheus-net.DotNetRuntime/EventListening/Counters.cs new file mode 100644 index 0000000..215ffd4 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/EventListening/Counters.cs @@ -0,0 +1,25 @@ +namespace Prometheus.DotNetRuntime.EventListening +{ + public readonly struct MeanCounterValue + { + public MeanCounterValue(int count, double mean) + { + Count = count; + Mean = mean; + } + + public int Count { get; } + public double Mean { get; } + public double Total => Count * Mean; + } + + public readonly struct IncrementingCounterValue + { + public IncrementingCounterValue(double value) + { + IncrementedBy = value; + } + + public double IncrementedBy { get; } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/EventListening/DotNetEventListener.cs b/src/prometheus-net.DotNetRuntime/EventListening/DotNetEventListener.cs new file mode 100644 index 0000000..a7ed9cd --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/EventListening/DotNetEventListener.cs @@ -0,0 +1,139 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.Tracing; +using Prometheus.DotNetRuntime.Metrics.Producers.Util; + +namespace Prometheus.DotNetRuntime.EventListening +{ + internal sealed class DotNetEventListener : EventListener + { + private readonly GlobalOptions _globalOptions; + private readonly string _nameSnakeCase; + private readonly HashSet _enabledEventSources = new(); + private readonly Stopwatch _sp; + private HashSet _threadIdsPublishingEvents; + + internal DotNetEventListener(IEventListener eventListener, EventLevel level, GlobalOptions globalOptions) + { + Level = level; + EventListener = eventListener; + _globalOptions = globalOptions; + + if (_globalOptions.EnabledDebuggingMetrics) + { + _nameSnakeCase = eventListener.GetType().Name.ToSnakeCase(); + _sp = new Stopwatch(); + _threadIdsPublishingEvents = new HashSet(); + } + + EventSourceCreated += OnEventSourceCreated; + } + + public EventLevel Level { get; } + internal bool StartedReceivingEvents { get; private set; } + internal IEventListener EventListener { get; private set; } + + private void OnEventSourceCreated(object sender, EventSourceCreatedEventArgs e) + { + var es = e.EventSource; + if (es.Guid == EventListener.EventSourceGuid) + { + EnableEvents(es, Level, EventListener.Keywords, GetEventListenerArguments(EventListener)); + _enabledEventSources.Add(e.EventSource); + StartedReceivingEvents = true; + } + } + + private Dictionary GetEventListenerArguments(IEventListener listener) + { + var args = new Dictionary(); + if (listener is IEventCounterListener counterListener) + { + args["EventCounterIntervalSec"] = counterListener.RefreshIntervalSeconds.ToString(); + } + + return args; + } + + protected override void OnEventWritten(EventWrittenEventArgs eventData) + { + try + { + if (_globalOptions.EnabledDebuggingMetrics) + { + _globalOptions.DebuggingMetrics.EventTypeCounts.Labels(_nameSnakeCase, eventData.EventSource.Name, eventData.EventName ?? "unknown").Inc(); + _sp.Restart(); + _threadIdsPublishingEvents.Add(eventData.OSThreadId); + _globalOptions.DebuggingMetrics.ThreadCount.Labels(_nameSnakeCase).Set(_threadIdsPublishingEvents.Count); + } + + // Event counters are present in every EventListener, regardless of if they subscribed to them. + // Kind of odd but just filter them out by source here. + if (eventData.EventSource.Guid == EventListener.EventSourceGuid) + EventListener.ProcessEvent(eventData); + + if (_globalOptions.EnabledDebuggingMetrics) + { + _sp.Stop(); + _globalOptions.DebuggingMetrics.TimeConsumed.Labels(_nameSnakeCase, eventData.EventSource.Name, eventData.EventName ?? "unknown").Inc(_sp.Elapsed.TotalSeconds); + } + } + catch (Exception e) + { + _globalOptions.ErrorHandler(e); + } + } + + public override void Dispose() + { + EventSourceCreated -= OnEventSourceCreated; + EventListener.Dispose(); + base.Dispose(); + } + + internal class GlobalOptions + { + internal GlobalOptions() + { + } + + internal static GlobalOptions CreateFrom(DotNetRuntimeStatsCollector.Options opts, MetricFactory factory) + { + var instance = new GlobalOptions(); + if (opts.EnabledDebuggingMetrics) + { + instance.DebuggingMetrics = new( + factory.CreateCounter($"dotnet_debug_event_count_total", "The total number of .NET diagnostic events processed", "listener_name", "event_source_name", "event_name"), + factory.CreateCounter("dotnet_debug_event_seconds_total", + "The total time consumed by processing .NET diagnostic events (does not include the CPU cost to generate the events)", + "listener_name", "event_source_name", "event_name"), + factory.CreateGauge("dotnet_debug_publish_thread_count", "The number of threads that have published events", "listener_name") + ); + } + + instance.EnabledDebuggingMetrics = opts.EnabledDebuggingMetrics; + instance.ErrorHandler = opts.ErrorHandler; + + return instance; + } + + public Action ErrorHandler { get; set; } = (e => { }); + public bool EnabledDebuggingMetrics { get; set; } = false; + public DebugMetrics DebuggingMetrics { get; set; } + + public class DebugMetrics + { + public DebugMetrics(Counter eventTypeCounts, Counter timeConsumed, Gauge threadCount) + { + EventTypeCounts = eventTypeCounts; + TimeConsumed = timeConsumed; + ThreadCount = threadCount; + } + public Counter TimeConsumed { get; } + public Counter EventTypeCounts { get; } + public Gauge ThreadCount { get; } + } + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/EventListening/EventCounterParserBase.cs b/src/prometheus-net.DotNetRuntime/EventListening/EventCounterParserBase.cs new file mode 100644 index 0000000..4731e34 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/EventListening/EventCounterParserBase.cs @@ -0,0 +1,147 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.Tracing; +using System.Linq; +using System.Reflection; +using System.Threading; +using Prometheus.DotNetRuntime.EventListening; + +namespace Prometheus.DotNetRuntime.EventListening +{ + /// + /// A reflection-based event parser that can extract typed counter values for a given event counter source. + /// + /// + /// While using reflection isn't ideal from a performance standpoint, this is fine for now- event counters are collected at most + /// every second so won't have to deal with high throughput of events. + /// + /// + public abstract class EventCounterParserBase : IEventCounterParser + where T : ICounterEvents + { + private readonly Dictionary>> _countersToParsers; + private long _timeSinceLastCounter; + + protected EventCounterParserBase() + { + var eventNames = GetType().GetInterfaces() + .Where(x => typeof(ICounterEvents).IsAssignableFrom(x)) + .SelectMany(x => x.GetEvents(), (t, e) => e.Name) + .ToHashSet(); + + var eventsAndNameAttrs = GetType() + .GetEvents() + .Where(e => eventNames.Contains(e.Name)) + .Select(x => (@event: x, nameAttr: x.GetCustomAttribute())) + .ToArray(); + + if (eventsAndNameAttrs.Length == 0) + throw new Exception("Could not locate any events to map to event counters!"); + + var eventsWithoutAttrs = eventsAndNameAttrs.Where(x => x.nameAttr == null).ToArray(); + if (eventsWithoutAttrs.Length > 0) + throw new Exception($"All events part of an {nameof(ICounterEvents)} interface require a [{nameof(CounterNameAttribute)}] attribute. Events without attribute: {string.Join(", ", eventsWithoutAttrs.Select(x => x.@event.Name))}."); + + _countersToParsers = eventsAndNameAttrs.ToDictionary( + k => k.nameAttr.Name, + v => GetParseFunction(v.@event, v.nameAttr.Name) + ); + } + + public abstract Guid EventSourceGuid { get; } + public abstract EventKeywords Keywords { get; } + public abstract int RefreshIntervalSeconds { get; set; } + + public void ProcessEvent(EventWrittenEventArgs e) + { + if (e.EventName == null || !e.EventName.Equals("EventCounters")) + return; + + var eventPayload = e.Payload[0] as IDictionary; + if (eventPayload == null) + return; + + Interlocked.Exchange(ref _timeSinceLastCounter, Stopwatch.GetTimestamp()); + + if (eventPayload.TryGetValue("Name", out var p) && p is string counterName) + { + if (!_countersToParsers.TryGetValue(counterName, out var parser)) + return; + + parser(eventPayload); + } + } + + private Action> GetParseFunction(EventInfo @event, string counterName) + { + var eventField = GetType().GetField(@event.Name, BindingFlags.NonPublic | BindingFlags.Instance); + + if (eventField == null) + throw new Exception($"Unable to locate backing field for event '{@event.Name}'."); + + var type = @event.EventHandlerType.GetGenericArguments().Single(); + Func, (bool, object)> parseCounterFunc; + + if (type == typeof(IncrementingCounterValue)) + { + parseCounterFunc = TryParseIncrementingCounter; + } + else if (type == typeof(MeanCounterValue)) + { + parseCounterFunc = TryParseCounter; + } + else + { + throw new Exception($"Unexpected counter type '{type}'!"); + } + + return payload => + { + var eventDelegate = (MulticastDelegate)eventField.GetValue(this); + + // No-one is listening to this event + if (eventDelegate == null) + return; + + foreach (var handler in eventDelegate.GetInvocationList()) + { + var (success, value) = parseCounterFunc(payload); + if (success) + handler.Method.Invoke(handler.Target, new []{ value }); + else + { + throw new MismatchedCounterTypeException($"Counter '{counterName}' could not be parsed by function {parseCounterFunc.Method} indicating the counter has been declared as the wrong type."); + } + } + }; + } + + private (bool, object) TryParseIncrementingCounter(IDictionary payload) + { + if (payload.TryGetValue("Increment", out var increment)) + return (true, new IncrementingCounterValue((double)increment)); + + return (false, new IncrementingCounterValue()); + } + + private (bool, object) TryParseCounter(IDictionary payload) + { + if (payload.TryGetValue("Mean", out var mean) && payload.TryGetValue("Count", out var count)) + return (true, new MeanCounterValue((int)count, (double)mean)); + + return (false, new MeanCounterValue()); + } + + public void Dispose() + { + } + } + + public class MismatchedCounterTypeException : Exception + { + public MismatchedCounterTypeException(string message) : base(message) + { + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/EventListening/EventParserTypes.cs b/src/prometheus-net.DotNetRuntime/EventListening/EventParserTypes.cs new file mode 100644 index 0000000..7e8ad55 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/EventListening/EventParserTypes.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics.Tracing; +using System.Linq; +using System.Reflection; + +namespace Prometheus.DotNetRuntime.EventListening +{ + internal static class EventParserTypes + { + private static readonly ImmutableHashSet InterfaceTypesToIgnore = new[] + { + typeof(IEvents), + typeof(IVerboseEvents), + typeof(IInfoEvents), + typeof(IWarningEvents), + typeof(IErrorEvents), + typeof(IAlwaysEvents), + typeof(ICriticalEvents), + typeof(ICounterEvents), + }.ToImmutableHashSet(); + + internal static IEnumerable GetEventInterfaces(Type t) + { + return t.GetInterfaces() + .Where(i => typeof(IEvents).IsAssignableFrom(i) && !InterfaceTypesToIgnore.Contains(i)); + } + + internal static IEnumerable GetEventInterfaces(Type t, EventLevel atLevelAndBelow) + { + return GetEventInterfaces(t) + .Where(t => GetEventLevel(t) <= atLevelAndBelow); + } + + internal static ImmutableHashSet GetLevelsFromParser(Type type) + { + return GetEventInterfaces(type) + .Select(GetEventLevel) + .ToImmutableHashSet(); + } + + private static EventLevel GetEventLevel(Type t) + { + // Captures ICounterEvents too as it inherits from IAlwaysEvents + if (typeof(IAlwaysEvents).IsAssignableFrom(t)) + return EventLevel.LogAlways; + + if (typeof(IVerboseEvents).IsAssignableFrom(t)) + return EventLevel.Verbose; + + if (typeof(IInfoEvents).IsAssignableFrom(t)) + return EventLevel.Informational; + + if (typeof(IWarningEvents).IsAssignableFrom(t)) + return EventLevel.Warning; + + if (typeof(IErrorEvents).IsAssignableFrom(t)) + return EventLevel.Error; + + if (typeof(ICriticalEvents).IsAssignableFrom(t)) + return EventLevel.Critical; + + throw new InvalidOperationException($"Unexpected type '{t}'"); + } + + internal static IEnumerable GetEventParsers() + { + return GetEventParsers(typeof(IEventListener).Assembly); + } + + internal static IEnumerable GetEventParsers(Assembly fromAssembly) + { + return fromAssembly + .GetTypes() + .Where(x => x.IsClass && x.GetInterfaces().Any(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IEventParser<>))); + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/EventListening/IEventCounterListener.cs b/src/prometheus-net.DotNetRuntime/EventListening/IEventCounterListener.cs new file mode 100644 index 0000000..71b4c9b --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/EventListening/IEventCounterListener.cs @@ -0,0 +1,10 @@ +namespace Prometheus.DotNetRuntime.EventListening +{ + /// + /// An that listens for event counters. + /// + public interface IEventCounterListener : IEventListener + { + public int RefreshIntervalSeconds { get; set; } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/EventListening/IEventCounterParser.cs b/src/prometheus-net.DotNetRuntime/EventListening/IEventCounterParser.cs new file mode 100644 index 0000000..316dcfe --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/EventListening/IEventCounterParser.cs @@ -0,0 +1,13 @@ +using Prometheus.DotNetRuntime.EventListening; + +namespace Prometheus.DotNetRuntime.EventListening +{ + /// + /// An that turns untyped counter values into strongly-typed counter events. + /// + /// + public interface IEventCounterParser : IEventParser, IEventCounterListener + where TCounters : ICounterEvents + { + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/EventListening/IEventListener.cs b/src/prometheus-net.DotNetRuntime/EventListening/IEventListener.cs new file mode 100644 index 0000000..e61ccfb --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/EventListening/IEventListener.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Immutable; +using System.Diagnostics.Tracing; + +namespace Prometheus.DotNetRuntime.EventListening +{ + public interface IEventListener : IDisposable + { + /// + /// The unique id of the event source to receive events from. + /// + Guid EventSourceGuid { get; } + + /// + /// The keywords to enable in the event source. + /// + /// + /// Keywords act as a "if-any-match" filter- specify multiple keywords to obtain multiple categories of events + /// from the event source. + /// + EventKeywords Keywords { get; } + + /// + /// The levels of events supported. + /// + ImmutableHashSet SupportedLevels { get; } + + /// + /// Process a received event. + /// + /// + /// Implementors should listen to events and perform some kind of processing. + /// + void ProcessEvent(EventWrittenEventArgs e); + + void IDisposable.Dispose() + { + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/EventListening/IEventParser.cs b/src/prometheus-net.DotNetRuntime/EventListening/IEventParser.cs new file mode 100644 index 0000000..3cba101 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/EventListening/IEventParser.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Immutable; +using System.Diagnostics.Tracing; +using Prometheus.DotNetRuntime.EventListening; + +namespace Prometheus.DotNetRuntime.EventListening +{ + /// + /// A that receives "untyped" events of into strongly-typed events. + /// + /// + /// Represents the set of strongly-typed events emitted by this parser. Implementors should not directly implement , rather + /// implement inheriting interfaces such as , , etc. + /// + public interface IEventParser : IEventListener + where TEvents : IEvents + { + ImmutableHashSet IEventListener.SupportedLevels => EventParserDefaults.GetSupportedLevels(this); + + private static class EventParserDefaults + { + private static ImmutableHashSet SupportedLevels; + + public static ImmutableHashSet GetSupportedLevels(IEventParser listener) + { + if (SupportedLevels == null) + SupportedLevels = EventParserTypes.GetLevelsFromParser(listener.GetType()); + + return SupportedLevels; + } + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/EventListening/IEvents.cs b/src/prometheus-net.DotNetRuntime/EventListening/IEvents.cs new file mode 100644 index 0000000..e1ef4c8 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/EventListening/IEvents.cs @@ -0,0 +1,34 @@ +namespace Prometheus.DotNetRuntime.EventListening +{ + public interface IEvents + { + } + + public interface IInfoEvents : IEvents + { + } + + public interface IVerboseEvents : IEvents + { + } + + public interface IErrorEvents : IEvents + { + } + + public interface ICriticalEvents : IEvents + { + } + + public interface IWarningEvents : IEvents + { + } + + public interface IAlwaysEvents : IEvents + { + } + + public interface ICounterEvents : IAlwaysEvents + { + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/EventListening/Parsers/ContentionEventParser.cs b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/ContentionEventParser.cs new file mode 100644 index 0000000..c9e710c --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/ContentionEventParser.cs @@ -0,0 +1,85 @@ +using System; +using System.Diagnostics.Tracing; +using Prometheus.DotNetRuntime.EventListening.EventSources; +using Prometheus.DotNetRuntime.EventListening.Parsers.Util; + +namespace Prometheus.DotNetRuntime.EventListening.Parsers +{ + public class ContentionEventParser : IEventParser, ContentionEventParser.Events.Info + { + private readonly SamplingRate _samplingRate; + private const int EventIdContentionStart = 81, EventIdContentionStop = 91; + private readonly EventPairTimer _eventPairTimer; + + public event Action ContentionStart; + public event Action ContentionEnd; + + public ContentionEventParser(SamplingRate samplingRate) + { + _samplingRate = samplingRate; + _eventPairTimer = new EventPairTimer( + EventIdContentionStart, + EventIdContentionStop, + x => x.OSThreadId, + samplingRate + ); + } + + public EventKeywords Keywords => (EventKeywords)DotNetRuntimeEventSource.Keywords.Contention; + public Guid EventSourceGuid => DotNetRuntimeEventSource.Id; + + public void ProcessEvent(EventWrittenEventArgs e) + { + switch (_eventPairTimer.TryGetDuration(e, out var duration)) + { + case DurationResult.Start: + ContentionStart?.Invoke(Events.ContentionStartEvent.Instance); + return; + + case DurationResult.FinalWithDuration: + ContentionEnd?.InvokeManyTimes(_samplingRate.SampleEvery, Events.ContentionEndEvent.GetFrom(duration)); + return; + + default: + return; + } + } + + public static class Events + { + public interface Info : IInfoEvents + { + event Action ContentionStart; + event Action ContentionEnd; + } + + + + public class ContentionStartEvent + { + public static readonly ContentionStartEvent Instance = new(); + + private ContentionStartEvent() + { + } + } + + public class ContentionEndEvent + { + private static readonly ContentionEndEvent Instance = new(); + + private ContentionEndEvent() + { + } + + public TimeSpan ContentionDuration { get; private set; } + + public static ContentionEndEvent GetFrom(TimeSpan contentionDuration) + { + Instance.ContentionDuration = contentionDuration; + return Instance; + } + } + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/EventListening/Parsers/ExceptionEventParser.cs b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/ExceptionEventParser.cs new file mode 100644 index 0000000..72f9438 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/ExceptionEventParser.cs @@ -0,0 +1,50 @@ +using System; +using System.Diagnostics.Tracing; +using Prometheus.DotNetRuntime.EventListening.EventSources; + +namespace Prometheus.DotNetRuntime.EventListening.Parsers +{ + public class ExceptionEventParser : IEventParser, ExceptionEventParser.Events.Error + { + public event Action ExceptionThrown; + + public Guid EventSourceGuid => DotNetRuntimeEventSource.Id; + public EventKeywords Keywords => (EventKeywords) DotNetRuntimeEventSource.Keywords.Exception; + + public void ProcessEvent(EventWrittenEventArgs e) + { + const int EventIdExceptionThrown = 80; + + if (e.EventId == EventIdExceptionThrown) + { + ExceptionThrown?.Invoke(Events.ExceptionThrownEvent.ParseFrom(e)); + } + } + + public static class Events + { + public interface Error : IErrorEvents + { + event Action ExceptionThrown; + } + + public class ExceptionThrownEvent + { + private static readonly ExceptionThrownEvent Instance = new(); + + private ExceptionThrownEvent() + { + } + + public string ExceptionType { get; private set; } + + public static ExceptionThrownEvent ParseFrom(EventWrittenEventArgs e) + { + Instance.ExceptionType = (string) e.Payload[0]; + + return Instance; + } + } + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/EventListening/Parsers/GcEventParser.cs b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/GcEventParser.cs new file mode 100644 index 0000000..870c479 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/GcEventParser.cs @@ -0,0 +1,223 @@ +using System; +using System.Diagnostics.Tracing; +using Prometheus.DotNetRuntime.EventListening.EventSources; +using Prometheus.DotNetRuntime.EventListening.Parsers.Util; + +namespace Prometheus.DotNetRuntime.EventListening.Parsers +{ + public class GcEventParser : IEventParser, GcEventParser.Events.Info, GcEventParser.Events.Verbose + { + private const int + EventIdGcStart = 1, + EventIdGcStop = 2, + EventIdSuspendEEStart = 9, + EventIdRestartEEStop = 3, + EventIdHeapStats = 4, + EventIdAllocTick = 10; + + private readonly EventPairTimer _gcEventTimer = new EventPairTimer( + EventIdGcStart, + EventIdGcStop, + x => (uint) x.Payload[0], + x => new GcData((uint) x.Payload[1], (DotNetRuntimeEventSource.GCType) x.Payload[3]), + SampleEvery.OneEvent); + + private readonly EventPairTimer _gcPauseEventTimer = new EventPairTimer( + EventIdSuspendEEStart, + EventIdRestartEEStop, + // Suspensions/ Resumptions are always done sequentially so there is no common value to match events on. Return a constant value as the event id. + x => 1, + SampleEvery.OneEvent); + + public event Action HeapStats; + public event Action PauseComplete; + public event Action CollectionStart; + public event Action CollectionComplete; + public event Action AllocationTick; + + public Guid EventSourceGuid => DotNetRuntimeEventSource.Id; + public EventKeywords Keywords => (EventKeywords) DotNetRuntimeEventSource.Keywords.GC; + + public void ProcessEvent(EventWrittenEventArgs e) + { + if (e.EventId == EventIdAllocTick) + { + AllocationTick?.Invoke(Events.AllocationTickEvent.ParseFrom(e)); + return; + } + + if (e.EventId == EventIdHeapStats) + { + HeapStats?.Invoke(Events.HeapStatsEvent.ParseFrom(e)); + return; + } + + // flags representing the "Garbage Collection" + "Preparation for garbage collection" pause reasons + const uint suspendGcReasons = 0x1 | 0x6; + + if (e.EventId == EventIdSuspendEEStart && ((uint) e.Payload[0] & suspendGcReasons) == 0) + { + // Execution engine is pausing for a reason other than GC, discard event. + return; + } + + if (_gcPauseEventTimer.TryGetDuration(e, out var pauseDuration) == DurationResult.FinalWithDuration) + { + PauseComplete?.Invoke(Events.PauseCompleteEvent.GetFrom(pauseDuration)); + return; + } + + if (e.EventId == EventIdGcStart) + { + CollectionStart?.Invoke(Events.CollectionStartEvent.ParseFrom(e)); + } + + if (_gcEventTimer.TryGetDuration(e, out var gcDuration, out var gcData) == DurationResult.FinalWithDuration) + { + CollectionComplete?.Invoke(Events.CollectionCompleteEvent.GetFrom(gcData.Generation, gcData.Type, gcDuration)); + } + } + + private struct GcData + { + public GcData(uint generation, DotNetRuntimeEventSource.GCType type) + { + Generation = generation; + Type = type; + } + + public uint Generation { get; } + public DotNetRuntimeEventSource.GCType Type { get; } + } + + public static class Events + { + public interface Info : IInfoEvents + { + event Action HeapStats; + event Action PauseComplete; + event Action CollectionStart; + event Action CollectionComplete; + } + + public interface Verbose : IVerboseEvents + { + event Action AllocationTick; + } + + public class HeapStatsEvent + { + private static readonly HeapStatsEvent Instance = new(); + + private HeapStatsEvent() + { + } + + public static HeapStatsEvent ParseFrom(EventWrittenEventArgs e) + { + Instance.Gen0SizeBytes = ((ulong) e.Payload[0]); + Instance.Gen1SizeBytes = ((ulong) e.Payload[2]); + Instance.Gen2SizeBytes = ((ulong) e.Payload[4]); + Instance.LohSizeBytes = ((ulong) e.Payload[6]); + Instance.FinalizationQueueLength = ((ulong) e.Payload[9]); + Instance.NumPinnedObjects = ((uint) e.Payload[10]); + + return Instance; + } + + public ulong FinalizationQueueLength { get; private set; } + + public ulong LohSizeBytes { get; private set; } + + public ulong Gen2SizeBytes { get; private set; } + + public ulong Gen1SizeBytes { get; private set; } + + public uint NumPinnedObjects { get; private set; } + + public ulong Gen0SizeBytes { get; private set; } + } + + public class PauseCompleteEvent + { + private static readonly PauseCompleteEvent Instance = new PauseCompleteEvent(); + + private PauseCompleteEvent() + { + } + + public static PauseCompleteEvent GetFrom(TimeSpan pauseDuration) + { + Instance.PauseDuration = pauseDuration; + return Instance; + } + + public TimeSpan PauseDuration { get; private set; } + } + + public class AllocationTickEvent + { + private static readonly AllocationTickEvent Instance = new(); + + private AllocationTickEvent() + { + } + + public static AllocationTickEvent ParseFrom(EventWrittenEventArgs e) + { + const uint lohHeapFlag = 0x1; + Instance.IsLargeObjectHeap = ((uint) e.Payload[1] & lohHeapFlag) == lohHeapFlag; + Instance.AllocatedBytes = (uint) e.Payload[0]; + + return Instance; + } + + public uint AllocatedBytes { get; private set; } + public bool IsLargeObjectHeap { get; private set; } + } + + public class CollectionStartEvent + { + private static readonly CollectionStartEvent Instance = new(); + + private CollectionStartEvent() + { + } + + public static CollectionStartEvent ParseFrom(EventWrittenEventArgs e) + { + Instance.Count = (uint)e.Payload[0]; + Instance.Generation = (uint) e.Payload[1]; + Instance.Reason = (DotNetRuntimeEventSource.GCReason) e.Payload[2]; + return Instance; + } + + public uint Generation { get; private set; } + public uint Count { get; private set; } + public DotNetRuntimeEventSource.GCReason Reason { get; private set; } + } + + public class CollectionCompleteEvent + { + private static readonly CollectionCompleteEvent Instance = new(); + + private CollectionCompleteEvent() + { + } + + public static CollectionCompleteEvent GetFrom(uint generation, DotNetRuntimeEventSource.GCType type, TimeSpan duration) + { + Instance.Generation = generation; + Instance.Type = type; + Instance.Duration = duration; + + return Instance; + } + + public TimeSpan Duration { get; private set; } + public DotNetRuntimeEventSource.GCType Type { get; private set; } + public uint Generation { get; private set; } + } + } + } +} diff --git a/src/prometheus-net.DotNetRuntime/EventListening/Parsers/JitEventParser.cs b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/JitEventParser.cs new file mode 100644 index 0000000..c61a124 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/JitEventParser.cs @@ -0,0 +1,68 @@ +using System; +using System.Diagnostics.Tracing; +using Prometheus.DotNetRuntime.EventListening.EventSources; +using Prometheus.DotNetRuntime.EventListening.Parsers; +using Prometheus.DotNetRuntime.EventListening.Parsers.Util; + +namespace Prometheus.DotNetRuntime.EventListening.Parsers +{ + public class JitEventParser : IEventParser, JitEventParser.Events.Verbose + { + private readonly SamplingRate _samplingRate; + private const int EventIdMethodJittingStarted = 145, EventIdMethodLoadVerbose = 143; + private readonly EventPairTimer _eventPairTimer; + + public event Action CompilationComplete; + + public JitEventParser(SamplingRate samplingRate) + { + _samplingRate = samplingRate; + _eventPairTimer = new EventPairTimer( + EventIdMethodJittingStarted, + EventIdMethodLoadVerbose, + x => (ulong)x.Payload[0], + samplingRate + ); + } + + public EventKeywords Keywords => (EventKeywords) DotNetRuntimeEventSource.Keywords.Jit; + public Guid EventSourceGuid => DotNetRuntimeEventSource.Id; + + public void ProcessEvent(EventWrittenEventArgs e) + { + if (_eventPairTimer.TryGetDuration(e, out var duration) == DurationResult.FinalWithDuration) + { + CompilationComplete?.InvokeManyTimes(_samplingRate.SampleEvery, Events.CompilationCompleteEvent.ParseFrom(e, duration)); + } + } + + public static class Events + { + public interface Verbose : IVerboseEvents + { + event Action CompilationComplete; + } + + public class CompilationCompleteEvent + { + private static readonly CompilationCompleteEvent Instance = new(); + + private CompilationCompleteEvent() { } + + public TimeSpan CompilationDuration { get; private set; } + public bool IsMethodDynamic { get; private set; } + + public static CompilationCompleteEvent ParseFrom(EventWrittenEventArgs e, TimeSpan compilationDuration) + { + // dynamic methods are of special interest to us- only a certain number of JIT'd dynamic methods + // will be cached. Frequent use of dynamic can cause methods to be evicted from the cache and re-JIT'd + var methodFlags = (uint)e.Payload[5]; + Instance.IsMethodDynamic = (methodFlags & 0x1) == 0x1; + Instance.CompilationDuration = compilationDuration; + + return Instance; + } + } + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/EventListening/Parsers/RuntimeEventParser.cs b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/RuntimeEventParser.cs new file mode 100644 index 0000000..f7cf9de --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/RuntimeEventParser.cs @@ -0,0 +1,92 @@ +using System; +using System.Diagnostics.Tracing; +using Prometheus.DotNetRuntime.EventListening; + +#nullable enable + +namespace Prometheus.DotNetRuntime.EventListening.Parsers +{ + public class RuntimeEventParser : EventCounterParserBase, RuntimeEventParser.Events.Counters + { +#pragma warning disable CS0067 + [CounterName("threadpool-thread-count")] + public event Action? ThreadPoolThreadCount; + + [CounterName("threadpool-queue-length")] + public event Action? ThreadPoolQueueLength; + + [CounterName("threadpool-completed-items-count")] + public event Action? ThreadPoolCompletedItemsCount; + + [CounterName("monitor-lock-contention-count")] + public event Action? MonitorLockContentionCount; + + [CounterName("active-timer-count")] + public event Action? ActiveTimerCount; + + [CounterName("exception-count")] + public event Action? ExceptionCount; + + [CounterName("assembly-count")] + public event Action? NumAssembliesLoaded; + + [CounterName("il-bytes-jitted")] + public event Action? IlBytesJitted; + + [CounterName("methods-jitted-count")] + public event Action? MethodsJittedCount; + + [CounterName("alloc-rate")] + public event Action? AllocRate; + + [CounterName("gc-heap-size")] + public event Action? GcHeapSize; + [CounterName("gen-0-gc-count")] + public event Action? Gen0GcCount; + [CounterName("gen-1-gc-count")] + public event Action? Gen1GcCount; + [CounterName("gen-2-gc-count")] + public event Action? Gen2GcCount; + [CounterName("time-in-gc")] + public event Action? TimeInGc; + [CounterName("gen-0-size")] + public event Action? Gen0Size; + [CounterName("gen-1-size")] + public event Action? Gen1Size; + [CounterName("gen-2-size")] + public event Action? Gen2Size; + [CounterName("loh-size")] + public event Action? LohSize; +#pragma warning restore CS0067 + + public override Guid EventSourceGuid => EventSources.SystemRuntimeEventSource.Id; + public override EventKeywords Keywords { get; } + public override int RefreshIntervalSeconds { get; set; } = 1; + + public static class Events + { + public interface Counters : ICounterEvents + { + event Action ThreadPoolThreadCount; + event Action ThreadPoolQueueLength; + event Action ThreadPoolCompletedItemsCount; + event Action MonitorLockContentionCount; + event Action ActiveTimerCount; + event Action ExceptionCount; + event Action NumAssembliesLoaded; + event Action IlBytesJitted; + event Action MethodsJittedCount; + event Action AllocRate; + event Action GcHeapSize; + event Action Gen0GcCount; + event Action Gen1GcCount; + event Action Gen2GcCount; + event Action TimeInGc; + event Action Gen0Size; + event Action Gen1Size; + event Action Gen2Size; + event Action LohSize; + } + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/EventListening/Parsers/ThreadPoolEventParser.cs b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/ThreadPoolEventParser.cs new file mode 100644 index 0000000..1ac6985 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/ThreadPoolEventParser.cs @@ -0,0 +1,80 @@ +using System; +using System.Diagnostics.Tracing; +using Prometheus.DotNetRuntime.EventListening.EventSources; + +namespace Prometheus.DotNetRuntime.EventListening.Parsers +{ + public class ThreadPoolEventParser : IEventParser, ThreadPoolEventParser.Events.Info + { + private const int + EventIdThreadPoolSample = 54, + EventIdThreadPoolAdjustment = 55, + EventIdIoThreadCreate = 44, + EventIdIoThreadRetire = 46, + EventIdIoThreadUnretire = 47, + EventIdIoThreadTerminate = 45; + + public event Action ThreadPoolAdjusted; + public event Action IoThreadPoolAdjusted; + + public Guid EventSourceGuid => DotNetRuntimeEventSource.Id; + public EventKeywords Keywords => (EventKeywords) DotNetRuntimeEventSource.Keywords.Threading; + + public void ProcessEvent(EventWrittenEventArgs e) + { + switch (e.EventId) + { + case EventIdThreadPoolAdjustment: + ThreadPoolAdjusted?.Invoke(Events.ThreadPoolAdjustedEvent.ParseFrom(e)); + return; + + case EventIdIoThreadCreate: + case EventIdIoThreadRetire: + case EventIdIoThreadUnretire: + case EventIdIoThreadTerminate: + IoThreadPoolAdjusted?.Invoke(Events.IoThreadPoolAdjustedEvent.ParseFrom(e)); + return; + } + } + + public static class Events + { + public interface Info : IInfoEvents + { + event Action ThreadPoolAdjusted; + event Action IoThreadPoolAdjusted; + } + + public class ThreadPoolAdjustedEvent + { + private static readonly ThreadPoolAdjustedEvent Instance = new (); + private ThreadPoolAdjustedEvent() { } + + public DotNetRuntimeEventSource.ThreadAdjustmentReason AdjustmentReason { get; private set; } + public uint NumThreads { get; private set; } + + public static ThreadPoolAdjustedEvent ParseFrom(EventWrittenEventArgs e) + { + Instance.NumThreads = (uint) e.Payload[1]; + Instance.AdjustmentReason = (DotNetRuntimeEventSource.ThreadAdjustmentReason) e.Payload[2]; + return Instance; + } + } + + public class IoThreadPoolAdjustedEvent + { + private static readonly IoThreadPoolAdjustedEvent Instance = new (); + + private IoThreadPoolAdjustedEvent() { } + + public uint NumThreads { get; private set; } + + public static IoThreadPoolAdjustedEvent ParseFrom(EventWrittenEventArgs e) + { + Instance.NumThreads = (uint) e.Payload[0]; + return Instance; + } + } + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/StatsCollectors/Util/Cache.cs b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/Util/Cache.cs similarity index 98% rename from src/prometheus-net.DotNetRuntime/StatsCollectors/Util/Cache.cs rename to src/prometheus-net.DotNetRuntime/EventListening/Parsers/Util/Cache.cs index 832f2dc..c586196 100644 --- a/src/prometheus-net.DotNetRuntime/StatsCollectors/Util/Cache.cs +++ b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/Util/Cache.cs @@ -4,7 +4,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Prometheus.DotNetRuntime.StatsCollectors.Util +namespace Prometheus.DotNetRuntime.EventListening.Parsers.Util { /// /// A strongly-typed cache that periodically evicts items. diff --git a/src/prometheus-net.DotNetRuntime/EventListening/Parsers/Util/EventExtensions.cs b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/Util/EventExtensions.cs new file mode 100644 index 0000000..1abe81c --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/Util/EventExtensions.cs @@ -0,0 +1,15 @@ +using System; + +namespace Prometheus.DotNetRuntime.EventListening.Parsers +{ + internal static class DelegateExtensions + { + internal static void InvokeManyTimes(this Action d, int count, T payload) + { + for (int i = 0; i < count; i++) + { + d(payload); + } + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/StatsCollectors/Util/EventPairTimer.cs b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/Util/EventPairTimer.cs similarity index 98% rename from src/prometheus-net.DotNetRuntime/StatsCollectors/Util/EventPairTimer.cs rename to src/prometheus-net.DotNetRuntime/EventListening/Parsers/Util/EventPairTimer.cs index 4205e48..e8773d9 100644 --- a/src/prometheus-net.DotNetRuntime/StatsCollectors/Util/EventPairTimer.cs +++ b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/Util/EventPairTimer.cs @@ -1,7 +1,7 @@ using System; using System.Diagnostics.Tracing; -namespace Prometheus.DotNetRuntime.StatsCollectors.Util +namespace Prometheus.DotNetRuntime.EventListening.Parsers.Util { /// /// To generate metrics, we are often interested in the duration between two events. This class diff --git a/src/prometheus-net.DotNetRuntime/StatsCollectors/Util/SamplingRate.cs b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/Util/SamplingRate.cs similarity index 95% rename from src/prometheus-net.DotNetRuntime/StatsCollectors/Util/SamplingRate.cs rename to src/prometheus-net.DotNetRuntime/EventListening/Parsers/Util/SamplingRate.cs index 63d4a58..f2be441 100644 --- a/src/prometheus-net.DotNetRuntime/StatsCollectors/Util/SamplingRate.cs +++ b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/Util/SamplingRate.cs @@ -1,6 +1,6 @@ using System.Threading; -namespace Prometheus.DotNetRuntime.StatsCollectors.Util +namespace Prometheus.DotNetRuntime.EventListening.Parsers.Util { /// /// The rate at which high-frequency events are sampled diff --git a/src/prometheus-net.DotNetRuntime/EventSources/DotNetRuntimeEventSource.cs b/src/prometheus-net.DotNetRuntime/EventListening/Sources/DotNetRuntimeEventSource.cs similarity index 99% rename from src/prometheus-net.DotNetRuntime/EventSources/DotNetRuntimeEventSource.cs rename to src/prometheus-net.DotNetRuntime/EventListening/Sources/DotNetRuntimeEventSource.cs index 5a447ae..b7735de 100644 --- a/src/prometheus-net.DotNetRuntime/EventSources/DotNetRuntimeEventSource.cs +++ b/src/prometheus-net.DotNetRuntime/EventListening/Sources/DotNetRuntimeEventSource.cs @@ -1,6 +1,6 @@ using System; -namespace Prometheus.DotNetRuntime.EventSources +namespace Prometheus.DotNetRuntime.EventListening.EventSources { /// /// Provider name: Microsoft-Windows-DotNETRuntime. Provides events generated by the .NET runtime (unmanaged code). diff --git a/src/prometheus-net.DotNetRuntime/EventSources/FrameworkEventSource.cs b/src/prometheus-net.DotNetRuntime/EventListening/Sources/FrameworkEventSource.cs similarity index 90% rename from src/prometheus-net.DotNetRuntime/EventSources/FrameworkEventSource.cs rename to src/prometheus-net.DotNetRuntime/EventListening/Sources/FrameworkEventSource.cs index 4045ba1..a827733 100644 --- a/src/prometheus-net.DotNetRuntime/EventSources/FrameworkEventSource.cs +++ b/src/prometheus-net.DotNetRuntime/EventListening/Sources/FrameworkEventSource.cs @@ -1,6 +1,6 @@ using System; -namespace Prometheus.DotNetRuntime.EventSources +namespace Prometheus.DotNetRuntime.EventListening.EventSources { /// /// Provider name: System.Diagnostics.Eventing.FrameworkEventSource. Provides events generated by diff --git a/src/prometheus-net.DotNetRuntime/EventListening/Sources/SystemRuntimeEventSource.cs b/src/prometheus-net.DotNetRuntime/EventListening/Sources/SystemRuntimeEventSource.cs new file mode 100644 index 0000000..b56c4ee --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/EventListening/Sources/SystemRuntimeEventSource.cs @@ -0,0 +1,12 @@ +using System; + +namespace Prometheus.DotNetRuntime.EventListening.EventSources +{ + /// + /// Provider name: System.Runtime. Provides counters generated by the .NET runtime. + /// + public class SystemRuntimeEventSource + { + public static readonly Guid Id = new ("49592C0F-5A05-516D-AA4B-A64E02026C89"); + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/Extensions.cs b/src/prometheus-net.DotNetRuntime/Extensions.cs index 1a79834..644f9e8 100644 --- a/src/prometheus-net.DotNetRuntime/Extensions.cs +++ b/src/prometheus-net.DotNetRuntime/Extensions.cs @@ -1,10 +1,15 @@ +using System; using System.Collections.Generic; +using System.ComponentModel.Design; +using System.Diagnostics.Tracing; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; namespace Prometheus.DotNetRuntime { - public static class Extensions + internal static class Extensions { - public static void AddOrReplace(this ISet s, T toAddOrReplace) + internal static void AddOrReplace(this ISet s, T toAddOrReplace) { if (!s.Add(toAddOrReplace)) { @@ -12,5 +17,22 @@ public static void AddOrReplace(this ISet s, T toAddOrReplace) s.Add(toAddOrReplace); } } + + internal static void TryAddSingletonEnumerable(this IServiceCollection services) + where TService : class + where TImplementation : class, TService + { + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + } + + internal static EventLevel ToEventLevel(this CaptureLevel level) + { + return (EventLevel) (int)level; + } + + internal static CaptureLevel ToCaptureLevel(this EventLevel level) + { + return (CaptureLevel) (int)level; + } } } \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/IEventSourceStatsCollector.cs b/src/prometheus-net.DotNetRuntime/IEventSourceStatsCollector.cs deleted file mode 100644 index f028469..0000000 --- a/src/prometheus-net.DotNetRuntime/IEventSourceStatsCollector.cs +++ /dev/null @@ -1,54 +0,0 @@ -using System; -using System.Diagnostics.Tracing; -#if PROMV2 -using Prometheus.Advanced; -#endif -using Prometheus.DotNetRuntime.EventSources; - -namespace Prometheus.DotNetRuntime -{ - /// - /// Defines an interface register for and receive .NET runtime events. Events can then be aggregated - /// and measured as prometheus metrics. - /// - public interface IEventSourceStatsCollector - { - /// - /// The unique id of the event source to receive events from. - /// - Guid EventSourceGuid { get; } - - /// - /// The keywords to enable in the event source. - /// - /// - /// Keywords act as a "if-any-match" filter- specify multiple keywords to obtain multiple categories of events - /// from the event source. - /// - EventKeywords Keywords { get; } - - /// - /// The level of events to receive from the event source. - /// - EventLevel Level { get; } - - /// - /// Process a received event. - /// - /// - /// Implementors should listen to events and perform some kind of aggregation, emitting this to prometheus. - /// - void ProcessEvent(EventWrittenEventArgs e); - - /// - /// Called when the instance is associated with a collector registry, so that the collectors managed - /// by this instance can be registered. - /// - void RegisterMetrics(MetricFactory metrics); - - /// - /// Called before each collection. Any values in collectors managed by this instance should now be brought up to date. - /// - void UpdateMetrics(); - } -} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/ListenerRegistration.cs b/src/prometheus-net.DotNetRuntime/ListenerRegistration.cs new file mode 100644 index 0000000..7866fdd --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/ListenerRegistration.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics.Tracing; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Prometheus.DotNetRuntime.EventListening; + +namespace Prometheus.DotNetRuntime +{ + internal class ListenerRegistration : IEquatable + { + private ListenerRegistration(EventLevel level, Type type, Func factory) + { + Level = level; + Type = type; + Factory = factory; + } + + public static ListenerRegistration Create(CaptureLevel level, Func factory) + where T : IEventListener + { + var supportedLevels = EventParserTypes.GetLevelsFromParser(typeof(T)); + var eventLevel = level.ToEventLevel(); + + if (!supportedLevels.Contains(eventLevel)) + throw new UnsupportedEventParserLevelException(typeof(T), level, supportedLevels); + + + return new ListenerRegistration(eventLevel, typeof(T), sp => (object)factory(sp)); + } + + internal void RegisterServices(IServiceCollection services) + { + services.AddSingleton(Type, Factory); + services.AddSingleton(typeof(IEventListener), sp => sp.GetService(Type)); + + // Register each events interface exposed at the level specified + foreach (var i in EventParserTypes.GetEventInterfaces(Type, Level)) + services.AddSingleton(i, sp => sp.GetService(Type)); + } + + public EventLevel Level { get; set; } + public Type Type { get; } + public Func Factory { get; } + + public bool Equals(ListenerRegistration other) + { + if (ReferenceEquals(null, other)) return false; + if (ReferenceEquals(this, other)) return true; + return Equals(Type, other.Type); + } + + public override bool Equals(object obj) + { + if (ReferenceEquals(null, obj)) return false; + if (ReferenceEquals(this, obj)) return true; + if (obj.GetType() != this.GetType()) return false; + return Equals((ListenerRegistration) obj); + } + + public override int GetHashCode() + { + return (Type != null ? Type.GetHashCode() : 0); + } + } + + public class UnsupportedCaptureLevelException : Exception + { + public UnsupportedCaptureLevelException(CaptureLevel specifiedLevel, ISet supportedLevels) + : base($"The level '{specifiedLevel}' is not supported- please use one of: {string.Join(", ", supportedLevels)}") + { + SpecifiedLevel = specifiedLevel; + SupportedLevels = supportedLevels; + } + + public UnsupportedCaptureLevelException(UnsupportedEventParserLevelException ex) + : this (ex.SpecifiedLevel, ex.SupportedLevels.Select(x => x.ToCaptureLevel()).ToImmutableHashSet()) + { + } + + public static UnsupportedCaptureLevelException CreateWithCounterSupport(UnsupportedEventParserLevelException ex) + { + return new ( + ex.SpecifiedLevel, + ex.SupportedLevels + .Select(x => x.ToCaptureLevel()) + .ToImmutableHashSet() + .Add(CaptureLevel.Counters) + ); + } + + public CaptureLevel SpecifiedLevel { get; } + public ISet SupportedLevels { get; } + } + + public class UnsupportedEventParserLevelException : Exception + { + public UnsupportedEventParserLevelException(Type eventParserType, CaptureLevel specifiedLevel, ISet supportedLevels) + : base($"The event parser '{eventParserType.Name}' does not support the level '{specifiedLevel}'- please use one of: {string.Join(", ", supportedLevels)}") + { + EventParserType = eventParserType; + SpecifiedLevel = specifiedLevel; + SupportedLevels = supportedLevels; + } + + public Type EventParserType { get; } + public CaptureLevel SpecifiedLevel { get; } + public ISet SupportedLevels { get; } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/Metrics/IMetricProducer.cs b/src/prometheus-net.DotNetRuntime/Metrics/IMetricProducer.cs new file mode 100644 index 0000000..f4639dc --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/Metrics/IMetricProducer.cs @@ -0,0 +1,19 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Prometheus.DotNetRuntime.Metrics +{ + public interface IMetricProducer + { + /// + /// Called when the producer is associated with a metrics registry, allowing metrics to be created via the passed . + /// + void RegisterMetrics(MetricFactory metrics); + + /// + /// Called before each metrics collection. Any metrics managed by this producer should now be brought up to date. + /// + void UpdateMetrics(); + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/MetricExtensions.cs b/src/prometheus-net.DotNetRuntime/Metrics/MetricExtensions.cs similarity index 52% rename from src/prometheus-net.DotNetRuntime/MetricExtensions.cs rename to src/prometheus-net.DotNetRuntime/Metrics/MetricExtensions.cs index 720e49a..4b84c71 100644 --- a/src/prometheus-net.DotNetRuntime/MetricExtensions.cs +++ b/src/prometheus-net.DotNetRuntime/Metrics/MetricExtensions.cs @@ -1,64 +1,13 @@ using System; -using System.Collections; using System.Collections.Generic; -using System.Linq; -using System.Reflection; -#if PROMV2 -using Prometheus.Advanced; -using Prometheus.Advanced.DataContracts; -#endif - -namespace Prometheus.DotNetRuntime +namespace Prometheus.DotNetRuntime.Metrics { /// /// Provides helper functions for accessing values of metrics. /// - /// - /// The API around accessing metric values dramatically altered in V3. This wrapper class provides support - /// for both v2.* + v3.* of prometheus-net. - /// internal static class MetricExtensions { -#if PROMV2 - /// - /// Collects all values of a counter recorded across both unlabeled and labeled metrics. - /// - internal static IEnumerable CollectAllValues(this Counter counter, bool excludeUnlabeled = false) - { - return CollectAllMetrics(counter, excludeUnlabeled).Select(x => x.counter.value); - } - - /// - /// Collects all sum values of a histogram recorded across both unlabeled and labeled metrics. - /// - internal static IEnumerable CollectAllSumValues(this Histogram histogram, bool excludeUnlabeled = false) - { - return CollectAllMetrics(histogram, excludeUnlabeled).Select(x => x.histogram.sample_sum); - } - - /// - /// Collects all count values of a histogram recorded across both unlabeled and labeled metrics. - /// - internal static IEnumerable CollectAllCountValues(this Histogram histogram) - { - return CollectAllMetrics(histogram).Select(x => x.histogram.sample_count); - } - - internal static IEnumerable CollectAllMetrics(this ICollector collector, bool excludeUnlabeled = false) - { - return collector.Collect().Single().metric.Where(x => !excludeUnlabeled || x.label.Count > 0); - } - - internal static void Observe(this Histogram h, double val, int samples) - { - // Ugly hack for V2 :( - for (int i = 0; i < samples; i++) - h.Observe(val); - } -#endif - -#if PROMV3 /// /// Collects all values of a counter recorded across both unlabeled and labeled metrics. /// @@ -102,6 +51,5 @@ private static IEnumerable GetLabelValues(Collector co { return collector.GetAllLabelValues(); } -#endif } } \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/Metrics/Producers/ContentionMetricsProducer.cs b/src/prometheus-net.DotNetRuntime/Metrics/Producers/ContentionMetricsProducer.cs new file mode 100644 index 0000000..3850d98 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/Metrics/Producers/ContentionMetricsProducer.cs @@ -0,0 +1,36 @@ +using Prometheus.DotNetRuntime.EventListening.Parsers; + +namespace Prometheus.DotNetRuntime.Metrics.Producers +{ + public class ContentionMetricsProducer : IMetricProducer + { + private readonly Consumes _contentionInfo; + private readonly Consumes _runtimeCounters; + + public ContentionMetricsProducer(Consumes contentionInfo, Consumes runtimeCounters) + { + _contentionInfo = contentionInfo; + _runtimeCounters = runtimeCounters; + } + + internal Counter ContentionSecondsTotal { get; private set; } + internal Counter ContentionTotal { get; private set; } + + public void RegisterMetrics(MetricFactory metrics) + { + if (!_contentionInfo.Enabled && !_runtimeCounters.Enabled) + return; + + ContentionTotal = metrics.CreateCounter("dotnet_contention_total", "The number of locks contended"); + _runtimeCounters.Events.MonitorLockContentionCount += e => ContentionTotal.Inc(e.IncrementedBy); + + if (_contentionInfo.Enabled) + { + ContentionSecondsTotal = metrics.CreateCounter("dotnet_contention_seconds_total", "The total amount of time spent contending locks"); + _contentionInfo.Events.ContentionEnd += e => ContentionSecondsTotal.Inc(e.ContentionDuration.TotalSeconds); + } + } + + public void UpdateMetrics() { } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/Metrics/Producers/ExceptionMetricsProducer.cs b/src/prometheus-net.DotNetRuntime/Metrics/Producers/ExceptionMetricsProducer.cs new file mode 100644 index 0000000..4f2c791 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/Metrics/Producers/ExceptionMetricsProducer.cs @@ -0,0 +1,47 @@ +using Prometheus.DotNetRuntime.EventListening.Parsers; + +namespace Prometheus.DotNetRuntime.Metrics.Producers +{ + public class ExceptionMetricsProducer : IMetricProducer + { + private readonly Consumes _exceptionError; + private readonly Consumes _runtimeCounters; + private const string LabelType = "type"; + + public ExceptionMetricsProducer(Consumes exceptionError, Consumes runtimeCounters) + { + _exceptionError = exceptionError; + _runtimeCounters = runtimeCounters; + } + + internal Counter ExceptionCount { get; private set; } + + public void RegisterMetrics(MetricFactory metrics) + { + if (!_exceptionError.Enabled && !_runtimeCounters.Enabled) + return; + + if (_exceptionError.Enabled) + { + ExceptionCount = metrics.CreateCounter( + "dotnet_exceptions_total", + "Count of exceptions thrown, broken down by type", + LabelType + ); + + _exceptionError.Events.ExceptionThrown += e => ExceptionCount.Labels(e.ExceptionType).Inc(); + } + else if (_runtimeCounters.Enabled) + { + ExceptionCount = metrics.CreateCounter( + "dotnet_exceptions_total", + "Count of exceptions thrown" + ); + + _runtimeCounters.Events.ExceptionCount += e => ExceptionCount.Inc(e.IncrementedBy); + } + } + + public void UpdateMetrics() { } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/Metrics/Producers/GcMetricsProducer.cs b/src/prometheus-net.DotNetRuntime/Metrics/Producers/GcMetricsProducer.cs new file mode 100644 index 0000000..09f892b --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/Metrics/Producers/GcMetricsProducer.cs @@ -0,0 +1,188 @@ +using System; +using System.Collections.Generic; +using Prometheus.DotNetRuntime.EventListening.Parsers; +using Prometheus.DotNetRuntime.EventListening; +using Prometheus.DotNetRuntime.EventListening.EventSources; +using Prometheus.DotNetRuntime.Metrics.Producers.Util; + + +namespace Prometheus.DotNetRuntime.Metrics.Producers +{ + public class GcMetricsProducer : IMetricProducer + { + private const string + LabelHeap = "gc_heap", + LabelGeneration = "gc_generation", + LabelReason = "gc_reason", + LabelType = "gc_type"; + + private static readonly Dictionary GcTypeToLabels = LabelGenerator.MapEnumToLabelValues(); + private static readonly Dictionary GcReasonToLabels = LabelGenerator.MapEnumToLabelValues(); + + private readonly Consumes _gcInfo; + private readonly Consumes _gcVerbose; + private readonly Consumes _runtimeCounters; + private readonly Ratio _gcCpuRatio = Ratio.ProcessTotalCpu(); + private readonly Ratio _gcPauseRatio = Ratio.ProcessTime(); + private Options _options; + + public GcMetricsProducer( + Options options, + Consumes gcInfo, + Consumes gcVerbose, + Consumes runtimeCounters) + { + _options = options; + _gcInfo = gcInfo; + _gcVerbose = gcVerbose; + _runtimeCounters = runtimeCounters; + } + + internal Histogram GcCollectionSeconds { get; private set; } + internal Histogram GcPauseSeconds { get; private set; } + internal Counter GcCollections { get; private set; } + internal Gauge GcCpuRatio { get; private set; } + internal Gauge GcPauseRatio { get; private set; } + internal Counter AllocatedBytes { get; private set; } + internal Gauge GcHeapSizeBytes { get; private set; } + internal Gauge GcNumPinnedObjects { get; private set; } + internal Gauge GcFinalizationQueueLength { get; private set; } + internal Gauge AvailableMemory { get; private set; } + + public void RegisterMetrics(MetricFactory metrics) + { +#if !NETSTANDARD2_1 + AvailableMemory = metrics.CreateGauge("dotnet_gc_memory_total_available_bytes", "The upper limit on the amount of physical memory .NET can allocate to"); +#endif + + // No registered sources available- cannot produce metrics + if (!_gcInfo.Enabled && !_gcVerbose.Enabled && !_runtimeCounters.Enabled) + return; + + if (_runtimeCounters.Enabled && !_gcInfo.Enabled) + { + GcPauseRatio = metrics.CreateGauge("dotnet_gc_pause_ratio", "The percentage of time the process spent paused for garbage collection"); + _runtimeCounters.Events.TimeInGc += e => GcPauseRatio.Set(e.Mean / 100.0); + + GcHeapSizeBytes = metrics.CreateGauge( + "dotnet_gc_heap_size_bytes", + "The current size of all heaps (only updated after a garbage collection)", + LabelGeneration); + + _runtimeCounters.Events.Gen0Size += e => GcHeapSizeBytes.Labels("0").Set(e.Mean); + _runtimeCounters.Events.Gen1Size += e => GcHeapSizeBytes.Labels("1").Set(e.Mean); + _runtimeCounters.Events.Gen2Size += e => GcHeapSizeBytes.Labels("2").Set(e.Mean); + _runtimeCounters.Events.LohSize += e => GcHeapSizeBytes.Labels("loh").Set(e.Mean); + + GcCollections = metrics.CreateCounter( + "dotnet_gc_collection_count_total", + "Counts the number of garbage collections that have occurred, broken down by generation number.", + LabelGeneration); + + _runtimeCounters.Events.Gen0GcCount += e => GcCollections.Labels("0").Inc(e.IncrementedBy); + _runtimeCounters.Events.Gen1GcCount += e => GcCollections.Labels("1").Inc(e.IncrementedBy); + _runtimeCounters.Events.Gen2GcCount += e => GcCollections.Labels("2").Inc(e.IncrementedBy); + } + + if (_gcInfo.Enabled) + { + GcCollectionSeconds = metrics.CreateHistogram( + "dotnet_gc_collection_seconds", + "The amount of time spent running garbage collections", + new HistogramConfiguration() + { + Buckets = _options.HistogramBuckets, + LabelNames = new[] {LabelGeneration, LabelType} + } + ); + + _gcInfo.Events.CollectionComplete += (e) => GcCollectionSeconds.Labels(GetGenerationToString(e.Generation), GcTypeToLabels[e.Type]).Observe(e.Duration.TotalSeconds); + + GcPauseSeconds = metrics.CreateHistogram( + "dotnet_gc_pause_seconds", + "The amount of time execution was paused for garbage collection", + new HistogramConfiguration() + { + Buckets = _options.HistogramBuckets + } + ); + + _gcInfo.Events.PauseComplete += (e) => GcPauseSeconds.Observe(e.PauseDuration.TotalSeconds); + + GcCollections = metrics.CreateCounter( + "dotnet_gc_collection_count_total", + "Counts the number of garbage collections that have occurred, broken down by generation number and the reason for the collection.", + LabelGeneration, LabelReason); + + _gcInfo.Events.CollectionStart += (e) => GcCollections.Labels(GetGenerationToString(e.Generation), GcReasonToLabels[e.Reason]).Inc(); + + GcCpuRatio = metrics.CreateGauge("dotnet_gc_cpu_ratio", "The percentage of process CPU time spent running garbage collections"); + GcPauseRatio = metrics.CreateGauge("dotnet_gc_pause_ratio", "The percentage of time the process spent paused for garbage collection"); + + GcHeapSizeBytes = metrics.CreateGauge( + "dotnet_gc_heap_size_bytes", + "The current size of all heaps (only updated after a garbage collection)", + LabelGeneration); + + GcNumPinnedObjects = metrics.CreateGauge("dotnet_gc_pinned_objects", "The number of pinned objects"); + GcFinalizationQueueLength = metrics.CreateGauge("dotnet_gc_finalization_queue_length", "The number of objects waiting to be finalized"); + + _gcInfo.Events.HeapStats += e => + { + GcHeapSizeBytes.Labels("0").Set(e.Gen0SizeBytes); + GcHeapSizeBytes.Labels("1").Set(e.Gen1SizeBytes); + GcHeapSizeBytes.Labels("2").Set(e.Gen2SizeBytes); + GcHeapSizeBytes.Labels("loh").Set(e.LohSizeBytes); + GcFinalizationQueueLength.Set(e.FinalizationQueueLength); + GcNumPinnedObjects.Set(e.NumPinnedObjects); + }; + } + + if (_gcVerbose.Enabled || _runtimeCounters.Enabled) + { + AllocatedBytes = metrics.CreateCounter( + "dotnet_gc_allocated_bytes_total", + "The total number of bytes allocated on the managed heap", + labelNames: _gcVerbose.Enabled ? new [] { LabelHeap } : new string[0]); + + if (_gcVerbose.Enabled) + _gcVerbose.Events.AllocationTick += e => AllocatedBytes.Labels(e.IsLargeObjectHeap ? "loh" : "soh").Inc(e.AllocatedBytes); + else + _runtimeCounters.Events.AllocRate += r => AllocatedBytes.Inc(r.IncrementedBy); + } + } + + public void UpdateMetrics() + { + if (_gcInfo.Enabled) + { + GcCpuRatio?.Set(_gcCpuRatio.CalculateConsumedRatio(GcCollectionSeconds)); + GcPauseRatio?.Set(_gcPauseRatio.CalculateConsumedRatio(GcPauseSeconds)); + } + +#if !NETSTANDARD2_1 + AvailableMemory?.Set(GC.GetGCMemoryInfo().TotalAvailableMemoryBytes); +#endif + } + + private static string GetGenerationToString(uint generation) + { + return generation switch + { + 0 => "0", + 1 => "1", + 2 => "2", + // large object heap + 3 => "loh", + // pinned object heap, .NET 5+ only + 4 => "poh", + _ => generation.ToString() + }; + } + + public class Options + { + public double[] HistogramBuckets { get; set; } = Constants.DefaultHistogramBuckets; + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/Metrics/Producers/JitMetricsProducer.cs b/src/prometheus-net.DotNetRuntime/Metrics/Producers/JitMetricsProducer.cs new file mode 100644 index 0000000..96a528c --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/Metrics/Producers/JitMetricsProducer.cs @@ -0,0 +1,45 @@ +using Prometheus.DotNetRuntime.EventListening.Parsers; +using Prometheus.DotNetRuntime.Metrics.Producers.Util; + +namespace Prometheus.DotNetRuntime.Metrics.Producers +{ + public class JitMetricsProducer : IMetricProducer + { + private const string DynamicLabel = "dynamic"; + private const string LabelValueTrue = "true"; + private const string LabelValueFalse = "false"; + + private readonly Consumes _jitVerbose; + private readonly Ratio _jitCpuRatio = Ratio.ProcessTotalCpu(); + + public JitMetricsProducer(Consumes jitVerbose) + { + _jitVerbose = jitVerbose; + } + + internal Counter MethodsJittedTotal { get; private set; } + internal Counter MethodsJittedSecondsTotal { get; private set; } + internal Gauge CpuRatio { get; private set; } + + public void RegisterMetrics(MetricFactory metrics) + { + if (!_jitVerbose.Enabled) + return; + + MethodsJittedTotal = metrics.CreateCounter("dotnet_jit_method_total", "Total number of methods compiled by the JIT compiler", DynamicLabel); + MethodsJittedSecondsTotal = metrics.CreateCounter("dotnet_jit_method_seconds_total", "Total number of seconds spent in the JIT compiler", DynamicLabel); + _jitVerbose.Events.CompilationComplete += e => + { + MethodsJittedTotal.Labels(e.IsMethodDynamic.ToLabel()).Inc(); + MethodsJittedSecondsTotal.Labels(e.IsMethodDynamic.ToLabel()).Inc(e.CompilationDuration.TotalSeconds); + }; + + CpuRatio = metrics.CreateGauge("dotnet_jit_cpu_ratio", "The amount of total CPU time consumed spent JIT'ing"); + } + + public void UpdateMetrics() + { + CpuRatio.Set(_jitCpuRatio.CalculateConsumedRatio(MethodsJittedSecondsTotal)); + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/Metrics/Producers/ThreadPoolMetricsProducer.cs b/src/prometheus-net.DotNetRuntime/Metrics/Producers/ThreadPoolMetricsProducer.cs new file mode 100644 index 0000000..5c15b70 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/Metrics/Producers/ThreadPoolMetricsProducer.cs @@ -0,0 +1,82 @@ +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using Prometheus.DotNetRuntime.EventListening.EventSources; +using Prometheus.DotNetRuntime.EventListening.Parsers; +using Prometheus.DotNetRuntime.Metrics.Producers.Util; + +namespace Prometheus.DotNetRuntime.Metrics.Producers +{ + public class ThreadPoolMetricsProducer : IMetricProducer + { + private readonly Dictionary _adjustmentReasonToLabel = LabelGenerator.MapEnumToLabelValues(); + private readonly Options _options; + private readonly Consumes _threadPoolInfo; + private readonly Consumes _runtimeCounters; + + public ThreadPoolMetricsProducer(Options options, Consumes threadPoolInfo, Consumes runtimeCounters) + { + _options = options; + _threadPoolInfo = threadPoolInfo; + _runtimeCounters = runtimeCounters; + } + + internal Gauge NumThreads { get; private set; } + internal Gauge NumIocThreads { get; private set; } + internal Counter AdjustmentsTotal { get; private set; } + internal Counter Throughput { get; private set; } + internal Histogram QueueLength { get; private set; } + internal Gauge NumTimers { get; private set; } + + public void RegisterMetrics(MetricFactory metrics) + { + if (!_threadPoolInfo.Enabled && !_runtimeCounters.Enabled) + return; + + NumThreads = metrics.CreateGauge("dotnet_threadpool_num_threads", "The number of active threads in the thread pool"); + _runtimeCounters.Events.ThreadPoolThreadCount += e => NumThreads.Set(e.Mean); + + Throughput = metrics.CreateCounter("dotnet_threadpool_throughput_total", "The total number of work items that have finished execution in the thread pool"); + _runtimeCounters.Events.ThreadPoolCompletedItemsCount += e => Throughput.Inc(e.IncrementedBy); + + QueueLength = metrics.CreateHistogram("dotnet_threadpool_queue_length", + "Measures the queue length of the thread pool. Values greater than 0 indicate a backlog of work for the threadpool to process.", + new HistogramConfiguration {Buckets = _options.QueueLengthHistogramBuckets} + ); + _runtimeCounters.Events.ThreadPoolQueueLength += e => QueueLength.Observe(e.Mean); + + NumTimers = metrics.CreateGauge("dotnet_threadpool_timer_count", "The number of timers active"); + _runtimeCounters.Events.ActiveTimerCount += e => NumTimers.Set(e.Mean); + + if (_threadPoolInfo.Enabled) + { + AdjustmentsTotal = metrics.CreateCounter( + "dotnet_threadpool_adjustments_total", + "The total number of changes made to the size of the thread pool, labeled by the reason for change", + "adjustment_reason"); + + _threadPoolInfo.Events.ThreadPoolAdjusted += e => + { + AdjustmentsTotal.Labels(_adjustmentReasonToLabel[e.AdjustmentReason]).Inc(); + }; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + // IO threadpool only exists on windows + NumIocThreads = metrics.CreateGauge("dotnet_threadpool_io_num_threads", "The number of active threads in the IO thread pool"); + _threadPoolInfo.Events.IoThreadPoolAdjusted += e => NumIocThreads.Set(e.NumThreads); + } + } + } + + + public void UpdateMetrics() + { + } + + public class Options + { + public double[] QueueLengthHistogramBuckets { get; set; } = new double[] { 0, 1, 10, 100, 1000 }; + } + } +} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/StatsCollectors/Util/Constants.cs b/src/prometheus-net.DotNetRuntime/Metrics/Producers/Util/Constants.cs similarity index 86% rename from src/prometheus-net.DotNetRuntime/StatsCollectors/Util/Constants.cs rename to src/prometheus-net.DotNetRuntime/Metrics/Producers/Util/Constants.cs index 38028e6..51f3ada 100644 --- a/src/prometheus-net.DotNetRuntime/StatsCollectors/Util/Constants.cs +++ b/src/prometheus-net.DotNetRuntime/Metrics/Producers/Util/Constants.cs @@ -1,4 +1,4 @@ -namespace Prometheus.DotNetRuntime.StatsCollectors.Util +namespace Prometheus.DotNetRuntime.Metrics.Producers.Util { internal class Constants { diff --git a/src/prometheus-net.DotNetRuntime/StatsCollectors/Util/LabelGenerator.cs b/src/prometheus-net.DotNetRuntime/Metrics/Producers/Util/LabelGenerator.cs similarity index 57% rename from src/prometheus-net.DotNetRuntime/StatsCollectors/Util/LabelGenerator.cs rename to src/prometheus-net.DotNetRuntime/Metrics/Producers/Util/LabelGenerator.cs index d935bda..a596a88 100644 --- a/src/prometheus-net.DotNetRuntime/StatsCollectors/Util/LabelGenerator.cs +++ b/src/prometheus-net.DotNetRuntime/Metrics/Producers/Util/LabelGenerator.cs @@ -2,19 +2,25 @@ using System.Collections.Generic; using System.Linq; -namespace Prometheus.DotNetRuntime.StatsCollectors.Util +namespace Prometheus.DotNetRuntime.Metrics.Producers.Util { /// /// Generating tags often involves heavy use of String.Format, which takes CPU time and needlessly re-allocates /// strings. Pre-generating these labels helps keep resource use to a minimum. /// - public class LabelGenerator + internal static class LabelGenerator { - public static Dictionary MapEnumToLabelValues() + internal static Dictionary MapEnumToLabelValues() where TEnum : Enum { return Enum.GetValues(typeof(TEnum)).Cast() .ToDictionary(k => k, v => Enum.GetName(typeof(TEnum), v).ToSnakeCase()); } + + internal static string ToLabel(this bool b) + { + const string LabelValueTrue = "true", LabelValueFalse = "false"; + return b ? LabelValueTrue : LabelValueFalse; + } } } \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/StatsCollectors/Util/Ratio.cs b/src/prometheus-net.DotNetRuntime/Metrics/Producers/Util/Ratio.cs similarity index 97% rename from src/prometheus-net.DotNetRuntime/StatsCollectors/Util/Ratio.cs rename to src/prometheus-net.DotNetRuntime/Metrics/Producers/Util/Ratio.cs index 40d9b94..a3f1d53 100644 --- a/src/prometheus-net.DotNetRuntime/StatsCollectors/Util/Ratio.cs +++ b/src/prometheus-net.DotNetRuntime/Metrics/Producers/Util/Ratio.cs @@ -1,11 +1,9 @@ using System; using System.Diagnostics; using System.Linq; -#if PROMV2 -using Prometheus.Advanced; -#endif +using Prometheus.DotNetRuntime.Metrics; -namespace Prometheus.DotNetRuntime.StatsCollectors.Util +namespace Prometheus.DotNetRuntime.Metrics.Producers.Util { /// /// Helps calculate the ratio of process resources consumed by some activity. diff --git a/src/prometheus-net.DotNetRuntime/StatsCollectors/Util/StringExtensions.cs b/src/prometheus-net.DotNetRuntime/Metrics/Producers/Util/StringExtensions.cs similarity index 93% rename from src/prometheus-net.DotNetRuntime/StatsCollectors/Util/StringExtensions.cs rename to src/prometheus-net.DotNetRuntime/Metrics/Producers/Util/StringExtensions.cs index 4d615bd..0f3888b 100644 --- a/src/prometheus-net.DotNetRuntime/StatsCollectors/Util/StringExtensions.cs +++ b/src/prometheus-net.DotNetRuntime/Metrics/Producers/Util/StringExtensions.cs @@ -1,6 +1,6 @@ using System.Text; -namespace Prometheus.DotNetRuntime.StatsCollectors.Util +namespace Prometheus.DotNetRuntime.Metrics.Producers.Util { public static class StringExtensions { diff --git a/src/prometheus-net.DotNetRuntime/Properties.cs b/src/prometheus-net.DotNetRuntime/Properties.cs index 5d287e3..65a4713 100644 --- a/src/prometheus-net.DotNetRuntime/Properties.cs +++ b/src/prometheus-net.DotNetRuntime/Properties.cs @@ -1,2 +1,3 @@ using System.Runtime.CompilerServices; -[assembly:InternalsVisibleTo("prometheus-net.DotNetRuntime.Tests")] \ No newline at end of file +[assembly:InternalsVisibleTo("prometheus-net.DotNetRuntime.Tests")] +[assembly:InternalsVisibleTo("DocsGenerator")] \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/StatsCollectors/ContentionStatsCollector.cs b/src/prometheus-net.DotNetRuntime/StatsCollectors/ContentionStatsCollector.cs deleted file mode 100644 index 06ce356..0000000 --- a/src/prometheus-net.DotNetRuntime/StatsCollectors/ContentionStatsCollector.cs +++ /dev/null @@ -1,70 +0,0 @@ -using System; -using System.Diagnostics.Tracing; -#if PROMV2 -using Prometheus.Advanced; -#endif -using Prometheus.DotNetRuntime.EventSources; -using Prometheus.DotNetRuntime.StatsCollectors.Util; - -namespace Prometheus.DotNetRuntime.StatsCollectors -{ - /// - /// Measures the level of contention in a .NET process, capturing the number - /// of locks contended and the total amount of time spent contending a lock. - /// - /// - /// Due to the way ETW events are triggered, only monitors contended will fire an event- spin locks, etc. - /// do not trigger contention events and so cannot be tracked. - /// - internal sealed class ContentionStatsCollector : IEventSourceStatsCollector - { - private readonly SamplingRate _samplingRate; - private const int EventIdContentionStart = 81, EventIdContentionStop = 91; - private readonly EventPairTimer _eventPairTimer; - - public ContentionStatsCollector(SamplingRate samplingRate) - { - _samplingRate = samplingRate; - _eventPairTimer = new EventPairTimer( - EventIdContentionStart, - EventIdContentionStop, - x => x.OSThreadId, - samplingRate - ); - } - - public EventKeywords Keywords => (EventKeywords)DotNetRuntimeEventSource.Keywords.Contention; - public EventLevel Level => EventLevel.Informational; - public Guid EventSourceGuid => DotNetRuntimeEventSource.Id; - - internal Counter ContentionSecondsTotal { get; private set; } - internal Counter ContentionTotal { get; private set; } - - public void RegisterMetrics(MetricFactory metrics) - { - ContentionSecondsTotal = metrics.CreateCounter("dotnet_contention_seconds_total", "The total amount of time spent contending locks"); - ContentionTotal = metrics.CreateCounter("dotnet_contention_total", "The number of locks contended"); - } - - public void UpdateMetrics() - { - } - - public void ProcessEvent(EventWrittenEventArgs e) - { - switch (_eventPairTimer.TryGetDuration(e, out var duration)) - { - case DurationResult.Start: - ContentionTotal.Inc(); - return; - - case DurationResult.FinalWithDuration: - ContentionSecondsTotal.Inc(duration.TotalSeconds * _samplingRate.SampleEvery); - return; - - default: - return; - } - } - } -} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/StatsCollectors/ExceptionStatsCollector.cs b/src/prometheus-net.DotNetRuntime/StatsCollectors/ExceptionStatsCollector.cs deleted file mode 100644 index 4dd355f..0000000 --- a/src/prometheus-net.DotNetRuntime/StatsCollectors/ExceptionStatsCollector.cs +++ /dev/null @@ -1,46 +0,0 @@ -using Prometheus.DotNetRuntime.EventSources; -using System; -using System.Diagnostics.Tracing; -#if PROMV2 -using Prometheus.Advanced; -#endif - - -namespace Prometheus.DotNetRuntime.StatsCollectors -{ - public class ExceptionStatsCollector : IEventSourceStatsCollector - { - private const int EventIdExceptionThrown = 80; - private const string LabelType = "type"; - - internal Counter ExceptionCount { get; private set; } - - public Guid EventSourceGuid => DotNetRuntimeEventSource.Id; - - public EventKeywords Keywords => (EventKeywords)DotNetRuntimeEventSource.Keywords.Exception; - public EventLevel Level => EventLevel.Informational; - - public void RegisterMetrics(MetricFactory metrics) - { - ExceptionCount = metrics.CreateCounter( - "dotnet_exceptions_total", - "Count of exceptions broken down by type", - LabelType - ); - } - - public void UpdateMetrics() - { - - } - - public void ProcessEvent(EventWrittenEventArgs e) - { - if (e.EventId == EventIdExceptionThrown) - { - ExceptionCount.Labels((string)e.Payload[0]).Inc(); - } - } - } - -} diff --git a/src/prometheus-net.DotNetRuntime/StatsCollectors/GcStatsCollector.cs b/src/prometheus-net.DotNetRuntime/StatsCollectors/GcStatsCollector.cs deleted file mode 100644 index b9c70f7..0000000 --- a/src/prometheus-net.DotNetRuntime/StatsCollectors/GcStatsCollector.cs +++ /dev/null @@ -1,199 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.Tracing; -#if PROMV2 -using Prometheus.Advanced; -#endif -using Prometheus.DotNetRuntime.EventSources; -using Prometheus.DotNetRuntime.StatsCollectors.Util; - -namespace Prometheus.DotNetRuntime.StatsCollectors -{ - /// - /// Measures how the frequency and duration of garbage collections and volume of allocations. Includes information - /// such as the generation the collection is running for, what triggered the collection and the type of the collection. - /// - internal sealed class GcStatsCollector : IEventSourceStatsCollector - { - private const string - LabelHeap = "gc_heap", - LabelGeneration = "gc_generation", - LabelReason = "gc_reason", - LabelType = "gc_type"; - - private const int - EventIdGcStart = 1, - EventIdGcStop = 2, - EventIdSuspendEEStart = 9, - EventIdRestartEEStop = 3, - EventIdHeapStats = 4, - EventIdAllocTick = 10; - - private readonly EventPairTimer _gcEventTimer = new EventPairTimer( - EventIdGcStart, - EventIdGcStop, - x => (uint) x.Payload[0], - x => new GcData((uint) x.Payload[1], (DotNetRuntimeEventSource.GCType) x.Payload[3]), - SampleEvery.OneEvent); - - private readonly EventPairTimer _gcPauseEventTimer = new EventPairTimer( - EventIdSuspendEEStart, - EventIdRestartEEStop, - // Suspensions/ Resumptions are always done sequentially so there is no common value to match events on. Return a constant value as the event id. - x => 1, - SampleEvery.OneEvent); - - private readonly Dictionary _gcReasonToLabels = LabelGenerator.MapEnumToLabelValues(); - private readonly Ratio _gcCpuRatio = Ratio.ProcessTotalCpu(); - private readonly Ratio _gcPauseRatio = Ratio.ProcessTime(); - private readonly double[] _histogramBuckets; - - public GcStatsCollector(double[] histogramBuckets) - { - _histogramBuckets = histogramBuckets; - } - - public GcStatsCollector() : this(Constants.DefaultHistogramBuckets) - { - } - - public Guid EventSourceGuid => DotNetRuntimeEventSource.Id; - public EventKeywords Keywords => (EventKeywords) DotNetRuntimeEventSource.Keywords.GC; - public EventLevel Level => EventLevel.Verbose; - - internal Histogram GcCollectionSeconds { get; private set; } - internal Histogram GcPauseSeconds { get; private set; } - internal Counter GcCollectionReasons { get; private set; } - internal Gauge GcCpuRatio { get; private set; } - internal Gauge GcPauseRatio { get; private set; } - internal Counter AllocatedBytes { get; private set; } - internal Gauge GcHeapSizeBytes { get; private set; } - internal Gauge GcNumPinnedObjects { get; private set; } - internal Gauge GcFinalizationQueueLength { get; private set; } - - public void RegisterMetrics(MetricFactory metrics) - { - GcCollectionSeconds = metrics.CreateHistogram( - "dotnet_gc_collection_seconds", - "The amount of time spent running garbage collections", - new HistogramConfiguration() - { - Buckets = _histogramBuckets, - LabelNames = new []{ LabelGeneration, LabelType } - } - ); - - GcPauseSeconds = metrics.CreateHistogram( - "dotnet_gc_pause_seconds", - "The amount of time execution was paused for garbage collection", - new HistogramConfiguration() - { - Buckets = _histogramBuckets - } - ); - - GcCollectionReasons = metrics.CreateCounter( - "dotnet_gc_collection_reasons_total", - "A tally of all the reasons that lead to garbage collections being run", - LabelReason); - - GcCpuRatio = metrics.CreateGauge("dotnet_gc_cpu_ratio", "The percentage of process CPU time spent running garbage collections"); - GcPauseRatio = metrics.CreateGauge("dotnet_gc_pause_ratio", "The percentage of time the process spent paused for garbage collection"); - - AllocatedBytes = metrics.CreateCounter( - "dotnet_gc_allocated_bytes_total", - "The total number of bytes allocated on the small and large object heaps (updated every 100KB of allocations)", - LabelHeap); - - GcHeapSizeBytes = metrics.CreateGauge( - "dotnet_gc_heap_size_bytes", - "The current size of all heaps (only updated after a garbage collection)", - LabelGeneration); - - GcNumPinnedObjects = metrics.CreateGauge("dotnet_gc_pinned_objects", "The number of pinned objects"); - GcFinalizationQueueLength = metrics.CreateGauge("dotnet_gc_finalization_queue_length", "The number of objects waiting to be finalized"); - } - - public void UpdateMetrics() - { - GcCpuRatio.Set(_gcCpuRatio.CalculateConsumedRatio(GcCollectionSeconds)); - GcPauseRatio.Set(_gcPauseRatio.CalculateConsumedRatio(GcPauseSeconds)); - } - - public void ProcessEvent(EventWrittenEventArgs e) - { - if (e.EventId == EventIdAllocTick) - { - const uint lohHeapFlag = 0x1; - var heapLabelValue = ((uint) e.Payload[1] & lohHeapFlag) == lohHeapFlag ? "loh" : "soh"; - AllocatedBytes.Labels(heapLabelValue).Inc((uint) e.Payload[0]); - return; - } - - if (e.EventId == EventIdHeapStats) - { - GcHeapSizeBytes.Labels("0").Set((ulong) e.Payload[0]); - GcHeapSizeBytes.Labels("1").Set((ulong) e.Payload[2]); - GcHeapSizeBytes.Labels("2").Set((ulong) e.Payload[4]); - GcHeapSizeBytes.Labels("loh").Set((ulong) e.Payload[6]); - GcFinalizationQueueLength.Set((ulong) e.Payload[9]); - GcNumPinnedObjects.Set((uint) e.Payload[10]); - return; - } - - // flags representing the "Garbage Collection" + "Preparation for garbage collection" pause reasons - const uint suspendGcReasons = 0x1 | 0x6; - - if (e.EventId == EventIdSuspendEEStart && ((uint) e.Payload[0] & suspendGcReasons) == 0) - { - // Execution engine is pausing for a reason other than GC, discard event. - return; - } - - if (_gcPauseEventTimer.TryGetDuration(e, out var pauseDuration) == DurationResult.FinalWithDuration) - { - GcPauseSeconds.Observe(pauseDuration.TotalSeconds); - return; - } - - if (e.EventId == EventIdGcStart) - { - GcCollectionReasons.Labels(_gcReasonToLabels[(DotNetRuntimeEventSource.GCReason) e.Payload[2]]).Inc(); - } - - if (_gcEventTimer.TryGetDuration(e, out var gcDuration, out var gcData) == DurationResult.FinalWithDuration) - { - GcCollectionSeconds.Labels(gcData.GetGenerationToString(), gcData.GetTypeToString()).Observe(gcDuration.TotalSeconds); - } - } - - private struct GcData - { - private static readonly Dictionary GcTypeToLabels = LabelGenerator.MapEnumToLabelValues(); - - public GcData(uint generation, DotNetRuntimeEventSource.GCType type) - { - Generation = generation; - Type = type; - } - - public uint Generation { get; } - public DotNetRuntimeEventSource.GCType Type { get; } - - public string GetTypeToString() - { - return GcTypeToLabels[Type]; - } - - public string GetGenerationToString() - { - if (Generation > 2) - { - return "loh"; - } - - return Generation.ToString(); - } - } - } -} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/StatsCollectors/JitStatsCollector.cs b/src/prometheus-net.DotNetRuntime/StatsCollectors/JitStatsCollector.cs deleted file mode 100644 index dec3775..0000000 --- a/src/prometheus-net.DotNetRuntime/StatsCollectors/JitStatsCollector.cs +++ /dev/null @@ -1,73 +0,0 @@ -using System; -using System.Diagnostics; -using System.Diagnostics.Tracing; -#if PROMV2 -using Prometheus.Advanced; -#endif -using Prometheus.DotNetRuntime.EventSources; -using Prometheus.DotNetRuntime.StatsCollectors.Util; - -namespace Prometheus.DotNetRuntime.StatsCollectors -{ - /// - /// Measures the activity of the JIT (Just In Time) compiler in a process. - /// Tracks how often it runs and how long it takes to compile methods - /// - internal sealed class JitStatsCollector : IEventSourceStatsCollector - { - private readonly SamplingRate _samplingRate; - private const int EventIdMethodJittingStarted = 145, EventIdMethodLoadVerbose = 143; - private const string DynamicLabel = "dynamic"; - private const string LabelValueTrue = "true"; - private const string LabelValueFalse = "false"; - - private readonly EventPairTimer _eventPairTimer; - - private readonly Ratio _jitCpuRatio = Ratio.ProcessTotalCpu(); - - public JitStatsCollector(SamplingRate samplingRate) - { - _samplingRate = samplingRate; - _eventPairTimer = new EventPairTimer( - EventIdMethodJittingStarted, - EventIdMethodLoadVerbose, - x => (ulong)x.Payload[0], - samplingRate - ); - } - - public EventKeywords Keywords => (EventKeywords) DotNetRuntimeEventSource.Keywords.Jit; - public EventLevel Level => EventLevel.Verbose; - public Guid EventSourceGuid => DotNetRuntimeEventSource.Id; - - internal Counter MethodsJittedTotal { get; private set; } - internal Counter MethodsJittedSecondsTotal { get; private set; } - internal Gauge CpuRatio { get; private set; } - - public void RegisterMetrics(MetricFactory metrics) - { - MethodsJittedTotal = metrics.CreateCounter("dotnet_jit_method_total", "Total number of methods compiled by the JIT compiler", DynamicLabel); - MethodsJittedSecondsTotal = metrics.CreateCounter("dotnet_jit_method_seconds_total", "Total number of seconds spent in the JIT compiler", DynamicLabel); - CpuRatio = metrics.CreateGauge("dotnet_jit_cpu_ratio", "The amount of total CPU time consumed spent JIT'ing"); - } - - public void UpdateMetrics() - { - CpuRatio.Set(_jitCpuRatio.CalculateConsumedRatio(MethodsJittedSecondsTotal)); - } - - public void ProcessEvent(EventWrittenEventArgs e) - { - if (_eventPairTimer.TryGetDuration(e, out var duration) == DurationResult.FinalWithDuration) - { - // dynamic methods are of special interest to us- only a certain number of JIT'd dynamic methods - // will be cached. Frequent use of dynamic can cause methods to be evicted from the cache and re-JIT'd - var methodFlags = (uint)e.Payload[5]; - var dynamicLabelValue = (methodFlags & 0x1) == 0x1 ? LabelValueTrue : LabelValueFalse; - - MethodsJittedTotal.Labels(dynamicLabelValue).Inc(_samplingRate.SampleEvery); - MethodsJittedSecondsTotal.Labels(dynamicLabelValue).Inc(duration.TotalSeconds * _samplingRate.SampleEvery); - } - } - } -} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/StatsCollectors/ThreadPoolSchedulingStatsCollector.cs b/src/prometheus-net.DotNetRuntime/StatsCollectors/ThreadPoolSchedulingStatsCollector.cs deleted file mode 100644 index ff48889..0000000 --- a/src/prometheus-net.DotNetRuntime/StatsCollectors/ThreadPoolSchedulingStatsCollector.cs +++ /dev/null @@ -1,81 +0,0 @@ -using System; -using System.Diagnostics.Tracing; -using System.Linq; -#if PROMV2 -using Prometheus.Advanced; -#endif -using Prometheus.DotNetRuntime.EventSources; -using Prometheus.DotNetRuntime.StatsCollectors.Util; - -namespace Prometheus.DotNetRuntime.StatsCollectors -{ - /// - /// Measures the volume of work scheduled on the thread pool and the delay between scheduling the work and it beginning execution. - /// - internal sealed class ThreadPoolSchedulingStatsCollector : IEventSourceStatsCollector - { - private const int EventIdThreadPoolEnqueueWork = 30, EventIdThreadPoolDequeueWork = 31; - private readonly double[] _histogramBuckets; - private readonly SamplingRate _samplingRate; - - private readonly EventPairTimer _eventPairTimer; - - public ThreadPoolSchedulingStatsCollector(double[] histogramBuckets, SamplingRate samplingRate) - { - _histogramBuckets = histogramBuckets; - _samplingRate = samplingRate; - _eventPairTimer = new EventPairTimer( - EventIdThreadPoolEnqueueWork, - EventIdThreadPoolDequeueWork, - x => (long)x.Payload[0], - samplingRate, - new Cache(TimeSpan.FromSeconds(30), initialCapacity: 512) - ); - } - - internal ThreadPoolSchedulingStatsCollector(): this(Constants.DefaultHistogramBuckets, SampleEvery.OneEvent) - { - } - - public EventKeywords Keywords => (EventKeywords) (FrameworkEventSource.Keywords.ThreadPool); - public EventLevel Level => EventLevel.Verbose; - public Guid EventSourceGuid => FrameworkEventSource.Id; - - internal Counter ScheduledCount { get; private set; } - internal Histogram ScheduleDelay { get; private set; } - - public void RegisterMetrics(MetricFactory metrics) - { - ScheduledCount = metrics.CreateCounter("dotnet_threadpool_scheduled_total", "The total number of items the thread pool has been instructed to execute"); - ScheduleDelay = metrics.CreateHistogram( - "dotnet_threadpool_scheduling_delay_seconds", - "A breakdown of the latency experienced between an item being scheduled for execution on the thread pool and it starting execution.", - new HistogramConfiguration() - { - Buckets = _histogramBuckets - } - ); - } - - public void UpdateMetrics() - { - } - - public void ProcessEvent(EventWrittenEventArgs e) - { - switch (_eventPairTimer.TryGetDuration(e, out var duration)) - { - case DurationResult.Start: - ScheduledCount.Inc(); - return; - - case DurationResult.FinalWithDuration: - ScheduleDelay.Observe(duration.TotalSeconds, _samplingRate.SampleEvery); - return; - - default: - return; - } - } - } -} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/StatsCollectors/ThreadPoolStatsCollector.cs b/src/prometheus-net.DotNetRuntime/StatsCollectors/ThreadPoolStatsCollector.cs deleted file mode 100644 index 61518e6..0000000 --- a/src/prometheus-net.DotNetRuntime/StatsCollectors/ThreadPoolStatsCollector.cs +++ /dev/null @@ -1,76 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.Tracing; -#if PROMV2 -using Prometheus.Advanced; -#endif -using Prometheus.DotNetRuntime.EventSources; -using Prometheus.DotNetRuntime.StatsCollectors.Util; - -namespace Prometheus.DotNetRuntime.StatsCollectors -{ - /// - /// Measures the size of the worker + IO thread pools, worker pool throughput and reasons for worker pool - /// adjustments. - /// - public class ThreadPoolStatsCollector : IEventSourceStatsCollector - { - private const int - EventIdThreadPoolSample = 54, - EventIdThreadPoolAdjustment = 55, - EventIdIoThreadCreate = 44, - EventIdIoThreadRetire = 46, - EventIdIoThreadUnretire = 47, - EventIdIoThreadTerminate = 45; - - private Dictionary _adjustmentReasonToLabel = LabelGenerator.MapEnumToLabelValues(); - - internal Gauge NumThreads { get; private set; } - internal Gauge NumIocThreads { get; private set; } - - // TODO resolve issue where throughput cannot be calculated (stats event is giving garbage values) - // internal Counter Throughput { get; private set; } - internal Counter AdjustmentsTotal { get; private set; } - - public Guid EventSourceGuid => DotNetRuntimeEventSource.Id; - public EventKeywords Keywords => (EventKeywords) DotNetRuntimeEventSource.Keywords.Threading; - public EventLevel Level => EventLevel.Informational; - - public void RegisterMetrics(MetricFactory metrics) - { - NumThreads = metrics.CreateGauge("dotnet_threadpool_num_threads", "The number of active threads in the thread pool"); - NumIocThreads = metrics.CreateGauge("dotnet_threadpool_io_num_threads", "The number of active threads in the IO thread pool"); - // Throughput = metrics.CreateCounter("dotnet_threadpool_throughput_total", "The total number of work items that have finished execution in the thread pool"); - AdjustmentsTotal = metrics.CreateCounter( - "dotnet_threadpool_adjustments_total", - "The total number of changes made to the size of the thread pool, labeled by the reason for change", - "adjustment_reason"); - } - - public void UpdateMetrics() - { - } - - public void ProcessEvent(EventWrittenEventArgs e) - { - switch (e.EventId) - { - case EventIdThreadPoolSample: - // Throughput.Inc((double) e.Payload[0]); - return; - - case EventIdThreadPoolAdjustment: - NumThreads.Set((uint) e.Payload[1]); - AdjustmentsTotal.Labels(_adjustmentReasonToLabel[(DotNetRuntimeEventSource.ThreadAdjustmentReason) e.Payload[2]]).Inc(); - return; - - case EventIdIoThreadCreate: - case EventIdIoThreadRetire: - case EventIdIoThreadUnretire: - case EventIdIoThreadTerminate: - NumIocThreads.Set((uint)e.Payload[0]); - return; - } - } - } -} \ No newline at end of file diff --git a/src/prometheus-net.DotNetRuntime/prometheus-net.DotNetRuntime.csproj b/src/prometheus-net.DotNetRuntime/prometheus-net.DotNetRuntime.csproj index 73e4d02..25021ab 100644 --- a/src/prometheus-net.DotNetRuntime/prometheus-net.DotNetRuntime.csproj +++ b/src/prometheus-net.DotNetRuntime/prometheus-net.DotNetRuntime.csproj @@ -5,8 +5,6 @@ Prometheus.DotNetRuntime prometheus-net.DotNetRuntime prometheus-net.DotNetRuntime - - $(PromMajorVersion).4.1 James Luck Prometheus prometheus-net IOnDemandCollector runtime metrics gc jit threadpool contention stats https://github.com/djluck/prometheus-net.DotNetRuntime @@ -14,22 +12,24 @@ Exposes .NET core runtime metrics (GC, JIT, lock contention, thread pool, exceptions) using the prometheus-net package. https://github.com/djluck/prometheus-net.DotNetRuntime/blob/master/LICENSE.txt - ReleaseV2;DebugV2;DebugV3;ReleaseV3 AnyCPU - netcoreapp2.2;netcoreapp3.0;netstandard2.1;net5.0 + net5.0;netcoreapp3.1;netstandard2.1 true 1701;1702;CS1591; + disable + 9 + true - - - bin\ReleaseV2\prometheus-net.DotNetRuntime.xml - - - bin\ReleaseV3\prometheus-net.DotNetRuntime.xml - + + + + + + + diff --git a/tools/DocsGenerator/DocsGenerator.csproj b/tools/DocsGenerator/DocsGenerator.csproj new file mode 100644 index 0000000..9e3c91b --- /dev/null +++ b/tools/DocsGenerator/DocsGenerator.csproj @@ -0,0 +1,19 @@ + + + + Exe + netcoreapp3.1 + 9 + + + + + + + + + + + + + diff --git a/tools/DocsGenerator/Program.cs b/tools/DocsGenerator/Program.cs new file mode 100644 index 0000000..5d7e110 --- /dev/null +++ b/tools/DocsGenerator/Program.cs @@ -0,0 +1,209 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Linq.Expressions; +using System.Reflection; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Fasterflect; +using Grynwald.MarkdownGenerator; +using Prometheus; +using Prometheus.DotNetRuntime; +using Prometheus.DotNetRuntime.Metrics.Producers; + +namespace DocsGenerator +{ + class Program + { + static async Task Main(string[] args) + { + var sources = new [] + { + SourceAndConfig.CreateFrom(b => b.WithThreadPoolStats(CaptureLevel.Counters, new ThreadPoolMetricsProducer.Options())), + SourceAndConfig.CreateFrom(b => b.WithThreadPoolStats(CaptureLevel.Informational, new ThreadPoolMetricsProducer.Options())), + SourceAndConfig.CreateFrom(b => b.WithGcStats(CaptureLevel.Counters, new double[0])), + SourceAndConfig.CreateFrom(b => b.WithGcStats(CaptureLevel.Informational, new double[0])), + SourceAndConfig.CreateFrom(b => b.WithGcStats(CaptureLevel.Verbose, new double[0])), + SourceAndConfig.CreateFrom(b => b.WithContentionStats(CaptureLevel.Counters, SampleEvery.OneEvent)), + SourceAndConfig.CreateFrom(b => b.WithContentionStats(CaptureLevel.Informational, SampleEvery.OneEvent)), + SourceAndConfig.CreateFrom(b => b.WithExceptionStats(CaptureLevel.Counters)), + SourceAndConfig.CreateFrom(b => b.WithExceptionStats(CaptureLevel.Errors)), + new SourceAndConfig(new Source(typeof(DotNetRuntimeStatsBuilder.Builder).GetMethod(nameof(DotNetRuntimeStatsBuilder.Builder.WithJitStats)), CaptureLevel.Verbose), b => b.WithJitStats(SampleEvery.OneEvent)) + }; + + var assemblyDocs = typeof(DotNetRuntimeStatsBuilder).Assembly.LoadXmlDocumentation(); + + var allMetrics = GetAllMetrics(sources); + + MdSpan[] GetCells(Collector m) + { + return new MdSpan[] + { + new MdRawMarkdownSpan($"`{m.Name}`"), + new MdRawMarkdownSpan($"`{m.GetType().Name}`"), + m.Help, + new MdRawMarkdownSpan(string.Join(", ", m.LabelNames.Select(x => $"`{x}`"))), + }; + } + + + + var document = new MdDocument(); + var root = document.Root; + root.Add(new MdHeading("Metrics exposed", 1)); + root.Add(new MdParagraph(new MdRawMarkdownSpan($"A breakdown of all the metrics exposed by this library. Each subheading details the metrics produced by calling builder methods with the specified `{nameof(CaptureLevel)}`."))); + + root.Add(new MdHeading("Default metrics", 2)); + root.Add(new MdParagraph("Metrics that are included by default, regardless of what stats collectors are enabled.")); + + root.Add(new MdTable(headerRow: new MdTableRow("Name", "Type", "Description", "Labels"), + allMetrics.CommonMetrics.Select(x => new MdTableRow(GetCells(x))).ToArray())); + + foreach (var methodAndSources in allMetrics.MethodsToSources) + { + root.Add(new MdHeading(new MdCodeSpan($".{methodAndSources.method.Name}()"), 2)); + root.Add(new MdParagraph(assemblyDocs.GetDocumentation(methodAndSources.method).Summary)); + + for (var i = 0; i < methodAndSources.sources.Length; i++) + { + var s = methodAndSources.sources[i]; + root.Add(new MdHeading(new MdCodeSpan($"{nameof(CaptureLevel)}." + s.Level), 3)); + + var previousLevels = methodAndSources.sources.Take(i).ToArray(); + + if (previousLevels.Length > 0) + { + root.Add(new MdParagraph( + new MdRawMarkdownSpan($"Includes metrics generated by {string.Join(", ", previousLevels.Select(c => $"`{nameof(CaptureLevel)}.{c.Level}`"))} plus:") + )); + } + + root.Add(new MdTable(headerRow: new MdTableRow("Name", "Type", "Description", "Labels"), + allMetrics.SourceToMetrics[s].Select(x => new MdTableRow(GetCells(x.Collector))).ToArray())); + } + } + + + document.Save("../../../../../docs/metrics-exposed.md"); + } + + private static GroupedMetrics GetAllMetrics(SourceAndConfig[] sources) + { + var methodsOrderedByLevel = sources + .GroupBy( + x => x.Source.Method, + v => v, + (m, levels) => (method: m, orderedConfigs: levels.OrderBy(l => l.Source.Level).ToArray()) + ) + .ToArray(); + + // Find all metrics exposed + var sourceToMetrics = methodsOrderedByLevel + .SelectMany(x => + x.orderedConfigs + // Build up metrics, going from lowest capture level (counters) to highest capture level + // This ensures metrics present across multiple capture levels are only recorded once + .Aggregate( + ImmutableHashSet.Empty.WithComparer(new MetricEquality()), + (acc, next) => acc.Union(GetExposedMetric(next)) + ) + ) + .GroupBy(x => x.Source) + .ToDictionary(k => k.Key, v => v.ToList()); + + // Find common metrics + var commonMetricsGrouped = sourceToMetrics.SelectMany(v => v.Value) + .GroupBy(x => x, new MetricEquality()) + .Where(x => x.Count() == methodsOrderedByLevel.Length); + + var commonExposedMetrics = commonMetricsGrouped + .Select(x => x.Key) + .ToHashSet(new MetricEquality()); + + // Remove common metrics from all sources + foreach (var s in sourceToMetrics) + s.Value.RemoveAll(x => commonExposedMetrics.Contains(x)); + + return new GroupedMetrics( + commonExposedMetrics.Select(x => x.Collector).ToImmutableList(), + sourceToMetrics.ToImmutableDictionary( + k => k.Key, + v => v.Value.Select(i => i).ToImmutableList() + ) + ); + } + + private static IEnumerable GetExposedMetric(SourceAndConfig source) + { + Console.WriteLine($"Getting metrics for {source}.."); + + // Start collector + var registry = new CollectorRegistry(); + using var statsCollector = source.ApplyConfig(DotNetRuntimeStatsBuilder.Customize()).StartCollecting(registry); + // Wait for metrics to be available (hacky!) + Thread.Sleep(1500); + + // Pull registered collectors + var collectors = registry.TryGetFieldValue("_collectors", Flags.InstancePrivate) as ConcurrentDictionary; + + return collectors.Values.Select(c => new ExposedMetric(c, source.Source)); + } + + public record Source(MethodInfo Method, CaptureLevel Level) + { + public CaptureLevel Level { get; } = Level; + public MethodInfo Method { get; } = Method; + } + + public record SourceAndConfig(Source Source, Func ApplyConfig) + { + public Source Source { get; } = Source; + public Func ApplyConfig { get; } = ApplyConfig; + + public static SourceAndConfig CreateFrom(Expression> fromMethod) + { + var mCall = (fromMethod.Body as MethodCallExpression); + var method = mCall.Method; + var captureLevel = ( CaptureLevel)mCall.Arguments.OfType().Single(x => x.Type == typeof(CaptureLevel)).Value; + + return new SourceAndConfig(new Source(method, captureLevel), fromMethod.Compile()); + } + } + + public record ExposedMetric(Collector Collector, Source Source) + { + public Collector Collector { get; } = Collector; + public Source Source { get; } = Source; + + public string Type => Collector.GetType().Name; + public string Labels => string.Join(", ", Collector.LabelNames); + } + + public class MetricEquality : IEqualityComparer + { + public bool Equals(ExposedMetric x, ExposedMetric y) + { + if (ReferenceEquals(x, y)) return true; + if (ReferenceEquals(x, null)) return false; + if (ReferenceEquals(y, null)) return false; + if (x.GetType() != y.GetType()) return false; + return x.Collector.Name == y.Collector.Name && x.Collector.Help == y.Collector.Help && x.Labels == y.Labels && x.Type == y.Type; + } + + public int GetHashCode(ExposedMetric obj) + { + return HashCode.Combine(obj.Collector.Name, obj.Collector.Help, obj.Labels, obj.Type); + } + } + + public record GroupedMetrics(ImmutableList CommonMetrics, ImmutableDictionary> SourceToMetrics) + { + public ImmutableDictionary> SourceToMetrics { get; } = SourceToMetrics; + public ImmutableList CommonMetrics { get; } = CommonMetrics; + public IEnumerable<(MethodInfo method, Source[] sources)> MethodsToSources => SourceToMetrics.Keys.GroupBy(x => x.Method, (k, v) => (method: k, sources: v.OrderBy(x => x.Level).ToArray())); + } + } +} \ No newline at end of file diff --git a/tools/DocsGenerator/README.md b/tools/DocsGenerator/README.md new file mode 100644 index 0000000..9b4126d --- /dev/null +++ b/tools/DocsGenerator/README.md @@ -0,0 +1,2 @@ +# DocsGenerator +A small tool used to generate [docs/metrics-exposed.md](docs/metrics-exposed.md). diff --git a/tools/DocsGenerator/XmlDocReading.cs b/tools/DocsGenerator/XmlDocReading.cs new file mode 100644 index 0000000..fce6236 --- /dev/null +++ b/tools/DocsGenerator/XmlDocReading.cs @@ -0,0 +1,108 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text.RegularExpressions; +using System.Xml; +using System.Xml.Linq; + +namespace DocsGenerator +{ + public static class Extensions + { + private static string GetDirectoryPath(this Assembly assembly) + { + string codeBase = assembly.CodeBase; + UriBuilder uri = new UriBuilder(codeBase); + string path = Uri.UnescapeDataString(uri.Path); + return Path.GetDirectoryName(path); + } + + public static AssemblyXmlDocs LoadXmlDocumentation(this Assembly assembly) + { + string directoryPath = assembly.GetDirectoryPath(); + string xmlFilePath = Path.Combine(directoryPath, assembly.GetName().Name + ".xml"); + if (File.Exists(xmlFilePath)) { + return new AssemblyXmlDocs(assembly, (File.ReadAllText(xmlFilePath))); + } + + throw new FileNotFoundException("Unable to locate xmldoc", xmlFilePath); + } + + } + + /// + /// Super hacky! + /// + public class AssemblyXmlDocs + { + private Dictionary _loadedXmlDocumentation = new(); + + public AssemblyXmlDocs(Assembly assembly, string xml) + { + var doc = XDocument.Parse(xml); + + _loadedXmlDocumentation = doc.Descendants("member") + .Select(m => + { + var nameAttr = m.Attribute("name"); + // e.g. M:Prometheus.DotNetRuntime.DotNetRuntimeStatsBuilder.Customize + var match = Regex.Match(nameAttr.Value, @"(?[\w]):(?[^\(]+)\.(?[^\.\(]+)"); + + var memberType = match.Groups["member_type"].Value switch + { + "T" => MemberTypes.TypeInfo, + "M" => MemberTypes.Method, + "P" => MemberTypes.Property, + "F" => MemberTypes.Field + }; + + var parentTypeName = match.Groups["parent_type"].Value; + var parentType = assembly.GetType(parentTypeName); + + if (parentType == null && memberType != MemberTypes.TypeInfo) // TypeInfo doesn't currently work + { + // SO. HACKY. + parentType = assembly.GetType("Prometheus.DotNetRuntime.DotNetRuntimeStatsBuilder+Builder"); + } + if (parentType == null && memberType != MemberTypes.TypeInfo) // TypeInfo doesn't currently work + { + throw new InvalidOperationException($"Could not locate type '{parentTypeName}'"); + } + var key = new DocKey(memberType, parentType, match.Groups["member_name"].Value); + + return (key, memberDocs: new MemberDocs() + { + Summary = m.Descendants("summary").SingleOrDefault()?.Value?.Trim() + }); + }) + .GroupBy(x => x.key) + // TODO if we ever need rely on override docs working correctly, fix this + .ToDictionary(k => k.Key, v => v.First().memberDocs); + + } + + public MemberDocs GetDocumentation(MethodInfo methodInfo) + { + if (_loadedXmlDocumentation.TryGetValue((new DocKey(MemberTypes.Method, methodInfo.DeclaringType, methodInfo.Name)), out var documentation)) + { + return documentation; + } + + throw new KeyNotFoundException($"Could not locate docs for {methodInfo}"); + } + + public class MemberDocs + { + public string Summary { get; set; } + } + } + + internal record DocKey(MemberTypes MemberType, Type ParentType, string Name) + { + public MemberTypes MemberType { get; } = MemberType; + public Type ParentType { get; } = ParentType; + public string Name { get; } = Name; + } +} \ No newline at end of file