Add cache hits/misses metrics (DB-20)

src/EventStore.ClusterNode/telemetryconfig.json

@@ -54,6 +54,11 @@
 		"FlushDuration": true
 	},
 
+	"CacheHitsMisses": {
+		"StreamInfo": true,
+		"Chunk": false
+	},
+
 	// this specifies what name to track each queue under, according to regular expressions to
 	// match the queue names against
 	"Queues": [

src/EventStore.Common/Configuration/TelemetryConfiguration.cs

@@ -84,6 +84,11 @@ public enum EventTracker {
 			Written,
 		}
 
+		public enum Cache {
+			StreamInfo = 1,
+			Chunk,
+		}
+
 		public class LabelMappingCase {
 			public string Regex { get; set; }
 			public string Label { get; set; }
@@ -105,6 +110,8 @@ public class LabelMappingCase {
 
 		public Dictionary<EventTracker, bool> Events { get; set; } = new();
 
+		public Dictionary<Cache, bool> CacheHitsMisses { get; set; } = new();
+
 		// must be 0, 1, 5, 10 or a multiple of 15
 		public int ExpectedScrapeIntervalSeconds { get; set; }
 

src/EventStore.Core.Tests/DisposableExtensions.cs

@@ -0,0 +1,12 @@
+using System;
+
+namespace EventStore.Core.Tests;
+
+public static class DisposableExtensions {
+	public static T DisposeWith<T>(this T disposable, Disposables disposables)
+		where T : IDisposable {
+
+		disposables.Add(disposable);
+		return disposable;
+	}
+}

src/EventStore.Core.Tests/Disposables.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+
+namespace EventStore.Core.Tests;
+
+public sealed class Disposables : IDisposable {
+	private readonly List<IDisposable> _disposables = new();
+
+	public void Dispose() {
+		for (int i = _disposables.Count - 1; i >= 0; i--) {
+			_disposables[i]?.Dispose();
+		}
+	}
+
+	public void Add(IDisposable disposable) =>
+		_disposables.Add(disposable);
+}

src/EventStore.Core.Tests/Services/Storage/BuildingIndex/when_building_an_index_off_tfile_with_duplicate_events_in_a_stream.cs

@@ -21,6 +21,7 @@
 using EventStore.Core.TransactionLog.LogRecords;
 using NUnit.Framework;
 using ReadStreamResult = EventStore.Core.Services.Storage.ReaderIndex.ReadStreamResult;
+using EventStore.Core.Telemetry;
 
 namespace EventStore.Core.Tests.Services.Storage.BuildingIndex {
 	[TestFixture(typeof(LogFormat.V2), typeof(string))]
@@ -170,7 +171,8 @@ public abstract class DuplicateReadIndexTestScenario<TLogFormat, TStreamId> : Sp
 				replicationCheckpoint: _db.Config.ReplicationCheckpoint,
 				indexCheckpoint: _db.Config.IndexCheckpoint,
 				indexStatusTracker: new IndexStatusTracker.NoOp(),
-				indexTracker: new IndexTracker.NoOp());
+				indexTracker: new IndexTracker.NoOp(),
+				cacheTracker: new CacheHitsMissesTracker.NoOp());
 
 
 			readIndex.IndexCommitter.Init(chaserCheckpoint.Read());
@@ -217,7 +219,8 @@ public abstract class DuplicateReadIndexTestScenario<TLogFormat, TStreamId> : Sp
 				replicationCheckpoint: _db.Config.ReplicationCheckpoint,
 				indexCheckpoint: _db.Config.IndexCheckpoint,
 				indexStatusTracker: new IndexStatusTracker.NoOp(),
-				indexTracker: new IndexTracker.NoOp());
+				indexTracker: new IndexTracker.NoOp(),
+				cacheTracker: new CacheHitsMissesTracker.NoOp());
 
 			readIndex.IndexCommitter.Init(chaserCheckpoint.Read());
 			ReadIndex = readIndex;

src/EventStore.Core.Tests/Services/Storage/ReadIndexTestScenario.cs

@@ -22,6 +22,7 @@
 using EventStore.Core.LogV3;
 using EventStore.Core.Tests.TransactionLog.Scavenging.Helpers;
 using EventStore.LogCommon;
+using EventStore.Core.Telemetry;
 
 namespace EventStore.Core.Tests.Services.Storage {
 	public abstract class ReadIndexTestScenario<TLogFormat, TStreamId> : SpecificationWithDirectoryPerTestFixture {
@@ -139,7 +140,8 @@ public abstract class ReadIndexTestScenario<TLogFormat, TStreamId> : Specificati
 				replicationCheckpoint: Db.Config.ReplicationCheckpoint,
 				indexCheckpoint: Db.Config.IndexCheckpoint,
 				indexStatusTracker: new IndexStatusTracker.NoOp(),
-				indexTracker: new IndexTracker.NoOp());
+				indexTracker: new IndexTracker.NoOp(),
+				cacheTracker: new CacheHitsMissesTracker.NoOp());
 
 			readIndex.IndexCommitter.Init(ChaserCheckpoint.Read());
 			ReadIndex = readIndex;

src/EventStore.Core.Tests/Services/Storage/RepeatableDbTestScenario.cs

@@ -16,6 +16,7 @@
 using System;
 using System.Threading.Tasks;
 using EventStore.Core.Caching;
+using EventStore.Core.Telemetry;
 
 namespace EventStore.Core.Tests.Services.Storage {
 	[TestFixture]
@@ -91,7 +92,8 @@ public abstract class RepeatableDbTestScenario<TLogFormat, TStreamId> : Specific
 				replicationCheckpoint: DbRes.Db.Config.ReplicationCheckpoint,
 				indexCheckpoint: DbRes.Db.Config.IndexCheckpoint,
 				indexStatusTracker: new IndexStatusTracker.NoOp(),
-				indexTracker: new IndexTracker.NoOp());
+				indexTracker: new IndexTracker.NoOp(),
+				cacheTracker: new CacheHitsMissesTracker.NoOp());
 
 			readIndex.IndexCommitter.Init(DbRes.Db.Config.ChaserCheckpoint.Read());
 			ReadIndex = readIndex;

src/EventStore.Core.Tests/Services/Storage/SimpleDbTestScenario.cs

@@ -6,6 +6,7 @@
 using EventStore.Core.Index.Hashes;
 using EventStore.Core.LogAbstraction;
 using EventStore.Core.Services.Storage.ReaderIndex;
+using EventStore.Core.Telemetry;
 using EventStore.Core.Tests.Fakes;
 using EventStore.Core.Tests.TransactionLog;
 using EventStore.Core.Tests.TransactionLog.Scavenging.Helpers;
@@ -89,7 +90,8 @@ public abstract class SimpleDbTestScenario<TLogFormat, TStreamId> : Specificatio
 				replicationCheckpoint: DbRes.Db.Config.ReplicationCheckpoint,
 				indexCheckpoint: DbRes.Db.Config.IndexCheckpoint,
 				indexStatusTracker: new IndexStatusTracker.NoOp(),
-				indexTracker: new IndexTracker.NoOp());
+				indexTracker: new IndexTracker.NoOp(),
+				cacheTracker: new CacheHitsMissesTracker.NoOp());
 
 			readIndex.IndexCommitter.Init(DbRes.Db.Config.ChaserCheckpoint.Read());
 			ReadIndex = readIndex;

src/EventStore.Core.Tests/Services/Storage/Transactions/when_rebuilding_index_for_partially_persisted_transaction.cs

@@ -13,6 +13,7 @@
 using NUnit.Framework;
 using EventStore.Core.Util;
 using EventStore.Core.Index.Hashes;
+using EventStore.Core.Telemetry;
 
 namespace EventStore.Core.Tests.Services.Storage.Transactions {
 	[TestFixture(typeof(LogFormat.V2), typeof(string))]
@@ -60,7 +61,8 @@ public class when_rebuilding_index_for_partially_persisted_transaction<TLogForma
 				replicationCheckpoint: Db.Config.ReplicationCheckpoint,
 				indexCheckpoint: Db.Config.IndexCheckpoint,
 				indexStatusTracker: new IndexStatusTracker.NoOp(),
-				indexTracker: new IndexTracker.NoOp());
+				indexTracker: new IndexTracker.NoOp(),
+				cacheTracker: new CacheHitsMissesTracker.NoOp());
 			readIndex.IndexCommitter.Init(ChaserCheckpoint.Read());
 			ReadIndex = readIndex;
 		}

src/EventStore.Core.Tests/TransactionLog/Scavenging/Helpers/ScavengeTestScenario.cs

@@ -15,6 +15,7 @@
 using NUnit.Framework;
 using EventStore.Core.Util;
 using EventStore.Core.LogAbstraction;
+using EventStore.Core.Telemetry;
 
 namespace EventStore.Core.Tests.TransactionLog.Scavenging.Helpers {
 	[TestFixture]
@@ -84,7 +85,10 @@ public abstract class ScavengeTestScenario<TLogFormat, TStreamId> : Specificatio
 				new LRUCache<TStreamId, IndexBackend<TStreamId>.MetadataCached>("StreamMetadata", 100),
 				true, _metastreamMaxCount,
 				Opts.HashCollisionReadLimitDefault, Opts.SkipIndexScanOnReadsDefault,
-				_dbResult.Db.Config.ReplicationCheckpoint,_dbResult.Db.Config.IndexCheckpoint, new IndexStatusTracker.NoOp(), new IndexTracker.NoOp());
+				_dbResult.Db.Config.ReplicationCheckpoint,_dbResult.Db.Config.IndexCheckpoint,
+				new IndexStatusTracker.NoOp(),
+				new IndexTracker.NoOp(),
+				new CacheHitsMissesTracker.NoOp());
 			readIndex.IndexCommitter.Init(_dbResult.Db.Config.WriterCheckpoint.Read());
 			ReadIndex = readIndex;
 

src/EventStore.Core.Tests/TransactionLog/Truncation/TruncateAndReOpenDbScenario.cs

@@ -13,6 +13,7 @@
 using EventStore.Core.Util;
 using EventStore.Core.Index.Hashes;
 using EventStore.Core.Tests.Services;
+using EventStore.Core.Telemetry;
 
 namespace EventStore.Core.Tests.TransactionLog.Truncation {
 	public abstract class TruncateAndReOpenDbScenario<TLogFormat, TStreamId> : TruncateScenario<TLogFormat, TStreamId> {
@@ -69,7 +70,8 @@ protected TruncateAndReOpenDbScenario(int maxEntriesInMemTable = 100, int metast
 				replicationCheckpoint: Db.Config.ReplicationCheckpoint,
 				indexCheckpoint: Db.Config.IndexCheckpoint,
 				indexStatusTracker: new IndexStatusTracker.NoOp(),
-				indexTracker: new IndexTracker.NoOp());
+				indexTracker: new IndexTracker.NoOp(),
+				cacheTracker: new CacheHitsMissesTracker.NoOp());
 			readIndex.IndexCommitter.Init(ChaserCheckpoint.Read());
 			ReadIndex = readIndex;
 		}

src/EventStore.Core.XUnit.Tests/Scavenge/Infrastructure/Scenario.cs

@@ -12,6 +12,7 @@
 using EventStore.Core.LogAbstraction;
 using EventStore.Core.Services.Storage.ReaderIndex;
 using EventStore.Core.Settings;
+using EventStore.Core.Telemetry;
 using EventStore.Core.Tests;
 using EventStore.Core.Tests.Fakes;
 using EventStore.Core.Tests.Index.Hashers;
@@ -284,7 +285,8 @@ public class Scenario<TLogFormat, TStreamId> : Scenario {
 				replicationCheckpoint: dbResult.Db.Config.ReplicationCheckpoint,
 				indexCheckpoint: dbResult.Db.Config.IndexCheckpoint,
 				indexStatusTracker: new IndexStatusTracker.NoOp(),
-				indexTracker: new IndexTracker.NoOp());
+				indexTracker: new IndexTracker.NoOp(),
+				cacheTracker: new CacheHitsMissesTracker.NoOp());
 
 			readIndex.IndexCommitter.Init(dbResult.Db.Config.WriterCheckpoint.Read());
 			// wait for tables to be merged. for one of the tests this takes a while

src/EventStore.Core.XUnit.Tests/Telemetry/CacheHitsMiseesTrackerTests.cs

@@ -0,0 +1,91 @@
+using System;
+using System.Diagnostics.Metrics;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using EventStore.Core.Telemetry;
+using EventStore.Core.Tests;
+using Xunit;
+
+using static EventStore.Common.Configuration.TelemetryConfiguration;
+
+namespace EventStore.Core.XUnit.Tests.Telemetry;
+
+public sealed class CacheHitsMissesTrackerTests : IDisposable {
+	private readonly Disposables _disposables = new();
+
+	public void Dispose() {
+		_disposables?.Dispose();
+	}
+
+	private (CacheHitsMissesTracker, TestMeterListener<long>) GenSut(
+		Cache[] enabledCaches,
+		[CallerMemberName] string callerName = "") {
+
+		var meter = new Meter($"{typeof(CacheHitsMissesTrackerTests)}-{callerName}").DisposeWith(_disposables);
+		var listener = new TestMeterListener<long>(meter).DisposeWith(_disposables);
+		var metric = new CacheHitsMissesMetric(meter, enabledCaches, "the-metric", new() {
+			{ Cache.StreamInfo, "stream-info" },
+			{ Cache.Chunk, "chunk" },
+		});
+		var sut =  new CacheHitsMissesTracker(metric);
+		return (sut, listener);
+	}
+
+	[Fact]
+	public void observes_all_caches() {
+		var (sut, listener) = GenSut(new[] { Cache.Chunk, Cache.StreamInfo });
+		sut.Register(Cache.Chunk, () => 1, () => 2);
+		sut.Register(Cache.StreamInfo, () => 3, () => 4);
+		AssertMeasurements(listener,
+			AssertMeasurement("chunk", "hits", 1),
+			AssertMeasurement("chunk", "misses", 2),
+			AssertMeasurement("stream-info", "hits", 3),
+			AssertMeasurement("stream-info", "misses", 4));
+	}
+
+	[Fact]
+	public void ignores_disabled_cache() {
+		var (sut, listener) = GenSut(new[] { Cache.StreamInfo });
+		sut.Register(Cache.Chunk, () => 1, () => 2);
+		sut.Register(Cache.StreamInfo, () => 3, () => 4);
+		AssertMeasurements(listener,
+			AssertMeasurement("stream-info", "hits", 3),
+			AssertMeasurement("stream-info", "misses", 4));
+	}
+
+	[Fact]
+	public void ignores_unregistered_cache() {
+		var (sut, listener) = GenSut(new[] { Cache.Chunk, Cache.StreamInfo });
+		sut.Register(Cache.Chunk, () => 1, () => 2);
+		AssertMeasurements(listener,
+			AssertMeasurement("chunk", "hits", 1),
+			AssertMeasurement("chunk", "misses", 2));
+	}
+
+	static Action<TestMeterListener<long>.TestMeasurement> AssertMeasurement(
+		string cacheName,
+		string kind,
+		long expectedValue) =>
+
+		actualMeasurement => {
+			Assert.Equal(expectedValue, actualMeasurement.Value);
+			Assert.Collection(
+				actualMeasurement.Tags.ToArray(),
+				tag => {
+					Assert.Equal("cache", tag.Key);
+					Assert.Equal(cacheName, tag.Value);
+				},
+				tag => {
+					Assert.Equal("kind", tag.Key);
+					Assert.Equal(kind, tag.Value);
+				});
+		};
+
+	static void AssertMeasurements(
+		TestMeterListener<long> listener,
+		params Action<TestMeterListener<long>.TestMeasurement>[] actions) {
+
+		listener.Observe();
+		Assert.Collection(listener.RetrieveMeasurements("the-metric"), actions);
+	}
+}

src/EventStore.Core/ClusterVNode.cs

@@ -671,7 +671,8 @@ public class ClusterVNode<TStreamId> :
 				Db.Config.ReplicationCheckpoint.AsReadOnly(),
 				Db.Config.IndexCheckpoint,
 				trackers.IndexStatusTracker,
-				trackers.IndexTracker);
+				trackers.IndexTracker,
+				trackers.CacheHitsMissesTracker);
 			_readIndex = readIndex;
 			var writer = new TFChunkWriter(Db);
 

src/EventStore.Core/MetricsBootstrapper.cs

@@ -27,6 +27,7 @@ public class Trackers {
 	public IIndexTracker IndexTracker { get; set; } = new IndexTracker.NoOp();
 	public IMaxTracker<long> WriterFlushSizeTracker { get; set; } = new MaxTracker<long>.NoOp();
 	public IDurationMaxTracker WriterFlushDurationTracker { get; set; } = new DurationMaxTracker.NoOp();
+	public ICacheHitsMissesTracker CacheHitsMissesTracker { get; set; } = new CacheHitsMissesTracker.NoOp();
 }
 
 public class GrpcTrackers {
@@ -82,6 +83,16 @@ public static class MetricsBootstrapper {
 			_ = new IncomingGrpcCallsMetric(coreMeter, "eventstore-incoming-grpc-calls", enabledCalls);
 		}
 
+		// cache hits/misses
+		var enabledCacheHitsMisses = conf.CacheHitsMisses.Where(kvp => kvp.Value).Select(kvp => kvp.Key).ToArray();
+		if (enabledCacheHitsMisses.Length > 0) {
+			var metric = new CacheHitsMissesMetric(coreMeter, enabledCacheHitsMisses, "eventstore-cache-hits-misses", new() {
+				{ Conf.Cache.StreamInfo, "stream-info" },
+				{ Conf.Cache.Chunk, "chunk" },
+			});
+			trackers.CacheHitsMissesTracker = new CacheHitsMissesTracker(metric);
+		}
+
 		// events
 		if (conf.Events.TryGetValue(Conf.EventTracker.Read, out var readEnabled) && readEnabled) {
 			var readTag = new KeyValuePair<string, object>("activity", "read");

src/EventStore.Core/Services/Storage/ReaderIndex/ReadIndex.cs

@@ -9,10 +9,12 @@
 using EventStore.Core.Index;
 using EventStore.Core.LogAbstraction;
 using EventStore.Core.Messages;
+using EventStore.Core.Telemetry;
 using EventStore.Core.TransactionLog;
 using EventStore.Core.TransactionLog.Checkpoint;
 using EventStore.Core.TransactionLog.Chunks;
 using EventStore.Core.Util;
+using static EventStore.Common.Configuration.TelemetryConfiguration;
 
 namespace EventStore.Core.Services.Storage.ReaderIndex {
 	public sealed class ReadIndex<TStreamId> : IDisposable, IReadIndex<TStreamId> {
@@ -56,7 +58,8 @@ public sealed class ReadIndex<TStreamId> : IDisposable, IReadIndex<TStreamId> {
 			IReadOnlyCheckpoint replicationCheckpoint,
 			ICheckpoint indexCheckpoint,
 			IIndexStatusTracker indexStatusTracker,
-			IIndexTracker indexTracker) {
+			IIndexTracker indexTracker,
+			ICacheHitsMissesTracker cacheTracker) {
 
 			Ensure.NotNull(bus, "bus");
 			Ensure.NotNull(readerPool, "readerPool");
@@ -91,6 +94,8 @@ public sealed class ReadIndex<TStreamId> : IDisposable, IReadIndex<TStreamId> {
 				_streamNames, eventTypeIndex, eventTypeNames, systemStreams, streamExistenceFilter,
 				streamExistenceFilterInitializer, indexCheckpoint, indexStatusTracker, indexTracker, additionalCommitChecks);
 			_allReader = new AllReader<TStreamId>(indexBackend, _indexCommitter, _streamNames, eventTypeNames);
+
+			RegisterHitsMisses(cacheTracker);
 		}
 
 		IndexReadEventResult IReadIndex<TStreamId>.ReadEvent(string streamName, TStreamId streamId, long eventNumber) {
@@ -181,6 +186,18 @@ public sealed class ReadIndex<TStreamId> : IDisposable, IReadIndex<TStreamId> {
 			return _indexReader.GetEffectiveAcl(streamId);
 		}
 
+		void RegisterHitsMisses(ICacheHitsMissesTracker tracker) {
+			tracker.Register(
+				Cache.Chunk,
+				() => Interlocked.Read(ref TFChunkReader.CachedReads),
+				() => Interlocked.Read(ref TFChunkReader.NotCachedReads));
+
+			tracker.Register(
+				Cache.StreamInfo,
+				() => _indexReader.CachedStreamInfo,
+				() => _indexReader.NotCachedStreamInfo);
+		}
+
 		ReadIndexStats IReadIndex.GetStatistics() {
 			return new ReadIndexStats(Interlocked.Read(ref TFChunkReader.CachedReads),
 				Interlocked.Read(ref TFChunkReader.NotCachedReads),