diff --git a/src/EventStore.ClusterNode/telemetryconfig.json b/src/EventStore.ClusterNode/telemetryconfig.json index 5a3c03da230..520057b2828 100644 --- a/src/EventStore.ClusterNode/telemetryconfig.json +++ b/src/EventStore.ClusterNode/telemetryconfig.json @@ -54,6 +54,11 @@ "FlushDuration": true }, + "CacheHitsMisses": { + "StreamInfo": true, + "Chunk": false + }, + // this specifies what name to track each queue under, according to regular expressions to // match the queue names against "Queues": [ diff --git a/src/EventStore.Common/Configuration/TelemetryConfiguration.cs b/src/EventStore.Common/Configuration/TelemetryConfiguration.cs index c2cc5da1aec..040df0b9d5d 100644 --- a/src/EventStore.Common/Configuration/TelemetryConfiguration.cs +++ b/src/EventStore.Common/Configuration/TelemetryConfiguration.cs @@ -84,6 +84,11 @@ public enum EventTracker { Written, } + public enum Cache { + StreamInfo = 1, + Chunk, + } + public class LabelMappingCase { public string Regex { get; set; } public string Label { get; set; } @@ -105,6 +110,8 @@ public class LabelMappingCase { public Dictionary Events { get; set; } = new(); + public Dictionary CacheHitsMisses { get; set; } = new(); + // must be 0, 1, 5, 10 or a multiple of 15 public int ExpectedScrapeIntervalSeconds { get; set; } diff --git a/src/EventStore.Core.Tests/DisposableExtensions.cs b/src/EventStore.Core.Tests/DisposableExtensions.cs new file mode 100644 index 00000000000..083a8591870 --- /dev/null +++ b/src/EventStore.Core.Tests/DisposableExtensions.cs @@ -0,0 +1,12 @@ +using System; + +namespace EventStore.Core.Tests; + +public static class DisposableExtensions { + public static T DisposeWith(this T disposable, Disposables disposables) + where T : IDisposable { + + disposables.Add(disposable); + return disposable; + } +} diff --git a/src/EventStore.Core.Tests/Disposables.cs b/src/EventStore.Core.Tests/Disposables.cs new file mode 100644 index 00000000000..2f7402e4071 --- /dev/null +++ b/src/EventStore.Core.Tests/Disposables.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; + +namespace EventStore.Core.Tests; + +public sealed class Disposables : IDisposable { + private readonly List _disposables = new(); + + public void Dispose() { + for (int i = _disposables.Count - 1; i >= 0; i--) { + _disposables[i]?.Dispose(); + } + } + + public void Add(IDisposable disposable) => + _disposables.Add(disposable); +} diff --git a/src/EventStore.Core.Tests/Services/Storage/BuildingIndex/when_building_an_index_off_tfile_with_duplicate_events_in_a_stream.cs b/src/EventStore.Core.Tests/Services/Storage/BuildingIndex/when_building_an_index_off_tfile_with_duplicate_events_in_a_stream.cs index a5095a34e36..f678c43d6db 100644 --- a/src/EventStore.Core.Tests/Services/Storage/BuildingIndex/when_building_an_index_off_tfile_with_duplicate_events_in_a_stream.cs +++ b/src/EventStore.Core.Tests/Services/Storage/BuildingIndex/when_building_an_index_off_tfile_with_duplicate_events_in_a_stream.cs @@ -21,6 +21,7 @@ using EventStore.Core.TransactionLog.LogRecords; using NUnit.Framework; using ReadStreamResult = EventStore.Core.Services.Storage.ReaderIndex.ReadStreamResult; +using EventStore.Core.Telemetry; namespace EventStore.Core.Tests.Services.Storage.BuildingIndex { [TestFixture(typeof(LogFormat.V2), typeof(string))] @@ -170,7 +171,8 @@ public abstract class DuplicateReadIndexTestScenario : Sp replicationCheckpoint: _db.Config.ReplicationCheckpoint, indexCheckpoint: _db.Config.IndexCheckpoint, indexStatusTracker: new IndexStatusTracker.NoOp(), - indexTracker: new IndexTracker.NoOp()); + indexTracker: new IndexTracker.NoOp(), + cacheTracker: new CacheHitsMissesTracker.NoOp()); readIndex.IndexCommitter.Init(chaserCheckpoint.Read()); @@ -217,7 +219,8 @@ public abstract class DuplicateReadIndexTestScenario : Sp replicationCheckpoint: _db.Config.ReplicationCheckpoint, indexCheckpoint: _db.Config.IndexCheckpoint, indexStatusTracker: new IndexStatusTracker.NoOp(), - indexTracker: new IndexTracker.NoOp()); + indexTracker: new IndexTracker.NoOp(), + cacheTracker: new CacheHitsMissesTracker.NoOp()); readIndex.IndexCommitter.Init(chaserCheckpoint.Read()); ReadIndex = readIndex; diff --git a/src/EventStore.Core.Tests/Services/Storage/ReadIndexTestScenario.cs b/src/EventStore.Core.Tests/Services/Storage/ReadIndexTestScenario.cs index ec530a1e740..944ccecab42 100644 --- a/src/EventStore.Core.Tests/Services/Storage/ReadIndexTestScenario.cs +++ b/src/EventStore.Core.Tests/Services/Storage/ReadIndexTestScenario.cs @@ -22,6 +22,7 @@ using EventStore.Core.LogV3; using EventStore.Core.Tests.TransactionLog.Scavenging.Helpers; using EventStore.LogCommon; +using EventStore.Core.Telemetry; namespace EventStore.Core.Tests.Services.Storage { public abstract class ReadIndexTestScenario : SpecificationWithDirectoryPerTestFixture { @@ -139,7 +140,8 @@ public abstract class ReadIndexTestScenario : Specificati replicationCheckpoint: Db.Config.ReplicationCheckpoint, indexCheckpoint: Db.Config.IndexCheckpoint, indexStatusTracker: new IndexStatusTracker.NoOp(), - indexTracker: new IndexTracker.NoOp()); + indexTracker: new IndexTracker.NoOp(), + cacheTracker: new CacheHitsMissesTracker.NoOp()); readIndex.IndexCommitter.Init(ChaserCheckpoint.Read()); ReadIndex = readIndex; diff --git a/src/EventStore.Core.Tests/Services/Storage/RepeatableDbTestScenario.cs b/src/EventStore.Core.Tests/Services/Storage/RepeatableDbTestScenario.cs index 28de02db1d7..f9edc5e8288 100644 --- a/src/EventStore.Core.Tests/Services/Storage/RepeatableDbTestScenario.cs +++ b/src/EventStore.Core.Tests/Services/Storage/RepeatableDbTestScenario.cs @@ -16,6 +16,7 @@ using System; using System.Threading.Tasks; using EventStore.Core.Caching; +using EventStore.Core.Telemetry; namespace EventStore.Core.Tests.Services.Storage { [TestFixture] @@ -91,7 +92,8 @@ public abstract class RepeatableDbTestScenario : Specific replicationCheckpoint: DbRes.Db.Config.ReplicationCheckpoint, indexCheckpoint: DbRes.Db.Config.IndexCheckpoint, indexStatusTracker: new IndexStatusTracker.NoOp(), - indexTracker: new IndexTracker.NoOp()); + indexTracker: new IndexTracker.NoOp(), + cacheTracker: new CacheHitsMissesTracker.NoOp()); readIndex.IndexCommitter.Init(DbRes.Db.Config.ChaserCheckpoint.Read()); ReadIndex = readIndex; diff --git a/src/EventStore.Core.Tests/Services/Storage/SimpleDbTestScenario.cs b/src/EventStore.Core.Tests/Services/Storage/SimpleDbTestScenario.cs index 2d314ec4abf..084056c1b01 100644 --- a/src/EventStore.Core.Tests/Services/Storage/SimpleDbTestScenario.cs +++ b/src/EventStore.Core.Tests/Services/Storage/SimpleDbTestScenario.cs @@ -6,6 +6,7 @@ using EventStore.Core.Index.Hashes; using EventStore.Core.LogAbstraction; using EventStore.Core.Services.Storage.ReaderIndex; +using EventStore.Core.Telemetry; using EventStore.Core.Tests.Fakes; using EventStore.Core.Tests.TransactionLog; using EventStore.Core.Tests.TransactionLog.Scavenging.Helpers; @@ -89,7 +90,8 @@ public abstract class SimpleDbTestScenario : Specificatio replicationCheckpoint: DbRes.Db.Config.ReplicationCheckpoint, indexCheckpoint: DbRes.Db.Config.IndexCheckpoint, indexStatusTracker: new IndexStatusTracker.NoOp(), - indexTracker: new IndexTracker.NoOp()); + indexTracker: new IndexTracker.NoOp(), + cacheTracker: new CacheHitsMissesTracker.NoOp()); readIndex.IndexCommitter.Init(DbRes.Db.Config.ChaserCheckpoint.Read()); ReadIndex = readIndex; diff --git a/src/EventStore.Core.Tests/Services/Storage/Transactions/when_rebuilding_index_for_partially_persisted_transaction.cs b/src/EventStore.Core.Tests/Services/Storage/Transactions/when_rebuilding_index_for_partially_persisted_transaction.cs index f68996d0ab5..2f4c06745d8 100644 --- a/src/EventStore.Core.Tests/Services/Storage/Transactions/when_rebuilding_index_for_partially_persisted_transaction.cs +++ b/src/EventStore.Core.Tests/Services/Storage/Transactions/when_rebuilding_index_for_partially_persisted_transaction.cs @@ -13,6 +13,7 @@ using NUnit.Framework; using EventStore.Core.Util; using EventStore.Core.Index.Hashes; +using EventStore.Core.Telemetry; namespace EventStore.Core.Tests.Services.Storage.Transactions { [TestFixture(typeof(LogFormat.V2), typeof(string))] @@ -60,7 +61,8 @@ public class when_rebuilding_index_for_partially_persisted_transaction : Specificatio new LRUCache.MetadataCached>("StreamMetadata", 100), true, _metastreamMaxCount, Opts.HashCollisionReadLimitDefault, Opts.SkipIndexScanOnReadsDefault, - _dbResult.Db.Config.ReplicationCheckpoint,_dbResult.Db.Config.IndexCheckpoint, new IndexStatusTracker.NoOp(), new IndexTracker.NoOp()); + _dbResult.Db.Config.ReplicationCheckpoint,_dbResult.Db.Config.IndexCheckpoint, + new IndexStatusTracker.NoOp(), + new IndexTracker.NoOp(), + new CacheHitsMissesTracker.NoOp()); readIndex.IndexCommitter.Init(_dbResult.Db.Config.WriterCheckpoint.Read()); ReadIndex = readIndex; diff --git a/src/EventStore.Core.Tests/TransactionLog/Truncation/TruncateAndReOpenDbScenario.cs b/src/EventStore.Core.Tests/TransactionLog/Truncation/TruncateAndReOpenDbScenario.cs index 27abab70080..fa176709c9d 100644 --- a/src/EventStore.Core.Tests/TransactionLog/Truncation/TruncateAndReOpenDbScenario.cs +++ b/src/EventStore.Core.Tests/TransactionLog/Truncation/TruncateAndReOpenDbScenario.cs @@ -13,6 +13,7 @@ using EventStore.Core.Util; using EventStore.Core.Index.Hashes; using EventStore.Core.Tests.Services; +using EventStore.Core.Telemetry; namespace EventStore.Core.Tests.TransactionLog.Truncation { public abstract class TruncateAndReOpenDbScenario : TruncateScenario { @@ -69,7 +70,8 @@ protected TruncateAndReOpenDbScenario(int maxEntriesInMemTable = 100, int metast replicationCheckpoint: Db.Config.ReplicationCheckpoint, indexCheckpoint: Db.Config.IndexCheckpoint, indexStatusTracker: new IndexStatusTracker.NoOp(), - indexTracker: new IndexTracker.NoOp()); + indexTracker: new IndexTracker.NoOp(), + cacheTracker: new CacheHitsMissesTracker.NoOp()); readIndex.IndexCommitter.Init(ChaserCheckpoint.Read()); ReadIndex = readIndex; } diff --git a/src/EventStore.Core.XUnit.Tests/Scavenge/Infrastructure/Scenario.cs b/src/EventStore.Core.XUnit.Tests/Scavenge/Infrastructure/Scenario.cs index 33b3d46ea76..637cf5a177b 100644 --- a/src/EventStore.Core.XUnit.Tests/Scavenge/Infrastructure/Scenario.cs +++ b/src/EventStore.Core.XUnit.Tests/Scavenge/Infrastructure/Scenario.cs @@ -12,6 +12,7 @@ using EventStore.Core.LogAbstraction; using EventStore.Core.Services.Storage.ReaderIndex; using EventStore.Core.Settings; +using EventStore.Core.Telemetry; using EventStore.Core.Tests; using EventStore.Core.Tests.Fakes; using EventStore.Core.Tests.Index.Hashers; @@ -284,7 +285,8 @@ public class Scenario : Scenario { replicationCheckpoint: dbResult.Db.Config.ReplicationCheckpoint, indexCheckpoint: dbResult.Db.Config.IndexCheckpoint, indexStatusTracker: new IndexStatusTracker.NoOp(), - indexTracker: new IndexTracker.NoOp()); + indexTracker: new IndexTracker.NoOp(), + cacheTracker: new CacheHitsMissesTracker.NoOp()); readIndex.IndexCommitter.Init(dbResult.Db.Config.WriterCheckpoint.Read()); // wait for tables to be merged. for one of the tests this takes a while diff --git a/src/EventStore.Core.XUnit.Tests/Telemetry/CacheHitsMiseesTrackerTests.cs b/src/EventStore.Core.XUnit.Tests/Telemetry/CacheHitsMiseesTrackerTests.cs new file mode 100644 index 00000000000..a174fba0c7c --- /dev/null +++ b/src/EventStore.Core.XUnit.Tests/Telemetry/CacheHitsMiseesTrackerTests.cs @@ -0,0 +1,88 @@ +using System; +using System.Diagnostics.Metrics; +using System.Linq; +using System.Runtime.CompilerServices; +using EventStore.Core.Telemetry; +using EventStore.Core.Tests; +using Xunit; + +using static EventStore.Common.Configuration.TelemetryConfiguration; + +namespace EventStore.Core.XUnit.Tests.Telemetry; + +public sealed class CacheHitsMissesTrackerTests : IDisposable { + private readonly Disposables _disposables = new(); + + public void Dispose() { + _disposables?.Dispose(); + } + + private (CacheHitsMissesTracker, TestMeterListener) GenSut( + Cache[] enabledCaches, + [CallerMemberName] string callerName = "") { + + var meter = new Meter($"{typeof(CacheHitsMissesTrackerTests)}-{callerName}").DisposeWith(_disposables); + var listener = new TestMeterListener(meter).DisposeWith(_disposables); + var metric = new CacheHitsMissesMetric(meter, "the-metric", enabledCaches); + var sut = new CacheHitsMissesTracker(metric); + return (sut, listener); + } + + [Fact] + public void observes_all_caches() { + var (sut, listener) = GenSut(new[] { Cache.Chunk, Cache.StreamInfo }); + sut.Register(Cache.Chunk, () => 1, () => 2); + sut.Register(Cache.StreamInfo, () => 3, () => 4); + AssertMeasurements(listener, + AssertMeasurement("chunk", "hits", 1), + AssertMeasurement("chunk", "misses", 2), + AssertMeasurement("stream-info", "hits", 3), + AssertMeasurement("stream-info", "misses", 4)); + } + + [Fact] + public void ignores_disabled_cache() { + var (sut, listener) = GenSut(new[] { Cache.StreamInfo }); + sut.Register(Cache.Chunk, () => 1, () => 2); + sut.Register(Cache.StreamInfo, () => 3, () => 4); + AssertMeasurements(listener, + AssertMeasurement("stream-info", "hits", 3), + AssertMeasurement("stream-info", "misses", 4)); + } + + [Fact] + public void ignores_unregistered_cache() { + var (sut, listener) = GenSut(new[] { Cache.Chunk, Cache.StreamInfo }); + sut.Register(Cache.Chunk, () => 1, () => 2); + AssertMeasurements(listener, + AssertMeasurement("chunk", "hits", 1), + AssertMeasurement("chunk", "misses", 2)); + } + + static Action.TestMeasurement> AssertMeasurement( + string cacheName, + string kind, + long expectedValue) => + + actualMeasurement => { + Assert.Equal(expectedValue, actualMeasurement.Value); + Assert.Collection( + actualMeasurement.Tags.ToArray(), + tag => { + Assert.Equal("cache", tag.Key); + Assert.Equal(cacheName, tag.Value); + }, + tag => { + Assert.Equal("kind", tag.Key); + Assert.Equal(kind, tag.Value); + }); + }; + + static void AssertMeasurements( + TestMeterListener listener, + params Action.TestMeasurement>[] actions) { + + listener.Observe(); + Assert.Collection(listener.RetrieveMeasurements("the-metric"), actions); + } +} diff --git a/src/EventStore.Core/ClusterVNode.cs b/src/EventStore.Core/ClusterVNode.cs index 7aa31f4ae72..ba9cb13f742 100644 --- a/src/EventStore.Core/ClusterVNode.cs +++ b/src/EventStore.Core/ClusterVNode.cs @@ -671,7 +671,8 @@ public class ClusterVNode : Db.Config.ReplicationCheckpoint.AsReadOnly(), Db.Config.IndexCheckpoint, trackers.IndexStatusTracker, - trackers.IndexTracker); + trackers.IndexTracker, + trackers.CacheHitsMissesTracker); _readIndex = readIndex; var writer = new TFChunkWriter(Db); diff --git a/src/EventStore.Core/MetricsBootstrapper.cs b/src/EventStore.Core/MetricsBootstrapper.cs index 866dd7ae858..f0a10e03777 100644 --- a/src/EventStore.Core/MetricsBootstrapper.cs +++ b/src/EventStore.Core/MetricsBootstrapper.cs @@ -27,6 +27,7 @@ public class Trackers { public IIndexTracker IndexTracker { get; set; } = new IndexTracker.NoOp(); public IMaxTracker WriterFlushSizeTracker { get; set; } = new MaxTracker.NoOp(); public IDurationMaxTracker WriterFlushDurationTracker { get; set; } = new DurationMaxTracker.NoOp(); + public ICacheHitsMissesTracker CacheHitsMissesTracker { get; set; } = new CacheHitsMissesTracker.NoOp(); } public class GrpcTrackers { @@ -82,6 +83,13 @@ public static class MetricsBootstrapper { _ = new IncomingGrpcCallsMetric(coreMeter, "eventstore-incoming-grpc-calls", enabledCalls); } + // cache hits/misses + var enabledCacheHitsMisses = conf.CacheHitsMisses.Where(kvp => kvp.Value).Select(kvp => kvp.Key).ToArray(); + if (enabledCacheHitsMisses.Length > 0) { + var metric = new CacheHitsMissesMetric(coreMeter, "eventstore-cache-hits-misses", enabledCacheHitsMisses); + trackers.CacheHitsMissesTracker = new CacheHitsMissesTracker(metric); + } + // events if (conf.Events.TryGetValue(Conf.EventTracker.Read, out var readEnabled) && readEnabled) { var readTag = new KeyValuePair("activity", "read"); diff --git a/src/EventStore.Core/Services/Storage/ReaderIndex/ReadIndex.cs b/src/EventStore.Core/Services/Storage/ReaderIndex/ReadIndex.cs index 8a42e4d9fc6..9eab2cefee8 100644 --- a/src/EventStore.Core/Services/Storage/ReaderIndex/ReadIndex.cs +++ b/src/EventStore.Core/Services/Storage/ReaderIndex/ReadIndex.cs @@ -9,10 +9,12 @@ using EventStore.Core.Index; using EventStore.Core.LogAbstraction; using EventStore.Core.Messages; +using EventStore.Core.Telemetry; using EventStore.Core.TransactionLog; using EventStore.Core.TransactionLog.Checkpoint; using EventStore.Core.TransactionLog.Chunks; using EventStore.Core.Util; +using static EventStore.Common.Configuration.TelemetryConfiguration; namespace EventStore.Core.Services.Storage.ReaderIndex { public sealed class ReadIndex : IDisposable, IReadIndex { @@ -56,7 +58,8 @@ public sealed class ReadIndex : IDisposable, IReadIndex { IReadOnlyCheckpoint replicationCheckpoint, ICheckpoint indexCheckpoint, IIndexStatusTracker indexStatusTracker, - IIndexTracker indexTracker) { + IIndexTracker indexTracker, + ICacheHitsMissesTracker cacheTracker) { Ensure.NotNull(bus, "bus"); Ensure.NotNull(readerPool, "readerPool"); @@ -91,6 +94,8 @@ public sealed class ReadIndex : IDisposable, IReadIndex { _streamNames, eventTypeIndex, eventTypeNames, systemStreams, streamExistenceFilter, streamExistenceFilterInitializer, indexCheckpoint, indexStatusTracker, indexTracker, additionalCommitChecks); _allReader = new AllReader(indexBackend, _indexCommitter, _streamNames, eventTypeNames); + + RegisterHitsMisses(cacheTracker); } IndexReadEventResult IReadIndex.ReadEvent(string streamName, TStreamId streamId, long eventNumber) { @@ -181,6 +186,18 @@ public sealed class ReadIndex : IDisposable, IReadIndex { return _indexReader.GetEffectiveAcl(streamId); } + void RegisterHitsMisses(ICacheHitsMissesTracker tracker) { + tracker.Register( + Cache.Chunk, + () => Interlocked.Read(ref TFChunkReader.CachedReads), + () => Interlocked.Read(ref TFChunkReader.NotCachedReads)); + + tracker.Register( + Cache.StreamInfo, + () => _indexReader.CachedStreamInfo, + () => _indexReader.NotCachedStreamInfo); + } + ReadIndexStats IReadIndex.GetStatistics() { return new ReadIndexStats(Interlocked.Read(ref TFChunkReader.CachedReads), Interlocked.Read(ref TFChunkReader.NotCachedReads), diff --git a/src/EventStore.Core/Telemetry/CacheHitsMissesMetric.cs b/src/EventStore.Core/Telemetry/CacheHitsMissesMetric.cs new file mode 100644 index 00000000000..26394db79fd --- /dev/null +++ b/src/EventStore.Core/Telemetry/CacheHitsMissesMetric.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Linq; +using static EventStore.Common.Configuration.TelemetryConfiguration; + +namespace EventStore.Core.Telemetry; + +public class CacheHitsMissesMetric { + private readonly Cache[] _enabledCaches; + private readonly List> _funcs = new(); + private readonly List[]> _tagss = new(); + private readonly object _lock = new(); + + public CacheHitsMissesMetric(Meter meter, string name, Cache[] enabledCaches) { + _enabledCaches = enabledCaches; + meter.CreateObservableCounter(name, Observe); + } + + public void Register(Cache cache, Func getHits, Func getMisses) { + Register(cache, "hits", getHits); + Register(cache, "misses", getMisses); + } + + private void Register(Cache cache, string kind, Func func) { + if (!_enabledCaches.Contains(cache)) + return; + + lock (_lock) { + _funcs.Add(func); + _tagss.Add(new KeyValuePair[] { + new("cache", KebabCase(cache)), + new("kind", kind), + }); + } + } + + private static string KebabCase(Cache cache) => cache switch { + Cache.StreamInfo => "stream-info", + Cache.Chunk => "chunk", + _ => throw new ArgumentOutOfRangeException(nameof(cache), cache, null), + }; + + private IEnumerable> Observe() { + lock (_lock) { + for (var i = 0; i < _funcs.Count; i++) { + yield return new(_funcs[i](), _tagss[i]); + } + } + } +} diff --git a/src/EventStore.Core/Telemetry/CacheHitsMissesTracker.cs b/src/EventStore.Core/Telemetry/CacheHitsMissesTracker.cs new file mode 100644 index 00000000000..556ccacb04f --- /dev/null +++ b/src/EventStore.Core/Telemetry/CacheHitsMissesTracker.cs @@ -0,0 +1,24 @@ +using System; +using static EventStore.Common.Configuration.TelemetryConfiguration; + +namespace EventStore.Core.Telemetry; + +public interface ICacheHitsMissesTracker { + void Register(Cache cache, Func getHits, Func getMisses); +} + +public class CacheHitsMissesTracker : ICacheHitsMissesTracker { + private readonly CacheHitsMissesMetric _metric; + + public CacheHitsMissesTracker(CacheHitsMissesMetric metric) { + _metric = metric; + } + + public void Register(Cache cache, Func getHits, Func getMisses) => + _metric.Register(cache, getHits, getMisses); + + public class NoOp : ICacheHitsMissesTracker { + public void Register(Cache cache, Func getHits, Func getMisses) { + } + } +}