Upgrade Kute (#6366)

* Introduce filter limits

- Filters can include optional limits

* Report progress at the start

- Fixes progress not updating

* Initial response validator

- Verifies that the response does not contain an 'error' field

* Include more examples of filters and limits

* Perform validation outside time measurement

* Add 'TracerValidator'

- Responses can be stored into a new file

* Document 'TracerValidator'

- Disabled by default
- Response tracing is ignored in dry mode

* Make config help text more consistent

* Capitalize

* Change name of tracing to verbose

* Update README.md

* Use `-r|--responses` to store responses

* Invert condition

- Append if file exists
- Create one if not

* Make 'IJsonRpcValidator' take also request into consideration

* Validate 'NewPayload' responses using custom strategy

- 'TracerValidator' is no longer a decorator

* Separate 'ResponseTracer' from Validator

- Tracer should be independent
- Add File and Null tracers
- Simplify services setup

* Validate and trace batch responses

* Add TickSucceeded

* Add temp sampleSize

* Tick succeeded on batches too

* Use 'CompleteReservoir' for sampling

- Removes need for hack/magic numbers

* Remove whitespace

* Remove `IAuth` from `NullJsonRpcSubmitter`

---------

Co-authored-by: Kamil Chodoła <43241881+kamilchodola@users.noreply.github.com>
Co-authored-by: Kamil Chodoła <kamil@nethermind.io>

tools/Nethermind.Tools.Kute/Application.cs

@@ -1,12 +1,15 @@
 // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
+using System.Text.Json;
 using Nethermind.Tools.Kute.Extensions;
 using Nethermind.Tools.Kute.MessageProvider;
 using Nethermind.Tools.Kute.JsonRpcMethodFilter;
 using Nethermind.Tools.Kute.JsonRpcSubmitter;
+using Nethermind.Tools.Kute.JsonRpcValidator;
 using Nethermind.Tools.Kute.MetricsConsumer;
 using Nethermind.Tools.Kute.ProgressReporter;
+using Nethermind.Tools.Kute.ResponseTracer;
 
 namespace Nethermind.Tools.Kute;
 
@@ -16,20 +19,26 @@ class Application
 
     private readonly IMessageProvider<JsonRpc?> _msgProvider;
     private readonly IJsonRpcSubmitter _submitter;
+    private readonly IJsonRpcValidator _validator;
+    private readonly IResponseTracer _responseTracer;
     private readonly IProgressReporter _progressReporter;
     private readonly IMetricsConsumer _metricsConsumer;
     private readonly IJsonRpcMethodFilter _methodFilter;
 
     public Application(
         IMessageProvider<JsonRpc?> msgProvider,
         IJsonRpcSubmitter submitter,
+        IJsonRpcValidator validator,
+        IResponseTracer responseTracer,
         IProgressReporter progressReporter,
         IMetricsConsumer metricsConsumer,
         IJsonRpcMethodFilter methodFilter
     )
     {
         _msgProvider = msgProvider;
         _submitter = submitter;
+        _validator = validator;
+        _responseTracer = responseTracer;
         _progressReporter = progressReporter;
         _metricsConsumer = metricsConsumer;
         _methodFilter = methodFilter;
@@ -43,24 +52,41 @@ public async Task Run()
         {
             await foreach (var (jsonRpc, n) in _msgProvider.Messages.Indexed(startingFrom: 1))
             {
+                _progressReporter.ReportProgress(n);
+
                 _metrics.TickMessages();
 
                 switch (jsonRpc)
                 {
                     case null:
+                    {
                         _metrics.TickFailed();
 
                         break;
-
+                    }
                     case JsonRpc.BatchJsonRpc batch:
+                    {
+                        JsonDocument? result;
                         using (_metrics.TimeBatch())
                         {
-                            await _submitter.Submit(batch);
+                            result = await _submitter.Submit(batch);
                         }
 
-                        break;
+                        if (_validator.IsInvalid(batch, result))
+                        {
+                            _metrics.TickFailed();
+                        }
+                        else
+                        {
+                            _metrics.TickSucceeded();
+                        }
 
+                        await _responseTracer.TraceResponse(result);
+
+                        break;
+                    }
                     case JsonRpc.SingleJsonRpc single:
+                    {
                         if (single.IsResponse)
                         {
                             _metrics.TickResponses();
@@ -79,18 +105,30 @@ await foreach (var (jsonRpc, n) in _msgProvider.Messages.Indexed(startingFrom: 1
                             continue;
                         }
 
+                        JsonDocument? result;
                         using (_metrics.TimeMethod(single.MethodName))
                         {
-                            await _submitter.Submit(single);
+                            result = await _submitter.Submit(single);
                         }
 
-                        break;
+                        if (_validator.IsInvalid(single, result))
+                        {
+                            _metrics.TickFailed();
+                        }
+                        else
+                        {
+                            _metrics.TickSucceeded();
+                        }
+
+                        await _responseTracer.TraceResponse(result);
 
+                        break;
+                    }
                     default:
+                    {
                         throw new ArgumentOutOfRangeException(nameof(jsonRpc));
+                    }
                 }
-
-                _progressReporter.ReportProgress(n);
             }
         }
 

tools/Nethermind.Tools.Kute/Config.cs

@@ -21,15 +21,15 @@ public class Config
         longName: "address",
         Required = false,
         Default = "http://localhost:8551",
-        HelpText = "Address where to send JSON RPC calls"
+        HelpText = "Address where to send JSON RPC requests"
     )]
     public string HostAddress { get; }
 
     [Option(
         shortName: 's',
         longName: "secret",
         Required = true,
-        HelpText = "Path to file with hex encoded secret for JWT authentication"
+        HelpText = "Path to File with hex encoded secret for JWT authentication"
     )]
     public string JwtSecretFilePath { get; }
 
@@ -75,10 +75,19 @@ public class Config
         Separator = ',',
         Required = false,
         Default = new string[] { },
-        HelpText = "A comma separated List of regexes of methods to be executed"
+        HelpText = "A comma separated List of regexes of methods to be executed with optional limits"
     )]
     public IEnumerable<string> MethodFilters { get; }
 
+    [Option(
+        shortName: 'r',
+        longName: "responses",
+        Required = false,
+        Default = null,
+        HelpText = "Path to File to store JSON-RPC responses"
+    )]
+    public string? ResponsesTraceFile { get; }
+
     public Config(
         string messagesFilePath,
         string hostAddress,
@@ -87,7 +96,8 @@ public class Config
         bool dryRun,
         bool showProgress,
         MetricsOutputFormatter metricsOutputFormatter,
-        IEnumerable<string> methodFilters
+        IEnumerable<string> methodFilters,
+        string? responsesTraceFile
     )
     {
         MessagesFilePath = messagesFilePath;
@@ -98,6 +108,6 @@ IEnumerable<string> methodFilters
         ShowProgress = showProgress;
         MetricsOutputFormatter = metricsOutputFormatter;
         MethodFilters = methodFilters;
-        ShowProgress = showProgress;
+        ResponsesTraceFile = responsesTraceFile;
     }
 }

tools/Nethermind.Tools.Kute/Extensions/CompleteReservoir.cs

@@ -0,0 +1,45 @@
+// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using App.Metrics;
+using App.Metrics.ReservoirSampling;
+using App.Metrics.ReservoirSampling.Uniform;
+
+namespace Nethermind.Tools.Kute.Extensions;
+
+public class CompleteReservoir : IReservoir
+{
+    private const int DefaultSize = 10_000;
+
+    private readonly List<UserValueWrapper> _values;
+
+    public CompleteReservoir() : this(DefaultSize) { }
+
+    public CompleteReservoir(int size)
+    {
+        _values = new List<UserValueWrapper>(size);
+    }
+
+    public IReservoirSnapshot GetSnapshot(bool resetReservoir)
+    {
+        long count = _values.Count;
+        double sum = _values.Sum(v => v.Value);
+        IEnumerable<long> values = _values.Select(v => v.Value);
+
+        if (resetReservoir)
+        {
+            _values.Clear();
+        }
+
+        return new UniformSnapshot(count, sum, values);
+    }
+
+    public IReservoirSnapshot GetSnapshot() => GetSnapshot(false);
+
+    public void Reset() => _values.Clear();
+
+    public void Update(long value, string userValue) => _values.Add(new UserValueWrapper(value, userValue));
+
+    public void Update(long value) => _values.Add(new UserValueWrapper(value));
+
+}

tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/LimitedJsonRpcMethodFilter.cs

@@ -0,0 +1,33 @@
+// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+namespace Nethermind.Tools.Kute.JsonRpcMethodFilter;
+
+class LimitedJsonRpcMethodFilter : IJsonRpcMethodFilter
+{
+    private readonly IJsonRpcMethodFilter _filter;
+
+    private int _usagesLeft;
+
+    public LimitedJsonRpcMethodFilter(IJsonRpcMethodFilter filter, int limit)
+    {
+        _filter = filter;
+        _usagesLeft = limit;
+    }
+
+    public bool ShouldSubmit(string methodName)
+    {
+        if (_filter.ShouldSubmit(methodName))
+        {
+            if (_usagesLeft == 0)
+            {
+                return false;
+            }
+
+            _usagesLeft--;
+            return true;
+        }
+
+        return false;
+    }
+}

tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/PatternJsonRpcMethodFilter.cs

@@ -1,18 +1,25 @@
 // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
-using System.Text.RegularExpressions;
-
 namespace Nethermind.Tools.Kute.JsonRpcMethodFilter;
 
-class PatternJsonRpcMethodFilter : IJsonRpcMethodFilter
+public class PatternJsonRpcMethodFilter : IJsonRpcMethodFilter
 {
-    private readonly Regex _pattern;
+    private const char PatternSeparator = '=';
 
+    private readonly IJsonRpcMethodFilter _filter;
     public PatternJsonRpcMethodFilter(string pattern)
     {
-        _pattern = new Regex(pattern);
+        var splitted = pattern.Split(PatternSeparator);
+
+        var regex = new RegexJsonRpcMethodFilter(splitted[0]);
+        _filter = splitted.Length switch
+        {
+            1 => regex,
+            2 => new LimitedJsonRpcMethodFilter(regex, int.Parse(splitted[1])),
+            _ => throw new ArgumentException($"Unexpected pattern: {pattern}"),
+        };
     }
 
-    public bool ShouldSubmit(string methodName) => _pattern.IsMatch(methodName);
+    public bool ShouldSubmit(string methodName) => _filter.ShouldSubmit(methodName);
 }

tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/RegexJsonRpcMethodFilter.cs

@@ -0,0 +1,18 @@
+// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System.Text.RegularExpressions;
+
+namespace Nethermind.Tools.Kute.JsonRpcMethodFilter;
+
+class RegexJsonRpcMethodFilter : IJsonRpcMethodFilter
+{
+    private readonly Regex _pattern;
+
+    public RegexJsonRpcMethodFilter(string pattern)
+    {
+        _pattern = new Regex(pattern);
+    }
+
+    public bool ShouldSubmit(string methodName) => _pattern.IsMatch(methodName);
+}

tools/Nethermind.Tools.Kute/JsonRpcSubmitter/HttpJsonRpcSubmitter.cs

@@ -5,6 +5,7 @@
 using System.Net.Http.Headers;
 using System.Net.Mime;
 using System.Text;
+using System.Text.Json;
 using Nethermind.Tools.Kute.Auth;
 
 namespace Nethermind.Tools.Kute.JsonRpcSubmitter;
@@ -22,7 +23,7 @@ public HttpJsonRpcSubmitter(HttpClient httpClient, IAuth auth, string hostAddres
         _uri = new Uri(hostAddress);
     }
 
-    public async Task Submit(JsonRpc rpc)
+    public async Task<JsonDocument?> Submit(JsonRpc rpc)
     {
         var request = new HttpRequestMessage(HttpMethod.Post, _uri)
         {
@@ -32,7 +33,9 @@ public async Task Submit(JsonRpc rpc)
         var response = await _httpClient.SendAsync(request);
         if (response.StatusCode != HttpStatusCode.OK)
         {
-            throw new HttpRequestException($"Expected {HttpStatusCode.OK}, got {response.StatusCode}");
+            return null;
         }
+        var content = await response.Content.ReadAsStreamAsync();
+        return await JsonSerializer.DeserializeAsync<JsonDocument>(content);
     }
 }

tools/Nethermind.Tools.Kute/JsonRpcSubmitter/IJsonRpcSubmitter.cs

@@ -1,9 +1,11 @@
 // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
+using System.Text.Json;
+
 namespace Nethermind.Tools.Kute.JsonRpcSubmitter;
 
 interface IJsonRpcSubmitter
 {
-    Task Submit(JsonRpc rpc);
+    Task<JsonDocument?> Submit(JsonRpc rpc);
 }

tools/Nethermind.Tools.Kute/JsonRpcSubmitter/NullJsonRpcSubmitter.cs

@@ -1,22 +1,12 @@
 // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
-using Nethermind.Tools.Kute.Auth;
+using System.Text.Json;
 
 namespace Nethermind.Tools.Kute.JsonRpcSubmitter;
 
 class NullJsonRpcSubmitter : IJsonRpcSubmitter
 {
-    private readonly IAuth _auth;
 
-    public NullJsonRpcSubmitter(IAuth auth)
-    {
-        _auth = auth;
-    }
-
-    public Task Submit(JsonRpc rpc)
-    {
-        _ = _auth.AuthToken;
-        return Task.CompletedTask;
-    }
+    public Task<JsonDocument?> Submit(JsonRpc rpc) => Task.FromResult<JsonDocument?>(null);
 }

tools/Nethermind.Tools.Kute/JsonRpcValidator/ComposedJsonRpcValidator.cs

@@ -0,0 +1,18 @@
+// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System.Text.Json;
+
+namespace Nethermind.Tools.Kute.JsonRpcValidator;
+
+public class ComposedJsonRpcValidator : IJsonRpcValidator
+{
+    private readonly IEnumerable<IJsonRpcValidator> _validators;
+
+    public ComposedJsonRpcValidator(IEnumerable<IJsonRpcValidator> validators)
+    {
+        _validators = validators;
+    }
+
+    public bool IsValid(JsonRpc request, JsonDocument? document) => _validators.All(validator => validator.IsValid(request, document));
+}