Skip to content

Commit

Permalink
Add tests, rollback pluginify FeeProvider
Browse files Browse the repository at this point in the history
  • Loading branch information
@NicolasDorier
NicolasDorier committed Jan 18, 2024
1 parent a7a3d03 commit c8e6f48
Show file tree
Hide file tree
Showing 5 changed files with 95 additions and 106 deletions.
33 changes: 33 additions & 0 deletions BTCPayServer.Tests/FastTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
using BTCPayServer.Rating;
using BTCPayServer.Services;
using BTCPayServer.Services.Apps;
using BTCPayServer.Services.Fees;
using BTCPayServer.Services.Invoices;
using BTCPayServer.Services.Labels;
using BTCPayServer.Services.Rates;
Expand Down Expand Up @@ -159,6 +160,38 @@ public void CanParseDecimals()
Assert.Equal("Test", data.FromAsset);
}

[Fact]
public void CanInterpolateOrBound()
{
var testData = new ((int Blocks, decimal Fee)[] Data, int Target, decimal Expected) []
{
([(0, 0m), (10, 100m)], 5, 50m),
([(50, 0m), (100, 100m)], 5, 0.0m),
([(50, 0m), (100, 100m)], 101, 100.0m),
([(50, 100m), (50, 100m)], 101, 100.0m),
([(50, 0m), (100, 100m)], 75, 50m),
([(0, 0m), (50, 50m), (100, 100m)], 75, 75m),
([(0, 0m), (500, 50m), (1000, 100m)], 750, 75m),
([(0, 0m), (500, 50m), (1000, 100m)], 100, 10m),
([(0, 0m), (100, 100m)], 80, 80m),
([(0, 0m), (100, 100m)], 25, 25m),
};
foreach (var t in testData)
{
var actual = MempoolSpaceFeeProvider.InterpolateOrBound(t.Data.Select(t => KeyValuePair.Create(t.Blocks, new FeeRate(t.Fee))).ToList(), t.Target);
Assert.Equal(new FeeRate(t.Expected), actual);
}
}
[Fact]
public void CanRandomizeByPercentage()
{
var generated = Enumerable.Range(0, 1000).Select(_ => MempoolSpaceFeeProvider.RandomizeByPercentage(100.0m, 10.0m)).ToArray();
Assert.Empty(generated.Where(g => g < 90m));
Assert.Empty(generated.Where(g => g > 110m));
Assert.NotEmpty(generated.Where(g => g < 91m));
Assert.NotEmpty(generated.Where(g => g > 109m));
}

private void CanParseDecimalsCore(string str, decimal expected)
{
var d = JsonConvert.DeserializeObject<LedgerEntryData>(str);
Expand Down
6 changes: 3 additions & 3 deletions BTCPayServer/Hosting/BTCPayServerServices.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,7 @@ public static IServiceCollection RegisterJsonConverter(this IServiceCollection s
services.AddSingleton<IJsonConverterRegistration, JsonConverterRegistration>((s) => new JsonConverterRegistration(create));
return services;
}
public static IServiceCollection AddBTCPayServer(this IServiceCollection services, IConfiguration configuration,
Logs logs, ServiceProvider bootstrapServiceProvider)
public static IServiceCollection AddBTCPayServer(this IServiceCollection services, IConfiguration configuration, Logs logs)
{
services.AddSingleton<MvcNewtonsoftJsonOptions>(o => o.GetRequiredService<IOptions<MvcNewtonsoftJsonOptions>>().Value);
services.AddSingleton<JsonSerializerSettings>(o => o.GetRequiredService<IOptions<MvcNewtonsoftJsonOptions>>().Value.SerializerSettings);
Expand Down Expand Up @@ -369,7 +368,8 @@ public static IServiceCollection RegisterJsonConverter(this IServiceCollection s
services.TryAddSingleton<WalletReceiveService>();
services.AddSingleton<IHostedService>(provider => provider.GetService<WalletReceiveService>());
services.TryAddSingleton<CurrencyNameTable>(CurrencyNameTable.Instance);
services.AddFeeProviders(bootstrapServiceProvider);
services.TryAddSingleton<IFeeProviderFactory, FeeProviderFactory>();

services.Configure<MvcOptions>((o) =>
{
o.ModelMetadataDetailsProviders.Add(new SuppressChildValidationMetadataProvider(typeof(WalletId)));
Expand Down
2 changes: 1 addition & 1 deletion BTCPayServer/Hosting/Startup.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ public void ConfigureServices(IServiceCollection services)
opts.ValidationInterval = TimeSpan.FromMinutes(5.0);
});

services.AddBTCPayServer(Configuration, Logs, bootstrapServiceProvider);
services.AddBTCPayServer(Configuration, Logs);
services.AddProviderStorage();
services.AddSession();
services.AddSignalR();
Expand Down
72 changes: 22 additions & 50 deletions BTCPayServer/Services/Fees/FeeProviderFactory.cs
View file
Original file line number Diff line number Diff line change
@@ -1,73 +1,45 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using Microsoft.AspNetCore.HttpOverrides;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using NBitcoin;
using NBXplorer;

namespace BTCPayServer.Services.Fees;

public static class FeeProviderExtensions
public class FeeProviderFactory : IFeeProviderFactory
{
public static IServiceCollection AddFeeProviders(this IServiceCollection services, IServiceProvider bootstrapServiceProvider)
public FeeProviderFactory(
BTCPayServerEnvironment Environment,
ExplorerClientProvider ExplorerClients,
IHttpClientFactory HttpClientFactory,
IMemoryCache MemoryCache)
{
var networkProvider = bootstrapServiceProvider.GetRequiredService<NBXplorerNetworkProvider>();
_FeeProviders = new ();

foreach (var network in networkProvider.GetAll())
// TODO: Pluginify this
foreach ((var network, var client) in ExplorerClients.GetAll())
{

if (network.CryptoCode == "BTC" && network.NBitcoinNetwork.ChainName != ChainName.Regtest)
List<IFeeProvider> providers = new List<IFeeProvider>();
if (network.IsBTC && Environment.NetworkType != ChainName.Regtest)
{

services.AddKeyedSingleton<IFeeProvider>(network.CryptoCode, (provider, o) => new MempoolSpaceFeeProvider(
provider.GetRequiredService<IMemoryCache>(),
providers.Add(new MempoolSpaceFeeProvider(
MemoryCache,
$"MempoolSpaceFeeProvider-{network.CryptoCode}",
provider.GetRequiredService<IHttpClientFactory>(),
network is { } n &&
HttpClientFactory,
network is BTCPayNetwork n &&
n.NBitcoinNetwork.ChainName == ChainName.Testnet));
}

services.AddKeyedSingleton<IFeeProvider>(network.CryptoCode, (provider, o) => new NBXplorerFeeProvider(
provider.GetRequiredService<ExplorerClientProvider>().GetExplorerClient(network.CryptoCode)));

services.AddKeyedSingleton<IFeeProvider>(network.CryptoCode,
(provider, o) => new StaticFeeProvider(new FeeRate(100L, 1)));
providers.Add(new NBXplorerFeeProvider(client));
providers.Add(new StaticFeeProvider(new FeeRate(100L, 1)));
var fallback = new FallbackFeeProvider(providers.ToArray());
_FeeProviders.Add(network, fallback);
}
services.AddSingleton<IFeeProviderFactory, FeeProviderFactory>();
return services;
}

}

public class FeeProviderFactory : IFeeProviderFactory
{
private readonly IEnumerable<IFeeProvider> _feeProviders;
private readonly IServiceProvider _serviceProvider;

public FeeProviderFactory(
ExplorerClientProvider explorerClients,
IHttpClientFactory httpClientFactory,
IMemoryCache memoryCache,
IEnumerable<IFeeProvider> feeProviders,
IServiceProvider serviceProvider)
{
_feeProviders = feeProviders;
_serviceProvider = serviceProvider;
}

private readonly ConcurrentDictionary<BTCPayNetworkBase, FallbackFeeProvider> _cached = new();
private readonly Dictionary<BTCPayNetworkBase, IFeeProvider> _FeeProviders;
public IFeeProvider CreateFeeProvider(BTCPayNetworkBase network)
{
return _cached.GetOrAdd(network, n =>
{
var feeProviders = _serviceProvider.GetKeyedServices<IFeeProvider>(network.CryptoCode).Concat(_feeProviders).ToArray();
if (!feeProviders.Any())
throw new NotSupportedException($"No fee provider for this network ({network.CryptoCode})");
return new FallbackFeeProvider(feeProviders.ToArray());
});
return _FeeProviders.TryGetValue(network, out var prov) ? prov : throw new NotSupportedException($"No fee provider for this network ({network.CryptoCode})");
}
}
88 changes: 36 additions & 52 deletions BTCPayServer/Services/Fees/MempoolSpaceFeeProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,16 @@
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using AngleSharp.Dom;
using ExchangeSharp;
using Microsoft.Extensions.Caching.Memory;
using NBitcoin;
using Org.BouncyCastle.Asn1.X509;
using YamlDotNet.Core.Tokens;

namespace BTCPayServer.Services.Fees;

Expand All @@ -27,41 +32,35 @@ public async Task<FeeRate> GetFeeRateAsync(int blockTarget = 20)
{
var result = await GetFeeRatesAsync();

return result.TryGetValue(blockTarget, out var feeRate)
? feeRate
: InterpolateOrBound(result, blockTarget);
return InterpolateOrBound(result, blockTarget);

}

static FeeRate InterpolateOrBound(Dictionary<int, FeeRate> dictionary, int target)
internal static FeeRate InterpolateOrBound(List<KeyValuePair<int, FeeRate>> ordered, int target)
{
// Find the keys closest to the target for interpolation
int? k1 = null;
int? k2 = null;

foreach (int k in dictionary.Keys.Order())
(int lb, int hb) = (ordered[0].Key, ordered[^1].Key);
(decimal lbv, decimal hbv) = (ordered[0].Value.SatoshiPerByte, ordered[^1].Value.SatoshiPerByte);
target = Math.Clamp(target, lb, hb);
for (int i = 0; i < ordered.Count; i++)
{
k1 = k1 is null ? k : k2;
k2 = k;
if(target < k)
if (ordered[i].Key > lb && ordered[i].Key <= target)
{
break;
lb = ordered[i].Key;
lbv = ordered[i].Value.SatoshiPerByte;
}
if (ordered[i].Key < hb && ordered[i].Key >= target)
{
hb = ordered[i].Key;
hbv = ordered[i].Value.SatoshiPerByte;
}
}

if (k1 is null)
{
throw new InvalidOperationException("No fee rate available");
}

var v1 = dictionary[k1!.Value].SatoshiPerByte;
var v2 = dictionary[k2!.Value].SatoshiPerByte;

// Linear interpolation formula
return new FeeRate((decimal) (v1 + (v2 - v1) / (k1 - k2) * (target - k1))!);
if (hb == lb)
return new FeeRate(lbv);
var a = (decimal)(target - lb) / (decimal)(hb - lb);
return new FeeRate((1 - a) * lbv + a * hbv);
}

public async Task<Dictionary<int, FeeRate>> GetFeeRatesAsync()
public async Task<List<KeyValuePair<int, FeeRate>>> GetFeeRatesAsync()
{
try
{
Expand All @@ -71,17 +70,18 @@ static FeeRate InterpolateOrBound(Dictionary<int, FeeRate> dictionary, int targe
return await GetFeeRatesCore();
}))!;
}
catch (Exception e)
catch (Exception)
{
memoryCache.Remove(cacheKey);
throw;
}
}

protected virtual async Task<Dictionary<int, FeeRate>> GetFeeRatesCore()
protected virtual async Task<List<KeyValuePair<int,FeeRate>>> GetFeeRatesCore()
{
var client = httpClientFactory.CreateClient(nameof(MempoolSpaceFeeProvider));
using var result = await client.GetAsync(ExplorerLink, new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10));
using var result = await client.GetAsync(ExplorerLink, cts.Token);
result.EnsureSuccessStatusCode();
var recommendedFees = await result.Content.ReadAsAsync<Dictionary<string, decimal>>();
var feesByBlockTarget = new Dictionary<int, FeeRate>();
Expand All @@ -99,39 +99,23 @@ static FeeRate InterpolateOrBound(Dictionary<int, FeeRate> dictionary, int targe
};
feesByBlockTarget.TryAdd(target, new FeeRate(value));
}
// order feesByBlockTarget and then randomize them by 10%, but never allow the numbers to go below the previous one or higher than the next
var ordered = feesByBlockTarget.OrderByDescending(kv => kv.Key).ToList();
for (var i = 0; i < ordered.Count; i++)
{
(int key, FeeRate value) = ordered[i];
if (i == 0)
{
feesByBlockTarget[key] = new FeeRate(RandomizeByPercentage(value.SatoshiPerByte, 10));
}
else
{
var previous = feesByBlockTarget[ordered[i - 1].Key];
var newValue = RandomizeByPercentage(value.SatoshiPerByte, 10);
if (newValue > previous.SatoshiPerByte)
{
newValue = previous.SatoshiPerByte;
}
feesByBlockTarget[key] = new FeeRate(newValue);
}
// Randomize a bit
feesByBlockTarget[ordered[i].Key] = new FeeRate(RandomizeByPercentage(ordered[i].Value.SatoshiPerByte, 10m));
if (i > 0) // Make sure feerate always increase
feesByBlockTarget[ordered[i].Key] = new FeeRate(Math.Max(ordered[i - 1].Value.SatoshiPerByte, ordered[i].Value.SatoshiPerByte));
}

return feesByBlockTarget;
return ordered;
}

static decimal RandomizeByPercentage(decimal value, int percentage)
internal static decimal RandomizeByPercentage(decimal value, decimal percentage)
{
if (value is 1)
{
return 1;
}
decimal range = value * percentage / 100m;
var res = value + range * (Random.Shared.NextDouble() < 0.5 ? -1 : 1);

decimal range = (value * percentage) / 100m;
var res = value + (range * 2.0m) * ((decimal)(Random.Shared.NextDouble() - 0.5));
return res switch
{
< 1m => 1m,
Expand Down

0 comments on commit c8e6f48

Please sign in to comment.