Skip to content

Commit

Permalink
Refactor fee provider (#5643)
Browse files Browse the repository at this point in the history 
* Refactor fee provider

The fee provider ended up glued with a hardcoded factory. This PR:
* removes this glue and uses the DI to register fee provider for a network. (allows plugins to add their own fee providers, for any network
* Add a 10 second timeout to mempoolspace fee fetching as they are slow at times

* use linear interpolation for mempool space fee estimation

* fix upper bound

* Add tests, rollback pluginify FeeProvider

---------

Co-authored-by: nicolas.dorier <nicolas.dorier@gmail.com>
  • Loading branch information
@Kukks @NicolasDorier
Kukks and NicolasDorier committed Jan 18, 2024
1 parent cd8ef0c commit 3eec9cb
Show file tree
Hide file tree
Showing 3 changed files with 146 additions and 39 deletions.
33 changes: 33 additions & 0 deletions BTCPayServer.Tests/FastTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
using BTCPayServer.Rating;
using BTCPayServer.Services;
using BTCPayServer.Services.Apps;
using BTCPayServer.Services.Fees;
using BTCPayServer.Services.Invoices;
using BTCPayServer.Services.Labels;
using BTCPayServer.Services.Rates;
Expand Down Expand Up @@ -159,6 +160,38 @@ public void CanParseDecimals()
Assert.Equal("Test", data.FromAsset);
}

[Fact]
public void CanInterpolateOrBound()
{
var testData = new ((int Blocks, decimal Fee)[] Data, int Target, decimal Expected) []
{
([(0, 0m), (10, 100m)], 5, 50m),
([(50, 0m), (100, 100m)], 5, 0.0m),
([(50, 0m), (100, 100m)], 101, 100.0m),
([(50, 100m), (50, 100m)], 101, 100.0m),
([(50, 0m), (100, 100m)], 75, 50m),
([(0, 0m), (50, 50m), (100, 100m)], 75, 75m),
([(0, 0m), (500, 50m), (1000, 100m)], 750, 75m),
([(0, 0m), (500, 50m), (1000, 100m)], 100, 10m),
([(0, 0m), (100, 100m)], 80, 80m),
([(0, 0m), (100, 100m)], 25, 25m),
};
foreach (var t in testData)
{
var actual = MempoolSpaceFeeProvider.InterpolateOrBound(t.Data.Select(t => new MempoolSpaceFeeProvider.BlockFeeRate(t.Blocks, new FeeRate(t.Fee))).ToArray(), t.Target);
Assert.Equal(new FeeRate(t.Expected), actual);
}
}
[Fact]
public void CanRandomizeByPercentage()
{
var generated = Enumerable.Range(0, 1000).Select(_ => MempoolSpaceFeeProvider.RandomizeByPercentage(100.0m, 10.0m)).ToArray();
Assert.Empty(generated.Where(g => g < 90m));
Assert.Empty(generated.Where(g => g > 110m));
Assert.NotEmpty(generated.Where(g => g < 91m));
Assert.NotEmpty(generated.Where(g => g > 109m));
}

private void CanParseDecimalsCore(string str, decimal expected)
{
var d = JsonConvert.DeserializeObject<LedgerEntryData>(str);
Expand Down
34 changes: 33 additions & 1 deletion BTCPayServer.Tests/ThirdPartyTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using BTCPayServer.Controllers;
using BTCPayServer.Data;
using BTCPayServer.Models.StoreViewModels;
using BTCPayServer.Models.WalletViewModels;
using BTCPayServer.Rating;
using BTCPayServer.Services.Fees;
using BTCPayServer.Services.Rates;
Expand Down Expand Up @@ -92,7 +93,38 @@ public async Task CanQueryMempoolFeeProvider()
isTestnet);
var rates = await mempoolSpaceFeeProvider.GetFeeRatesAsync();
Assert.NotEmpty(rates);
await mempoolSpaceFeeProvider.GetFeeRateAsync(20);


var recommendedFees =
await Task.WhenAll(new[]
{
TimeSpan.FromMinutes(10.0), TimeSpan.FromMinutes(60.0), TimeSpan.FromHours(6.0),
TimeSpan.FromHours(24.0),
}.Select(async time =>
{
try
{
var result = await mempoolSpaceFeeProvider.GetFeeRateAsync(
(int)Network.Main.Consensus.GetExpectedBlocksFor(time));
return new WalletSendModel.FeeRateOption()
{
Target = time,
FeeRate = result.SatoshiPerByte
};
}
catch (Exception)
{
return null;
}
})
.ToArray());
//ENSURE THESE ARE LOGICAL
Assert.True(recommendedFees[0].FeeRate >= recommendedFees[1].FeeRate, $"{recommendedFees[0].Target}:{recommendedFees[0].FeeRate} >= {recommendedFees[1].Target}:{recommendedFees[1].FeeRate}");
Assert.True(recommendedFees[1].FeeRate >= recommendedFees[2].FeeRate, $"{recommendedFees[1].Target}:{recommendedFees[1].FeeRate} >= {recommendedFees[2].Target}:{recommendedFees[2].FeeRate}");
Assert.True(recommendedFees[2].FeeRate >= recommendedFees[3].FeeRate, $"{recommendedFees[2].Target}:{recommendedFees[2].FeeRate} >= {recommendedFees[3].Target}:{recommendedFees[3].FeeRate}");



}
}
[Fact]
Expand Down
118 changes: 80 additions & 38 deletions BTCPayServer/Services/Fees/MempoolSpaceFeeProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,26 @@
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using AngleSharp.Dom;
using ExchangeSharp;
using Microsoft.Extensions.Caching.Memory;
using NBitcoin;
using Org.BouncyCastle.Asn1.X509;
using YamlDotNet.Core.Tokens;

namespace BTCPayServer.Services.Fees;

public class MempoolSpaceFeeProvider(
IMemoryCache MemoryCache,
string CacheKey,
IHttpClientFactory HttpClientFactory,
bool Testnet) : IFeeProvider
IMemoryCache memoryCache,
string cacheKey,
IHttpClientFactory httpClientFactory,
bool testnet) : IFeeProvider
{
private readonly string ExplorerLink = Testnet switch
private string ExplorerLink = testnet switch
{
true => "https://mempool.space/testnet/api/v1/fees/recommended",
false => "https://mempool.space/api/v1/fees/recommended"
Expand All @@ -27,50 +32,87 @@ public async Task<FeeRate> GetFeeRateAsync(int blockTarget = 20)
{
var result = await GetFeeRatesAsync();

return result.TryGetValue(blockTarget, out var feeRate)
? feeRate
:
//try get the closest one
result[result.Keys.MinBy(key => Math.Abs(key - blockTarget))];
return InterpolateOrBound(result, blockTarget);

}

internal static FeeRate InterpolateOrBound(BlockFeeRate[] ordered, int target)
{
(BlockFeeRate lb, BlockFeeRate hb) = (ordered[0], ordered[^1]);
target = Math.Clamp(target, lb.Blocks, hb.Blocks);
for (int i = 0; i < ordered.Length; i++)
{
if (ordered[i].Blocks > lb.Blocks && ordered[i].Blocks <= target)
lb = ordered[i];
if (ordered[i].Blocks < hb.Blocks && ordered[i].Blocks >= target)
hb = ordered[i];
}
if (hb.Blocks == lb.Blocks)
return hb.FeeRate;
var a = (decimal)(target - lb.Blocks) / (decimal)(hb.Blocks - lb.Blocks);
return new FeeRate((1 - a) * lb.FeeRate.SatoshiPerByte + a * hb.FeeRate.SatoshiPerByte);
}

public Task<Dictionary<int, FeeRate>> GetFeeRatesAsync()
internal async Task<BlockFeeRate[]> GetFeeRatesAsync()
{
return MemoryCache.GetOrCreateAsync(CacheKey, async entry =>
try
{
entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(5);
var client = HttpClientFactory.CreateClient(nameof(MempoolSpaceFeeProvider));
using var result = await client.GetAsync(ExplorerLink);
result.EnsureSuccessStatusCode();
var recommendedFees = await result.Content.ReadAsAsync<Dictionary<string, decimal>>();
var feesByBlockTarget = new Dictionary<int, FeeRate>();
foreach ((var feeId, decimal value) in recommendedFees)
return (await memoryCache.GetOrCreateAsync(cacheKey, async entry =>
{
var target = feeId switch
{
"fastestFee" => 1,
"halfHourFee" => 3,
"hourFee" => 6,
"economyFee" when recommendedFees.TryGetValue("minimumFee", out var minFee) && minFee == value => 144,
"economyFee" => 72,
"minimumFee" => 144,
_ => -1
};
feesByBlockTarget.TryAdd(target, new FeeRate(RandomizeByPercentage(value, 10)));
}
return feesByBlockTarget;
})!;
entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(5);
return await GetFeeRatesCore();
}))!;
}
catch (Exception)
{
memoryCache.Remove(cacheKey);
throw;
}
}
internal record BlockFeeRate(int Blocks, FeeRate FeeRate);
async Task<BlockFeeRate[]> GetFeeRatesCore()
{
var client = httpClientFactory.CreateClient(nameof(MempoolSpaceFeeProvider));
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10));
using var result = await client.GetAsync(ExplorerLink, cts.Token);
result.EnsureSuccessStatusCode();
var recommendedFees = await result.Content.ReadAsAsync<Dictionary<string, decimal>>();
var r = new List<BlockFeeRate>();
foreach ((var feeId, decimal value) in recommendedFees)
{
var target = feeId switch
{
"fastestFee" => 1,
"halfHourFee" => 3,
"hourFee" => 6,
"economyFee" when recommendedFees.TryGetValue("minimumFee", out var minFee) && minFee == value => 144,
"economyFee" => 72,
"minimumFee" => 144,
_ => -1
};
r.Add(new(target, new FeeRate(value)));
}
var ordered = r.OrderBy(k => k.Blocks).ToArray();
for (var i = 0; i < ordered.Length; i++)
{
// Randomize a bit
ordered[i] = ordered[i] with { FeeRate = new FeeRate(RandomizeByPercentage(ordered[i].FeeRate.SatoshiPerByte, 10m)) };
if (i > 0) // Make sure feerate always increase
ordered[i] = ordered[i] with { FeeRate = FeeRate.Max(ordered[i - 1].FeeRate, ordered[i].FeeRate) };
}
return ordered;
}

static decimal RandomizeByPercentage(decimal value, int percentage)
internal static decimal RandomizeByPercentage(decimal value, decimal percentage)
{
decimal range = value * percentage / 100m;
var res = value + range * (Random.Shared.NextDouble() < 0.5 ? -1 : 1);

if (value is 1)
return 1;
decimal range = (value * percentage) / 100m;
var res = value + (range * 2.0m) * ((decimal)(Random.Shared.NextDouble() - 0.5));
return res switch
{
< 1m => 1m,
> 850m => 2000m,
> 2000m => 2000m,
_ => res
};
}
Expand Down

0 comments on commit 3eec9cb

Please sign in to comment.