Skip to content

Commit

Permalink
Add fee_est tool for debugging fee estimation code
Browse files Browse the repository at this point in the history
  • Loading branch information
@ryanofsky
ryanofsky committed Jun 5, 2017
1 parent 0637194 commit 43fa494
Show file tree
Hide file tree
Showing 3 changed files with 378 additions and 2 deletions.
11 changes: 9 additions & 2 deletions src/Makefile.test.include
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

TESTS += test/test_bitcoin
bin_PROGRAMS += test/test_bitcoin
bin_PROGRAMS += test/test_bitcoin test/fee_est/fee_est
noinst_PROGRAMS += test/test_bitcoin_fuzzy
TEST_SRCDIR = test
TEST_BINARY=test/test_bitcoin$(EXEEXT)
Expand Down Expand Up @@ -108,10 +108,17 @@ test_test_bitcoin_LDADD += $(LIBBITCOIN_CONSENSUS) $(BDB_LIBS) $(SSL_LIBS) $(CRY
test_test_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -static

if ENABLE_ZMQ
test_test_bitcoin_LDADD += $(ZMQ_LIBS)
test_test_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS)
endif
#

# fee_est/fee_est binary #
test_fee_est_fee_est_SOURCES = test/fee_est/fee_est.cpp
test_fee_est_fee_est_CPPFLAGS = $(test_test_bitcoin_CPPFLAGS)
test_fee_est_fee_est_CXXFLAGS = $(test_test_bitcoin_CXXFLAGS)
test_fee_est_fee_est_LDFLAGS = $(test_test_bitcoin_LDFLAGS)
test_fee_est_fee_est_LDADD = $(test_test_bitcoin_LDADD) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO)

# test_bitcoin_fuzzy binary #
test_test_bitcoin_fuzzy_SOURCES = test/test_bitcoin_fuzzy.cpp
test_test_bitcoin_fuzzy_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES)
Expand Down
37 changes: 37 additions & 0 deletions src/test/fee_est/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
src/test/fee_est -- Fee estimation offline testing tool
=======================================================

The `fee_est` tool is intended to help debug and test changes in bitcoin fee
estimation code using transaction data gathered from live bitcoin nodes.

Transaction data can be collected by running bitcoind with `-estlog` parameter
which will produce newline-delimited json file (http://ndjson.org/,
http://jsonlines.org/) which logs relevant transaction information. The
`fee_est` tool can parse the log to produce graphs of fee estimation data and do
some basic analysis of the results. The implementation is intended to be simple
enough that it can be easily customized to do more specific analysis.

Example usage:

Run bitcoind collecting fee estimation data:

```
$ make -C src bitcoind test/fee_est/fee_est
$ src/bitcoind -estlog=est.log &
```

Run fee estimation tool producing graph of estimates:

```
$ src/test/fee_est/fee_est -ograph=graph.html est.log
$ chrome graph.html
```

Run fee estimation tool computing some basic statistics:

```
$ src/test/fee_est/fee_est -cross est.log
Non-test txs: 1407226
Test txs: 5603 total (4345 kept, 631 discarded unconfirmed, 627 discarded outliers)
Mean squared error: 71.0651
```
332 changes: 332 additions & 0 deletions src/test/fee_est/fee_est.cpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,332 @@
#include "policy/fees.h"
#include "policy/fees_input.h"
#include "txmempool.h"
#include "util.h"

#include <univalue.h>

#include <fstream>
#include <stdio.h>

std::string Usage()
{
std::string usage = "Usage: fee_est [options] <estlog.txt>\n";
usage += HelpMessageGroup("Options:");
usage += HelpMessageOpt("-?, -help", "This help message");
usage += HelpMessageOpt("-ograph=<graph.html>", "Generate target vs feerate graph output after replaying transaction log.");
usage += HelpMessageOpt("-odat=<fee_estimates.dat>", "Save fee estimator state after after replaying transaction log.");
usage += HelpMessageOpt("-idat=<fee_estimates.dat>", "Load fee estimator state after before replaying transaction log.");
usage += HelpMessageOpt("-dat=<fee_estimates.dat>", "Shortcut for -idat=<fee_estimates.dat> -odat=<fee_estimates.dat>");
usage += HelpMessageOpt("-cross", "Treat half the transactions from the log as test data and print cross-validation statistics.");
return usage;
}

struct Options
{
std::string logFileIn;
std::string dataFileIn;
std::string dataFileOut;
std::string graphFileOut;
bool cross = false;
bool help = false;
bool error = false;
};

Options ParseCommandLine(int argc, char** argv)
{
auto match = [&](const char* arg, const char* name, const char** value = nullptr) {
if (arg[0] != '-') return false;
size_t n = strlen(name);
if (strncmp(arg + 1, name, n) != 0) return false;
if (value) return (*value = arg[n + 1] == '=' ? arg + n + 2 : nullptr) != nullptr;
return arg[n + 1] == '\0';
};

Options options;
for (int i = 1; i < argc; ++i) {
const char* arg = argv[i];
const char* value;
if (match(arg, "ograph", &value)) {
options.graphFileOut = value;
} else if (match(arg, "odat", &value)) {
options.dataFileOut = value;
} else if (match(arg, "idat", &value)) {
options.dataFileIn = value;
} else if (match(arg, "dat", &value)) {
options.dataFileIn = options.dataFileOut = value;
} else if (match(arg, "cross")) {
options.cross = true;
} else if (match(arg, "help") || match(arg, "h") || match(arg, "?")) {
options.help = true;
} else if (arg[0] != '-' && options.logFileIn.empty()) {
options.logFileIn = arg;
} else {
fprintf(stderr, "Error: unexpected argument: '%s'\n", arg);
options.error = true;
}
}

if (options.logFileIn.empty() && !options.help) {
fprintf(stderr, "Error: missing required log file argument.\n");
options.error = true;
}

return options;
}

struct TxData
{
TxData(CAmount fee, int size, int height) : fee(fee), size(size), height(height) {}
CAmount fee;
int size;
int height;
int expectedBlocks = -1;
int actualBlocks = -1;
};

using TxMap = std::map<uint256, TxData>;

const char* const GRAPH_HTML = R"(<!DOCTYPE html>
<meta charset="utf-8">
<title>Fee Graph</title>
<script src="https://d3js.org/d3.v3.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/vega/3.0.0-beta.30/vega.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/vega-lite/2.0.0-beta.2/vega-lite.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/vega-embed/3.0.0-beta.14/vega-embed.js"></script>
<style media="screen">
.vega-actions a {
margin-right: 5px;
}
</style>
<div id=vis></div>
<script>
var spec = {
"$schema": "https://vega.github.io/schema/vega-lite/v2.json",
"width": 800,
"height": 600,
"layer": [
{
"data": {
"values": %s
},
"mark": "tick",
"encoding": {
"x": {"field": "blocks", "type": "quantitative"},
"y": {"field": "feeRate", "type": "quantitative"},
"color": {"field": "height", "type": "quantitative"}
}
},
{
"data": {
"values": %s
},
"encoding": {
"x": {"field": "blocks", "type": "quantitative"},
"y": {"field": "feeRate", "type": "quantitative"},
"color": {"field": "mode", "type": "nominal"}
},
"mark": "line",
}
]
}
vega.embed("#vis", spec);
</script>
)";

// Maximum number of randomly sampled transactions to show on graph.
const int SAMPLE_TXS = 50000;

// Drop 1/20th of sampled transactions with highest feerates if they are
// outliers that would change the scale of the graph.
const int DROP_FRAC = 20;

void WriteGraph(const std::string& filename,
const TxMap& txMap,
const std::vector<unsigned int>& targets,
CBlockPolicyEstimator& estimator,
CTxMemPool& pool)
{
UniValue fees(UniValue::VARR);

int64_t maxFeeRate = 0;
const unsigned int maxTarget = estimator.getMaxTarget();
for (bool conservative : {false, true}) {
for (unsigned int target : targets) {
if (target > maxTarget) continue;
UniValue fee(UniValue::VOBJ);
fee.pushKV("mode", conservative ? "conservative" : "default");
fee.pushKV("blocks", int(target));
int64_t feeRate = estimator.estimateSmartFee(target, nullptr, pool, conservative).GetFeePerK();
fee.pushKV("feeRate", feeRate);
fees.push_back(std::move(fee));
maxFeeRate = std::max(maxFeeRate, feeRate);
}
}

std::vector<const TxMap::value_type*> sample;
sample.reserve(SAMPLE_TXS);
int i = 0;
for (const auto& entry : txMap) {
if (entry.second.actualBlocks > int(maxTarget)) {
continue;
} else if (sample.size() < SAMPLE_TXS) {
sample.emplace_back(&entry);
} else {
int j = rand() % i;
if (j < SAMPLE_TXS) {
sample[j] = &entry;
}
}
++i;
}

if (sample.size() > 0) {
std::vector<int64_t> feeRates;
feeRates.reserve(sample.size());
for (const auto& entry : sample) {
feeRates.emplace_back(CFeeRate(entry->second.fee, entry->second.size).GetFeePerK());
}
auto nth = feeRates.begin() + sample.size() / DROP_FRAC;
std::nth_element(feeRates.begin(), nth, feeRates.end(), std::greater<int64_t>());
maxFeeRate = std::max(maxFeeRate, *nth);
}

UniValue txs(UniValue::VARR);
for (const auto& entry : sample) {
if (entry->second.actualBlocks > 0) {
int64_t feeRate = CFeeRate(entry->second.fee, entry->second.size).GetFeePerK();
if (feeRate <= maxFeeRate) {
UniValue tx(UniValue::VOBJ);
tx.pushKV("feeRate", feeRate);
tx.pushKV("blocks", entry->second.actualBlocks);
tx.pushKV("height", entry->second.height);
txs.push_back(std::move(tx));
}
}
}

std::ofstream file(filename);
file << strprintf(GRAPH_HTML, txs.write(), fees.write());
}

bool UpdateCross(TxMap::value_type& tx,
const std::vector<unsigned int>& targets,
CBlockPolicyEstimator& estimator,
CTxMemPool& pool)
{
if (*tx.first.begin() == 0) {
auto it = std::lower_bound(targets.begin(), targets.end(), nullptr, [&](unsigned int target, std::nullptr_t) {
CAmount estFee = estimator.estimateSmartFee(target, nullptr, pool, false).GetFee(tx.second.size);
return estFee <= 1 || tx.second.fee < estFee;
});
tx.second.expectedBlocks = it == targets.end() ? std::numeric_limits<int>::max() : *it;
return false;
}
return true;
}

void PrintCross(const TxMap& txMap,
const std::vector<unsigned int>& targets,
CBlockPolicyEstimator& estimator,
CTxMemPool& pool)
{
int nonTestTxs = 0;
int unconfirmedTestTxs = 0;
int outlierTestTxs = 0;
int keptTestTxs = 0;
int64_t errsq = 0;
for (const auto& entry : txMap) {
if (entry.second.expectedBlocks == -1) {
++nonTestTxs;
} else if (entry.second.actualBlocks == -1) {
++unconfirmedTestTxs;
} else if (entry.second.expectedBlocks == std::numeric_limits<int>::max()) {
++outlierTestTxs;
} else {
++keptTestTxs;
int err = entry.second.expectedBlocks - entry.second.actualBlocks;
errsq += err * err;
}
}
printf("Non-test txs: %i\n", nonTestTxs);
printf("Test txs: %i total (%i kept, %i discarded unconfirmed, %i discarded outliers)\n",
keptTestTxs + unconfirmedTestTxs + outlierTestTxs, keptTestTxs, unconfirmedTestTxs, outlierTestTxs);
if (keptTestTxs > 0) {
printf("Mean squared error: %g\n", double(errsq) / double(keptTestTxs));
}
}

int main(int argc, char** argv)
{
Options options = ParseCommandLine(argc, argv);

if (options.help) {
printf("%s", Usage().c_str());
return EXIT_SUCCESS;
} else if (options.error) {
fprintf(stderr, "Try `fee_est -h` for more information.\n");
return EXIT_FAILURE;
} else if (options.dataFileOut.empty() && options.graphFileOut.empty() && !options.cross) {
fprintf(stderr, "Warning: No output options specified. Try -ograph, -odat, -cross options, or `fee_est -h` "
"for more information.\n");
}

CBlockPolicyEstimator estimator;
CBlockPolicyInput input(estimator);

if (!options.dataFileIn.empty() && !input.readData(options.dataFileIn)) {
fprintf(stderr, "Error: failed to load fee estimate data file '%s'\n", options.dataFileOut.c_str());
return 1;
}

const std::vector<unsigned int> targets = estimator.getUniqueTargets();
CTxMemPool pool;
TxMap txMap;

auto filter = [&](UniValue& value) {
bool keep = true;
const UniValue& tx = value["tx"];
if (tx.isObject()) {
auto inserted = txMap.emplace(uint256S(tx["hash"].get_str()),
TxData(tx["fee"].get_int64(), tx["size"].get_int(), tx["height"].get_int()));
if (options.cross && !UpdateCross(*inserted.first, targets, estimator, pool)) {
keep = false;
}
}

const UniValue& block = value["block"];
if (block.isObject()) {
int blockHeight = block["height"].get_int();
const auto& txs = value["txs"].getValues();
for (const UniValue& tx : txs) {
auto it = txMap.find(uint256S(tx["hash"].get_str()));
if (it != txMap.end()) {
it->second.actualBlocks = blockHeight - tx["height"].get_int();
}
}
}

return keep;
};

if (!input.readLog(options.logFileIn, std::ref(filter))) {
fprintf(stderr, "Error: failed to load fee data log file '%s'\n", options.logFileIn.c_str());
return EXIT_FAILURE;
}

if (!options.dataFileOut.empty() && !input.writeData(options.dataFileOut)) {
fprintf(stderr, "Error: failed to write fee estimate data file '%s'\n", options.dataFileOut.c_str());
return EXIT_FAILURE;
}

if (!options.graphFileOut.empty()) {
WriteGraph(options.graphFileOut, txMap, targets, estimator, pool);
return EXIT_FAILURE;
}

if (options.cross) {
PrintCross(txMap, targets, estimator, pool);
}

return EXIT_SUCCESS;
}

0 comments on commit 43fa494

Please sign in to comment.