Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Index for BIP 157 block filters #14121

Merged
merged 12 commits into from Apr 18, 2019

init: Add CLI option to enable block filter index.

  • Loading branch information...
jimpo committed Aug 30, 2018
commit ff351050968f290787cd5fa456d394380f64fec3
@@ -2,6 +2,9 @@
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <mutex>
#include <sstream>

#include <blockfilter.h>
#include <crypto/siphash.h>
#include <hash.h>
@@ -218,6 +221,40 @@ bool BlockFilterTypeByName(const std::string& name, BlockFilterType& filter_type
return false;
}

const std::vector<BlockFilterType>& AllBlockFilterTypes()
{
static std::vector<BlockFilterType> types;

static std::once_flag flag;
std::call_once(flag, []() {
types.reserve(g_filter_types.size());
for (auto entry : g_filter_types) {
types.push_back(entry.first);
}
});

return types;
}

const std::string& ListBlockFilterTypes()
{
static std::string type_list;

static std::once_flag flag;
std::call_once(flag, []() {
std::stringstream ret;
bool first = true;
for (auto entry : g_filter_types) {
if (!first) ret << ", ";
ret << entry.second;
first = false;
}
type_list = ret.str();
});

return type_list;
}

static GCSFilter::ElementSet BasicFilterElements(const CBlock& block,
const CBlockUndo& block_undo)
{
@@ -96,6 +96,12 @@ const std::string& BlockFilterTypeName(BlockFilterType filter_type);
/** Find a filter type by its human-readable name. */
bool BlockFilterTypeByName(const std::string& name, BlockFilterType& filter_type);

/** Get a list of known filter types. */
const std::vector<BlockFilterType>& AllBlockFilterTypes();

/** Get a comma-separated list of known filter type names. */
const std::string& ListBlockFilterTypes();

/**
* Complete block filter struct as defined in BIP 157. Serialization matches
* payload of "cfilter" messages.
@@ -12,6 +12,7 @@
#include <addrman.h>
#include <amount.h>
#include <banman.h>
#include <blockfilter.h>
#include <chain.h>
#include <chainparams.h>
#include <checkpoints.h>
@@ -20,6 +21,7 @@
#include <fs.h>
#include <httpserver.h>
#include <httprpc.h>
#include <index/blockfilterindex.h>
#include <interfaces/chain.h>
#include <index/txindex.h>
#include <key.h>
@@ -189,6 +191,7 @@ void Interrupt()
if (g_txindex) {
g_txindex->Interrupt();
}
ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Interrupt(); });
}

void Shutdown(InitInterfaces& interfaces)
@@ -220,6 +223,7 @@ void Shutdown(InitInterfaces& interfaces)
if (peerLogic) UnregisterValidationInterface(peerLogic.get());
if (g_connman) g_connman->Stop();
if (g_txindex) g_txindex->Stop();
ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Stop(); });

StopTorControl();

@@ -234,6 +238,7 @@ void Shutdown(InitInterfaces& interfaces)
g_connman.reset();
g_banman.reset();
g_txindex.reset();
DestroyAllBlockFilterIndexes();

if (g_is_mempool_loaded && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) {
DumpMempool();
@@ -404,6 +409,10 @@ void SetupServerArgs()
hidden_args.emplace_back("-sysperms");
#endif
gArgs.AddArg("-txindex", strprintf("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)", DEFAULT_TXINDEX), false, OptionsCategory::OPTIONS);
gArgs.AddArg("-blockfilterindex=<type>",
strprintf("Maintain an index of compact filters by block (default: %s, values: %s).", DEFAULT_BLOCKFILTERINDEX, ListBlockFilterTypes()) +
" If <type> is not supplied or if <type> = 1, indexes for all known types are enabled.",
false, OptionsCategory::OPTIONS);

gArgs.AddArg("-addnode=<ip>", "Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info). This option can be specified multiple times to add multiple nodes.", false, OptionsCategory::CONNECTION);
gArgs.AddArg("-banscore=<n>", strprintf("Threshold for disconnecting misbehaving peers (default: %u)", DEFAULT_BANSCORE_THRESHOLD), false, OptionsCategory::CONNECTION);
@@ -886,6 +895,7 @@ int nUserMaxConnections;
int nFD;
ServiceFlags nLocalServices = ServiceFlags(NODE_NETWORK | NODE_NETWORK_LIMITED);
int64_t peer_connect_timeout;
std::vector<BlockFilterType> g_enabled_filter_types;

} // namespace

@@ -954,10 +964,29 @@ bool AppInitParameterInteraction()
return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist."), gArgs.GetArg("-blocksdir", "").c_str()));
}

// parse and validate enabled filter types
std::string blockfilterindex_value = gArgs.GetArg("-blockfilterindex", DEFAULT_BLOCKFILTERINDEX);
if (blockfilterindex_value == "" || blockfilterindex_value == "1") {
g_enabled_filter_types = AllBlockFilterTypes();
} else if (blockfilterindex_value != "0") {
const std::vector<std::string> names = gArgs.GetArgs("-blockfilterindex");
g_enabled_filter_types.reserve(names.size());
for (const auto& name : names) {
BlockFilterType filter_type;
if (!BlockFilterTypeByName(name, filter_type)) {
return InitError(strprintf(_("Unknown -blockfilterindex value %s."), name));
}
g_enabled_filter_types.push_back(filter_type);
}
}

// if using block pruning, then disallow txindex
if (gArgs.GetArg("-prune", 0)) {
if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX))
return InitError(_("Prune mode is incompatible with -txindex."));
if (!g_enabled_filter_types.empty()) {
return InitError(_("Prune mode is incompatible with -blockfilterindex."));
}

This comment has been minimized.

Copy link
@MarcoFalke

MarcoFalke Apr 18, 2019

Member

in commit ff35105:

Why is it incompatible? Maybe should add a cpp comment to explain.

See also https://github.com/bitcoin/bips/blob/master/bip-0157.mediawiki#node-operation

"Nodes MAY prune block data after generating and storing all filters for a block."

This comment has been minimized.

Copy link
@jimpo

jimpo Apr 19, 2019

Author Contributor

You can turn on the filter index and it will build in the background using the blocks on disk. This wouldn't work in pruned mode. It might be safe though to allow pruning if the block filter index tip is already within chain tip height - MIN_BLOCKS_TO_KEEP. But then we couldn't prune until the block filter index is in sync (it has its own sort of initial sync logic).

Not prohibitively difficult though, just requires a bit more thought.

}

// -bind and -whitebind can't be set when not listening
@@ -1448,6 +1477,13 @@ bool AppInitMain(InitInterfaces& interfaces)
nTotalCache -= nBlockTreeDBCache;
int64_t nTxIndexCache = std::min(nTotalCache / 8, gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0);
nTotalCache -= nTxIndexCache;
int64_t filter_index_cache = 0;
if (!g_enabled_filter_types.empty()) {
size_t n_indexes = g_enabled_filter_types.size();
int64_t max_cache = std::min(nTotalCache / 8, max_filter_index_cache << 20);
filter_index_cache = max_cache / n_indexes;
nTotalCache -= filter_index_cache * n_indexes;
}
int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache
nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache
nTotalCache -= nCoinDBCache;
@@ -1458,6 +1494,10 @@ bool AppInitMain(InitInterfaces& interfaces)
if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
LogPrintf("* Using %.1f MiB for transaction index database\n", nTxIndexCache * (1.0 / 1024 / 1024));
}
for (BlockFilterType filter_type : g_enabled_filter_types) {
LogPrintf("* Using %.1f MiB for %s block filter index database\n",
filter_index_cache * (1.0 / 1024 / 1024), BlockFilterTypeName(filter_type));
}
LogPrintf("* Using %.1f MiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024));
LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024));

@@ -1645,6 +1685,11 @@ bool AppInitMain(InitInterfaces& interfaces)
g_txindex->Start();
}

for (const auto& filter_type : g_enabled_filter_types) {
InitBlockFilterIndex(filter_type, filter_index_cache, false, fReindex);
GetBlockFilterIndex(filter_type)->Start();
}

// ********************************************************* Step 9: load wallet
for (const auto& client : interfaces.chain_clients) {
if (!client->load()) {
@@ -37,6 +37,8 @@ static const int64_t nMaxBlockDBCache = 2;
// Unlike for the UTXO database, for the txindex scenario the leveldb cache make
// a meaningful difference: https://github.com/bitcoin/bitcoin/pull/8273#issuecomment-229601991
static const int64_t nMaxTxIndexCache = 1024;
//! Max memory allocated to all block filter index caches combined in MiB.
static const int64_t max_filter_index_cache = 1024;
//! Max memory allocated to coin DB specific cache (MiB)
static const int64_t nMaxCoinsDBCache = 8;

@@ -118,6 +118,7 @@ static const int64_t MAX_FEE_ESTIMATION_TIP_AGE = 3 * 60 * 60;
static const bool DEFAULT_PERMIT_BAREMULTISIG = true;
static const bool DEFAULT_CHECKPOINTS_ENABLED = true;
static const bool DEFAULT_TXINDEX = false;
static const char* const DEFAULT_BLOCKFILTERINDEX = "0";

This comment has been minimized.

Copy link
@MarcoFalke

MarcoFalke Apr 18, 2019

Member

I believe we make this extern in the header, so that it is not added to each obj file?

This comment has been minimized.

Copy link
@jimpo

jimpo Apr 19, 2019

Author Contributor

Yeah. This applies to all the defaults, right?

This comment has been minimized.

Copy link
@MarcoFalke

MarcoFalke Apr 22, 2019

Member

I think only strings and cstrings and not POD types

static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100;
/** Default for -persistmempool */
static const bool DEFAULT_PERSIST_MEMPOOL = true;
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.