diff --git a/console/executor.cpp b/console/executor.cpp index bd7088a5..fc0379fa 100644 --- a/console/executor.cpp +++ b/console/executor.cpp @@ -44,6 +44,8 @@ using namespace std::placeholders; const std::string executor::name_{ "bn" }; const std::string executor::close_{ "c" }; const std::string executor::backup_{ "b" }; +const std::string executor::measure_{ "m" }; +const std::string executor::test_{ "t" }; const std::unordered_map executor::defined_ { { levels::application, true }, @@ -281,7 +283,8 @@ void executor::measure_size() const query_.strong_tx_size() % query_.validated_tx_size() % query_.validated_bk_size() % - query_.address_size()); + query_.address_size() % + query_.neutrino_size()); console(format(BN_MEASURE_RECORDS) % query_.header_records() % query_.tx_records() % @@ -300,7 +303,8 @@ void executor::measure_size() const query_.strong_tx_buckets() % query_.validated_tx_buckets() % query_.validated_bk_buckets() % - query_.address_buckets()); + query_.address_buckets() % + query_.neutrino_buckets()); // This one can take a few seconds on cold iron. console(format(BN_MEASURE_COLLISION_RATES) % ((1.0 * query_.header_records()) / query_.header_buckets()) % @@ -311,7 +315,8 @@ void executor::measure_size() const ((1.0 * query_.strong_tx_records()) / query_.strong_tx_buckets()) % ((1.0 * query_.tx_records()) / query_.validated_tx_buckets()) % ((1.0 * query_.header_records()) / query_.validated_bk_buckets()) % - ((1.0 * query_.address_records()) / query_.address_buckets())); + ((1.0 * query_.address_records()) / query_.address_buckets()) % + ((1.0 * query_.header_records()) / query_.neutrino_buckets())); console(format(BN_MEASURE_PROGRESS) % query_.get_fork() % query_.get_top_confirmed() % @@ -690,34 +695,11 @@ void executor::scan_collisions() const spend.shrink_to_fit(); } -// arbitrary testing (const). -void executor::read_test() const -{ - console("No read test implemented."); -} - -#if defined(UNDEFINED) - -void executor::read_test() const -{ - // Binance wallet address with 1,380,169 transaction count. - // blockstream.info/address/bc1qm34lsc65zpw79lxes69zkqmk6ee3ewf0j77s3h - const auto data = base16_array("0014dc6bf86354105de2fcd9868a2b0376d6731cb92f"); - const chain::script output_script{ data, false }; - const auto mnemonic = output_script.to_string(chain::flags::all_rules); - console(format("Getting payments to {%1%}.") % mnemonic); - - const auto start = fine_clock::now(); - database::output_links outputs{}; - if (!query_.to_address_outputs(outputs, output_script.hash())) - return; - - const auto end = fine_clock::now(); - const auto span = (end - start).count() / 1'000'000; - - console(format("Found [%1%] outputs of {%2%} in [%3%] ms.") % - outputs.size() % mnemonic % span); -} +////// arbitrary testing (const). +////void executor::read_test() const +////{ +//// console("No read test implemented."); +////} void executor::read_test() const { @@ -729,8 +711,8 @@ void executor::read_test() const std::set keys{}; auto tx = start_tx; - ////console(format("Getting first [%1%] output address hashes.") % - //// target_count); + console(format("Getting first [%1%] output address hashes.") % + target_count); auto start = fine_clock::now(); while (!cancel_ && keys.size() < target_count) @@ -788,7 +770,7 @@ void executor::read_test() const outs.reserve(target_count); using namespace database; - start = fine_clock::now(); + start = fine_clock::now(); for (auto& key: keys) { auto address_it = store_.address.it(key); @@ -945,67 +927,90 @@ void executor::read_test() const console(format("Got all [%1%] payments to [%2%] addresses in [%3%] ms.") % outs.size() % keys.size() % ((end - start).count() / ns_to_ms)); - console( - "output_script_hash, " - "output_fk, " - "spend_fk, " - "input_fk, " - - "ouput_tx_fk, " - "ouput_tx_hash, " - "ouput_tx_pos, " - - "ouput_bk_fk, " - "ouput_bk_hash, " - "ouput_bk_height, " - - "input_tx_fk, " - "input_tx_hash, " - "input_tx_pos, " - - "input_bk_fk, " - "input_bk_hash, " - "sinput_bk_height, " - - "output_script " - "input_script, " - ); - - for (const auto& row: outs) - { - if (cancel_) break; - - const auto input = !row.input ? "{unspent}" : - row.input->script().to_string(chain::flags::all_rules); - - const auto output = !row.output ? "{error}" : - row.output->script().to_string(chain::flags::all_rules); + ////console( + //// "output_script_hash, " + //// "output_fk, " + //// "spend_fk, " + //// "input_fk, " + //// + //// "ouput_tx_fk, " + //// "ouput_tx_hash, " + //// "ouput_tx_pos, " + //// + //// "ouput_bk_fk, " + //// "ouput_bk_hash, " + //// "ouput_bk_height, " + //// + //// "input_tx_fk, " + //// "input_tx_hash, " + //// "input_tx_pos, " + //// + //// "input_bk_fk, " + //// "input_bk_hash, " + //// "sinput_bk_height, " + //// + //// "output_script " + //// "input_script, " + ////); + //// + ////for (const auto& row: outs) + ////{ + //// if (cancel_) break; + //// + //// const auto input = !row.input ? "{unspent}" : + //// row.input->script().to_string(chain::flags::all_rules); + //// + //// const auto output = !row.output ? "{error}" : + //// row.output->script().to_string(chain::flags::all_rules); + //// + //// console(format("%1%, %2%, %3%, %4%, %5%, %6%, %7%, %8%, %9%, %10%, %11%, %12%, %13%, %14%, %15%, %16%, %17%, %18%") % + //// encode_hash(row.address) % + //// row.output_fk % + //// row.spend_fk% + //// row.input_fk% + //// + //// row.tx_fk % + //// encode_hash(row.tx_hash) % + //// row.tx_position % + //// + //// row.bk_fk % + //// encode_hash(row.bk_hash) % + //// row.bk_height % + //// + //// row.in_tx_fk % + //// encode_hash(row.in_tx_hash) % + //// row.in_tx_position % + //// + //// row.in_bk_fk % + //// encode_hash(row.in_bk_hash) % + //// row.in_bk_height % + //// + //// output% + //// input); + ////} +} - console(format("%1%, %2%, %3%, %4%, %5%, %6%, %7%, %8%, %9%, %10%, %11%, %12%, %13%, %14%, %15%, %16%, %17%, %18%") % - encode_hash(row.address) % - row.output_fk % - row.spend_fk% - row.input_fk% +#if defined(UNDEFINED) - row.tx_fk % - encode_hash(row.tx_hash) % - row.tx_position % +void executor::read_test() const +{ + // Binance wallet address with 1,380,169 transaction count. + // blockstream.info/address/bc1qm34lsc65zpw79lxes69zkqmk6ee3ewf0j77s3h + const auto data = base16_array("0014dc6bf86354105de2fcd9868a2b0376d6731cb92f"); + const chain::script output_script{ data, false }; + const auto mnemonic = output_script.to_string(chain::flags::all_rules); + console(format("Getting payments to {%1%}.") % mnemonic); - row.bk_fk % - encode_hash(row.bk_hash) % - row.bk_height % + const auto start = fine_clock::now(); + database::output_links outputs{}; + if (!query_.to_address_outputs(outputs, output_script.hash())) + return; - row.in_tx_fk % - encode_hash(row.in_tx_hash) % - row.in_tx_position % + const auto end = fine_clock::now(); + const auto span = (end - start).count() / 1'000'000; - row.in_bk_fk % - encode_hash(row.in_bk_hash) % - row.in_bk_height % - - output% - input); - } + console(format("Found [%1%] outputs of {%2%} in [%3%] ms.") % + outputs.size() % mnemonic % span); } // This was caused by concurrent redundant downloads at tail following restart. @@ -1845,7 +1850,8 @@ bool executor::do_initchain() query_.strong_tx_size() % query_.validated_tx_size() % query_.validated_bk_size() % - query_.address_size()); + query_.address_size() % + query_.neutrino_buckets()); console(format(BN_MEASURE_RECORDS) % query_.header_records() % query_.tx_records() % @@ -1864,7 +1870,8 @@ bool executor::do_initchain() query_.strong_tx_buckets() % query_.validated_tx_buckets() % query_.validated_bk_buckets() % - query_.address_buckets()); + query_.address_buckets() % + query_.neutrino_buckets()); console(format(BN_MEASURE_PROGRESS) % query_.get_fork() % query_.get_top_confirmed() % @@ -2291,6 +2298,20 @@ void executor::subscribe_capture() return !error; } + // Execute measure (not a toggle). + if (token == measure_) + { + measure_size(); + return !ec; + } + + // Execute read test (not a toggle). + if (token == test_) + { + read_test(); + return !ec; + } + if (!keys_.contains(token)) { logger("CONSOLE: '" + line + "'"); @@ -2443,16 +2464,6 @@ bool executor::do_run() return false; } - logger(format(BN_MEASURE_BUCKETS) % - query_.header_buckets() % - query_.txs_buckets() % - query_.tx_buckets() % - query_.point_buckets() % - query_.spend_buckets() % - query_.strong_tx_buckets() % - query_.validated_tx_buckets() % - query_.validated_bk_buckets() % - query_.address_buckets()); logger(format(BN_MEASURE_SIZES) % query_.header_size() % query_.txs_size() % @@ -2467,7 +2478,8 @@ bool executor::do_run() query_.strong_tx_size() % query_.validated_tx_size() % query_.validated_bk_size() % - query_.address_size()); + query_.address_size() % + query_.neutrino_size()); logger(format(BN_MEASURE_RECORDS) % query_.header_records() % query_.tx_records() % @@ -2477,6 +2489,17 @@ bool executor::do_run() query_.spend_records() % query_.strong_tx_records() % query_.address_records()); + logger(format(BN_MEASURE_BUCKETS) % + query_.header_buckets() % + query_.txs_buckets() % + query_.tx_buckets() % + query_.point_buckets() % + query_.spend_buckets() % + query_.strong_tx_buckets() % + query_.validated_tx_buckets() % + query_.validated_bk_buckets() % + query_.address_buckets() % + query_.neutrino_buckets()); logger(format(BN_MEASURE_PROGRESS) % query_.get_fork() % query_.get_top_confirmed() % @@ -2526,7 +2549,8 @@ bool executor::do_run() query_.strong_tx_size() % query_.validated_tx_size() % query_.validated_bk_size() % - query_.address_size()); + query_.address_size() % + query_.neutrino_size()); logger(format(BN_MEASURE_RECORDS) % query_.header_records() % query_.tx_records() % diff --git a/console/executor.hpp b/console/executor.hpp index 807f755a..4fe0acf8 100644 --- a/console/executor.hpp +++ b/console/executor.hpp @@ -88,6 +88,8 @@ class executor static const std::string name_; static const std::string close_; static const std::string backup_; + static const std::string measure_; + static const std::string test_; static const std::unordered_map defined_; static const std::unordered_map display_; static const std::unordered_map keys_; diff --git a/console/localize.hpp b/console/localize.hpp index 824fe815..0e7e9dcf 100644 --- a/console/localize.hpp +++ b/console/localize.hpp @@ -66,16 +66,6 @@ namespace node { "Restored database in %1% ms." // --measure -#define BN_MEASURE_RECORDS \ - "Table records...\n" \ - " header :%1%\n" \ - " tx :%2%\n" \ - " point :%3%\n" \ - " candidate:%4%\n" \ - " confirmed:%5%\n" \ - " spend :%6%\n" \ - " strong_tx:%7%\n" \ - " address :%8%" #define BN_MEASURE_SIZES \ "Body sizes...\n" \ " header :%1%\n" \ @@ -91,7 +81,18 @@ namespace node { " strong_tx:%11%\n" \ " valid_tx :%12%\n" \ " valid_bk :%13%\n" \ - " address :%14%" + " address :%14%\n" \ + " neutrino :%15%" +#define BN_MEASURE_RECORDS \ + "Table records...\n" \ + " header :%1%\n" \ + " tx :%2%\n" \ + " point :%3%\n" \ + " candidate:%4%\n" \ + " confirmed:%5%\n" \ + " spend :%6%\n" \ + " strong_tx:%7%\n" \ + " address :%8%" #define BN_MEASURE_SLABS \ "Table slabs..." #define BN_MEASURE_SLABS_ROW \ @@ -100,8 +101,8 @@ namespace node { " input :%1%\n" \ " output :%2%\n" \ " seconds :%3%" -#define BN_MEASURE_COLLISION_RATES \ - "Collision rates...\n" \ +#define BN_MEASURE_BUCKETS \ + "Head buckets...\n" \ " header :%1%\n" \ " txs :%2%\n" \ " tx :%3%\n" \ @@ -110,9 +111,10 @@ namespace node { " strong_tx:%6%\n" \ " valid_tx :%7%\n" \ " valid_bk :%8%\n" \ - " address :%9%" -#define BN_MEASURE_BUCKETS \ - "Head buckets...\n" \ + " address :%9%\n" \ + " neutrino :%10%" +#define BN_MEASURE_COLLISION_RATES \ + "Collision rates...\n" \ " header :%1%\n" \ " txs :%2%\n" \ " tx :%3%\n" \ @@ -121,7 +123,8 @@ namespace node { " strong_tx:%6%\n" \ " valid_tx :%7%\n" \ " valid_bk :%8%\n" \ - " address :%9%" + " address :%9%\n" \ + " neutrino :%10%" #define BN_MEASURE_PROGRESS \ "Progress...\n" \ " fork pt :%1%\n" \ diff --git a/include/bitcoin/node/chasers/chaser_preconfirm.hpp b/include/bitcoin/node/chasers/chaser_preconfirm.hpp index 55b0f6f7..d043489e 100644 --- a/include/bitcoin/node/chasers/chaser_preconfirm.hpp +++ b/include/bitcoin/node/chasers/chaser_preconfirm.hpp @@ -49,15 +49,22 @@ class BCN_API chaser_preconfirm virtual void do_bump(height_t height) NOEXCEPT; private: - code validate(const database::header_link& link, - size_t height) const NOEXCEPT; + code validate(const database::header_link& link, size_t height) NOEXCEPT; + + // neutrino + system::hash_digest get_neutrino(size_t height) const NOEXCEPT; + bool update_neutrino(const database::header_link& link) NOEXCEPT; + bool update_neutrino(const database::header_link& link, + const system::chain::block& block) NOEXCEPT; + void update_cache(size_t height) NOEXCEPT; // These are thread safe. const uint64_t initial_subsidy_; const uint32_t subsidy_interval_blocks_; - // This is protected by strand. + // These are protected by strand. size_t validated_{}; + system::hash_digest neutrino_{}; }; } // namespace node diff --git a/src/chasers/chaser_check.cpp b/src/chasers/chaser_check.cpp index 3f923aae..0b1c0119 100644 --- a/src/chasers/chaser_check.cpp +++ b/src/chasers/chaser_check.cpp @@ -39,7 +39,6 @@ using namespace std::placeholders; // TODO: this becomes the block object cache size. constexpr auto maximum = max_size_t; - // Shared pointers required for lifetime in handler parameters. BC_PUSH_WARNING(NO_VALUE_OR_CONST_REF_SHARED_PTR) BC_PUSH_WARNING(SMART_PTR_NOT_NEEDED) diff --git a/src/chasers/chaser_preconfirm.cpp b/src/chasers/chaser_preconfirm.cpp index bd4e98ce..d1e1c208 100644 --- a/src/chasers/chaser_preconfirm.cpp +++ b/src/chasers/chaser_preconfirm.cpp @@ -30,6 +30,7 @@ namespace node { using namespace system; using namespace system::chain; +using namespace system::neutrino; using namespace database; using namespace std::placeholders; @@ -44,7 +45,7 @@ chaser_preconfirm::chaser_preconfirm(full_node& node) NOEXCEPT code chaser_preconfirm::start() NOEXCEPT { - validated_ = archive().get_fork(); + update_cache(archive().get_fork()); return SUBSCRIBE_EVENTS(handle_event, _1, _2, _3); } @@ -97,7 +98,7 @@ void chaser_preconfirm::do_regressed(height_t branch_point) NOEXCEPT // If branch point is at or above last validated there is nothing to do. if (branch_point < validated_) - validated_ = branch_point; + update_cache(branch_point); do_checked(branch_point); } @@ -107,8 +108,7 @@ void chaser_preconfirm::do_disorganized(height_t top) NOEXCEPT BC_ASSERT(stranded()); // Revert to confirmed top as the candidate chain is fully reverted. - validated_ = top; - + update_cache(top); do_checked(top); } @@ -139,6 +139,7 @@ void chaser_preconfirm::do_bump(height_t) NOEXCEPT // Accept/Connect block. // .................................................................... + // neutrino_ is advanced here if successful or bypassed. if (const auto code = validate(link, height)) { if (code == error::validation_bypass || @@ -203,17 +204,16 @@ void chaser_preconfirm::do_bump(height_t) NOEXCEPT } } -// TODO: compute neutrino filter with prevouts populated. -// TODO: neutrino filters are blob per block, must store fk somewhere. -// TODO: neutrino headers are hash per block, must create in order. code chaser_preconfirm::validate(const header_link& link, - size_t height) const NOEXCEPT + size_t height) NOEXCEPT { + code ec{}; const auto& query = archive(); if (is_under_bypass(height) && !query.is_malleable(link)) - return error::validation_bypass; + return update_neutrino(link) ? error::validation_bypass : + error::store_integrity; - auto ec = query.get_block_state(link); + ec = query.get_block_state(link); if (ec == database::error::block_confirmable || ec == database::error::block_unconfirmable || ec == database::error::block_preconfirmable) @@ -238,9 +238,62 @@ code chaser_preconfirm::validate(const header_link& link, {} // work_required }; - return - ec = block.accept(ctx, subsidy_interval_blocks_, initial_subsidy_) ? - ec : block.connect(ctx); + if ((ec = block.accept(ctx, subsidy_interval_blocks_, initial_subsidy_))) + return ec; + + if ((ec = block.connect(ctx))) + return ec; + + return update_neutrino(link, block) ? ec : error::store_integrity; +} + +// neutrino +// ---------------------------------------------------------------------------- + +void chaser_preconfirm::update_cache(size_t height) NOEXCEPT +{ + validated_ = height; + neutrino_ = get_neutrino(validated_); +} + +// Returns null_hash if not found, intended for genesis block. +hash_digest chaser_preconfirm::get_neutrino(size_t height) const NOEXCEPT +{ + hash_digest neutrino{}; + const auto& query = archive(); + if (query.neutrino_enabled()) + query.get_filter_head(neutrino, query.to_candidate(height)); + + return neutrino; +} + +bool chaser_preconfirm::update_neutrino(const header_link& link) NOEXCEPT +{ + const auto& query = archive(); + if (!query.neutrino_enabled()) + return true; + + const auto block_ptr = query.get_block(link); + if (!block_ptr) + return false; + + const auto& block = *block_ptr; + return query.populate(block) && update_neutrino(link, block); +} + +bool chaser_preconfirm::update_neutrino(const header_link& link, + const chain::block& block) NOEXCEPT +{ + auto& query = archive(); + if (!query.neutrino_enabled()) + return true; + + data_chunk filter{}; + if (!compute_filter(filter, block)) + return false; + + neutrino_ = compute_filter_header(neutrino_, filter); + return query.set_filter(link, neutrino_, filter); } BC_POP_WARNING() diff --git a/src/parser.cpp b/src/parser.cpp index 5ae69905..7e27cf74 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -116,14 +116,14 @@ parser::parser(system::chain::selection context) NOEXCEPT configured.database.address_size = 29'390'853'398; configured.database.address_rate = 5; + configured.database.neutrino_buckets = 524'493; + configured.database.neutrino_size = 1'000'000'000; + configured.database.neutrino_rate = 5; + configured.database.buffer_buckets = 100; configured.database.buffer_size = 1; configured.database.buffer_rate = 5; - configured.database.neutrino_buckets = 100; - configured.database.neutrino_size = 1; - configured.database.neutrino_rate = 5; - configured.database.bootstrap_size = 1; configured.database.bootstrap_rate = 5; } @@ -837,6 +837,23 @@ options_metadata parser::load_settings() THROWS "The percentage expansion of the address table body, defaults to '5'." ) + /* neutrino */ + ( + "database.neutrino_buckets", + value(&configured.database.neutrino_buckets), + "The number of buckets in the neutrino table head, defaults to '524493' (0 disables)." + ) + ( + "database.neutrino_size", + value(&configured.database.neutrino_size), + "The minimum allocation of the neutrino table body, defaults to '1000000000'." + ) + ( + "database.neutrino_rate", + value(&configured.database.neutrino_rate), + "The percentage expansion of the neutrino table body, defaults to '5'." + ) + /* [node] */ ( "node.headers_first",