Add a rust-based backup over-REST block downloader

src/Makefile.am

@@ -627,7 +627,7 @@ bitcoin_wallet_LDADD = \
   $(LIBSECP256K1) \
   $(LIBUNIVALUE)
 
-bitcoin_wallet_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(ZMQ_LIBS) $(LIBBITCOIN_RUST_LIBS) $(DL_LIBS)
+bitcoin_wallet_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(ZMQ_LIBS) $(DL_LIBS)
 #
 
 # bitcoinconsensus library #
@@ -659,9 +659,10 @@ RUSTY_DIST += rusty/Cargo.toml
 RUSTY_DIST += rusty/src/lib.rs
 
 if ENABLE_RUSTY
-RUSTY_H_NAMESPACE = rust_hello_world_example
+RUSTY_H_NAMESPACE = rust_block_fetch
 RUSTY_H = rusty/out/rusty.hpp
 LIBBITCOIN_RUST_LIBS += $(LIBBITCOIN_RUSTY)
+libbitcoin_server_a_SOURCES += rusty/src/cpp_bridge.cpp
 
 $(RUSTY_H): | $(LIBBITCOIN_RUSTY)
 $(libbitcoin_server_a_OBJECTS) : $(RUSTY_H)

src/init.cpp

@@ -344,10 +344,6 @@ static void OnRPCStopped()
 
 void SetupServerArgs()
 {
-#if ENABLE_RUSTY
-    assert(rust_hello_world_example::RUST_CONSTANT == 43);
-    rust_hello_world_example::hello_world();
-#endif
     SetupHelpOptions(gArgs);
     gArgs.AddArg("-help-debug", "Print help message with debugging options and exit", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); // server-only for now
 
@@ -375,6 +371,9 @@ void SetupServerArgs()
 #endif
     gArgs.AddArg("-blockreconstructionextratxn=<n>", strprintf("Extra transactions to keep in memory for compact block reconstructions (default: %u)", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     gArgs.AddArg("-blocksonly", strprintf("Whether to reject transactions from network peers. Transactions from the wallet, RPC and relay whitelisted inbound peers are not affected. (default: %u)", DEFAULT_BLOCKSONLY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+#if ENABLE_RUSTY
+    gArgs.AddArg("-blockfetchrest=<uri>", "A REST endpoint from which to fetch blocks. Acts as a redundant backup for P2P connectivity", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+#endif
     gArgs.AddArg("-conf=<file>", strprintf("Specify configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     gArgs.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     gArgs.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS);
@@ -1831,5 +1830,13 @@ bool AppInitMain(InitInterfaces& interfaces)
         g_banman->DumpBanlist();
     }, DUMP_BANS_INTERVAL * 1000);
 
+    // ********************************************************* Step 14: kick off backup block downloaders
+
+#if ENABLE_RUSTY
+    for (const std::string& uri : gArgs.GetArgs("-blockfetchrest")) {
+        rust_block_fetch::init_fetch_rest_blocks((const unsigned char*)uri.c_str());
+    }
+#endif
+
     return true;
 }

src/rusty/Cargo.toml

@@ -7,8 +7,22 @@ build = "build.rs"
 
 [dependencies]
 
+[dependencies.bitcoin]
+version = "0.20"
+default-features = false
+
 [lib]
 crate-type = ["staticlib"]
 
 [build-dependencies]
 cbindgen = {git="https://github.com/eqrion/cbindgen", rev="e19526e00b3fe6921b881682147a1fe5d6b64124"} # tag = v0.9.0
+
+# Always do overflow checks and allow panics to unwind so that even if
+# we have a crash bug we don't take down the rest of Bitcoin Core with us
+[profile.release]
+panic = "unwind"
+overflow-checks = true
+
+[profile.dev]
+panic = "unwind"
+overflow-checks = true

src/rusty/src/bridge.rs

@@ -0,0 +1,167 @@
+use std::ffi::c_void;
+extern "C" {
+    pub fn rusty_IsInitialBlockDownload() -> bool;
+    pub fn rusty_ShutdownRequested() -> bool;
+    pub fn rusty_ProcessNewBlock(blockdata: *const u8, blockdatalen: usize);
+
+    /// Connects count headers serialized in a block of memory, each stride bytes from each other.
+    /// Returns the last header which was connected, if any (or NULL).
+    fn rusty_ConnectHeaders(headers: *const u8, stride: usize, count: usize) -> *const c_void;
+
+    // Utilities to work with CBlockIndex pointers. Wrapped in a safe wrapper below.
+
+    /// Gets a CBlockIndex* pointer (casted to a c_void) representing the current tip.
+    /// Guaranteed to never be NULL (but may be genesis)
+    fn rusty_GetChainTip() -> *const c_void;
+
+    /// Gets a CBlockIndex* pointer (casted to a c_void) representing the genesis block.
+    /// Guaranteed to never be NULL
+    fn rusty_GetGenesisIndex() -> *const c_void;
+
+    #[allow(dead_code)]
+    /// Finds a CBlockIndex* for a given block hash, or NULL if none is found
+    fn rusty_HashToIndex(hash: *const u8) -> *const c_void;
+
+    #[allow(dead_code)]
+    /// Gets the height of a given CBlockIndex* pointer
+    fn rusty_IndexToHeight(index: *const c_void) -> i32;
+
+    /// Gets the hash of a given CBlockIndex* pointer
+    fn rusty_IndexToHash(index: *const c_void) -> *const u8;
+}
+
+#[allow(dead_code)]
+/// Connects the given array of (sorted, in chain order) headers (in serialized, 80-byte form).
+/// Returns the last header which was connected, if any.
+pub fn connect_headers(headers: &[[u8; 80]]) -> Option<BlockIndex> {
+    if headers.is_empty() { return None; }
+    let first_header = headers[0].as_ptr();
+    let index = if headers.len() == 1 {
+        unsafe { rusty_ConnectHeaders(first_header, 80, 1) }
+    } else {
+        let second_header = headers[1].as_ptr();
+        let stride = second_header as usize - first_header as usize;
+        unsafe { rusty_ConnectHeaders(first_header, stride, headers.len()) }
+    };
+    if index.is_null() { None } else { Some(BlockIndex { index }) }
+}
+
+/// Connects the given array of (sorted, in chain order) headers (in serialized, 80-byte form).
+/// Returns the last header which was connected, if any.
+pub fn connect_headers_flat_bytes(headers: &[u8]) -> Option<BlockIndex> {
+    if headers.len() % 80 != 0 { return None; }
+    if headers.is_empty() { return None; }
+    let index = unsafe { rusty_ConnectHeaders(headers.as_ptr(), 80, headers.len() / 80) };
+    if index.is_null() { None } else { Some(BlockIndex { index }) }
+}
+
+#[derive(PartialEq, Clone, Copy)]
+pub struct BlockIndex {
+    index: *const c_void,
+}
+
+impl BlockIndex {
+    pub fn tip() -> Self {
+        Self {
+            index: unsafe { rusty_GetChainTip() },
+        }
+    }
+
+    #[allow(dead_code)]
+    pub fn get_from_hash(hash: &[u8; 32]) -> Option<Self> {
+        let index = unsafe { rusty_HashToIndex(hash.as_ptr()) };
+        if index.is_null() {
+            None
+        } else {
+            Some(Self { index })
+        }
+    }
+
+    pub fn genesis() -> Self {
+        Self {
+            index: unsafe { rusty_GetGenesisIndex() },
+        }
+    }
+
+    #[allow(dead_code)]
+    pub fn height(&self) -> i32 {
+        unsafe { rusty_IndexToHeight(self.index) }
+    }
+
+    pub fn hash(&self) -> [u8; 32] {
+        let hashptr = unsafe { rusty_IndexToHash(self.index) };
+        let mut res = [0u8; 32];
+        unsafe { std::ptr::copy(hashptr, (&mut res).as_mut_ptr(), 32) };
+        res
+    }
+
+    /// Gets the hex formatted hash of this block, in byte-revered order (ie starting with the PoW
+    /// 0s, as is commonly used in Bitcoin APIs).
+    pub fn hash_hex(&self) -> String {
+        let hash_bytes = self.hash();
+        let mut res = String::with_capacity(64);
+        for b in hash_bytes.iter().rev() {
+            res.push(std::char::from_digit((b >> 4) as u32, 16).unwrap());
+            res.push(std::char::from_digit((b & 0x0f) as u32, 16).unwrap());
+        }
+        res
+    }
+}
+
+extern "C" {
+    // Utilities to work with BlockProviderState objects. Wrapped in a safe wrapper below.
+
+    /// Creates a new BlockProviderState with a given current best CBlockIndex*.
+    /// Don't forget to de-allocate!
+    fn rusty_ProviderStateInit(blockindex: *const c_void) -> *mut c_void;
+    /// De-allocates a BlockProviderState.
+    fn rusty_ProviderStateFree(provider_state: *mut c_void);
+
+    /// Sets the current best available CBlockIndex* for the given provider state.
+    fn rusty_ProviderStateSetBest(provider_state: *mut c_void, blockindex: *const c_void);
+
+    /// Gets the next CBlockIndex* a given provider should download, or NULL
+    fn rusty_ProviderStateGetNextDownloads(providerindexvoid: *mut c_void, has_witness: bool) -> *const c_void;
+}
+
+pub struct BlockProviderState {
+    // TODO: We should be smarter than to keep a copy of the current best pointer twice, but
+    // crossing the FFI boundary just to look it up again sucks.
+    current_best: BlockIndex,
+    state: *mut c_void,
+}
+impl BlockProviderState {
+    /// Initializes block provider state with a given current best header.
+    /// Note that you can use a guess on the current best that moves backwards as you discover the
+    /// providers' true chain state, though for effeciency you should try to avoid calling
+    /// get_next_block_to_download in such a state.
+    pub fn new_with_current_best(blockindex: BlockIndex) -> Self {
+        Self {
+            current_best: blockindex,
+            state: unsafe { rusty_ProviderStateInit(blockindex.index) }
+        }
+    }
+
+    /// Sets the current best available blockindex to the given one on this state.
+    pub fn set_current_best(&mut self, blockindex: BlockIndex) {
+        self.current_best = blockindex;
+        unsafe { rusty_ProviderStateSetBest(self.state, blockindex.index) };
+    }
+
+    /// Gets the current best available blockindex as provided previously by set_current_best or
+    /// new_with_current_best.
+    pub fn get_current_best(&self) -> BlockIndex {
+        self.current_best
+    }
+
+    /// Gets the BlockIndex representing the next block which should be downloaded, if any.
+    pub fn get_next_block_to_download(&mut self, has_witness: bool) -> Option<BlockIndex> {
+        let index = unsafe { rusty_ProviderStateGetNextDownloads(self.state, has_witness) };
+        if index.is_null() { None } else { Some(BlockIndex { index }) }
+    }
+}
+impl Drop for BlockProviderState {
+    fn drop(&mut self) {
+        unsafe { rusty_ProviderStateFree(self.state) };
+    }
+}

src/rusty/src/cpp_bridge.cpp

@@ -0,0 +1,142 @@
+#include <chainparams.h>
+#include <validation.h>
+#include <shutdown.h>
+#include <serialize.h>
+#include <consensus/validation.h>
+
+/** A class that deserializes a single thing one time. */
+class InputStream
+{
+public:
+    InputStream(int nTypeIn, int nVersionIn, const unsigned char *data, size_t datalen) :
+    m_type(nTypeIn),
+    m_version(nVersionIn),
+    m_data(data),
+    m_remaining(datalen)
+    {}
+
+    void read(char* pch, size_t nSize)
+    {
+        if (nSize > m_remaining)
+            throw std::ios_base::failure(std::string(__func__) + ": end of data");
+
+        if (pch == nullptr)
+            throw std::ios_base::failure(std::string(__func__) + ": bad destination buffer");
+
+        if (m_data == nullptr)
+            throw std::ios_base::failure(std::string(__func__) + ": bad source buffer");
+
+        memcpy(pch, m_data, nSize);
+        m_remaining -= nSize;
+        m_data += nSize;
+    }
+
+    template<typename T>
+    InputStream& operator>>(T&& obj)
+    {
+        ::Unserialize(*this, obj);
+        return *this;
+    }
+
+    int GetVersion() const { return m_version; }
+    int GetType() const { return m_type; }
+private:
+    const int m_type;
+    const int m_version;
+    const unsigned char* m_data;
+    size_t m_remaining;
+};
+
+extern "C" {
+
+bool rusty_IsInitialBlockDownload() {
+    return ::ChainstateActive().IsInitialBlockDownload();
+}
+
+bool rusty_ShutdownRequested() {
+    return ShutdownRequested();
+}
+
+void rusty_ProcessNewBlock(const uint8_t* blockdata, size_t blocklen) {
+    CBlock block;
+    try {
+        InputStream(SER_NETWORK, PROTOCOL_VERSION, blockdata, blocklen) >> block;
+    } catch (...) {}
+    ProcessNewBlock(::Params(), std::make_shared<const CBlock>(block), true, nullptr);
+}
+
+const void* rusty_ConnectHeaders(const uint8_t* headers_data, size_t stride, size_t count) {
+    std::vector<CBlockHeader> headers;
+    for(size_t i = 0; i < count; i++) {
+        CBlockHeader header;
+        try {
+            InputStream(SER_NETWORK, PROTOCOL_VERSION, headers_data + (stride * i), 80) >> header;
+        } catch (...) {}
+        headers.push_back(header);
+    }
+    CValidationState state_dummy;
+    const CBlockIndex* last_index = nullptr;
+    ProcessNewBlockHeaders(headers, state_dummy, ::Params(), &last_index);
+    return last_index;
+}
+
+const void* rusty_GetChainTip() {
+    LOCK(cs_main);
+    const CBlockIndex* tip = ::ChainActive().Tip();
+    assert(tip != nullptr);
+    return tip;
+}
+
+const void* rusty_GetGenesisIndex() {
+    LOCK(cs_main);
+    const CBlockIndex* genesis = ::ChainActive().Genesis();
+    assert(genesis != nullptr);
+    return genesis;
+}
+
+const void* rusty_HashToIndex(const uint8_t* hash_ptr) {
+    uint256 hash;
+    memcpy(hash.begin(), hash_ptr, 32);
+    LOCK(cs_main);
+    return LookupBlockIndex(hash);
+}
+
+int32_t rusty_IndexToHeight(const void* pindexvoid) {
+    const CBlockIndex *pindex = (const CBlockIndex*) pindexvoid;
+    assert(pindex != nullptr);
+    return pindex->nHeight;
+}
+
+const uint8_t* rusty_IndexToHash(const void* pindexvoid) {
+    const CBlockIndex *pindex = (const CBlockIndex*) pindexvoid;
+    assert(pindex != nullptr);
+    return pindex->phashBlock->begin();
+}
+
+void* rusty_ProviderStateInit(const void* pindexvoid) {
+    const CBlockIndex *pindex = (const CBlockIndex*) pindexvoid;
+    BlockProviderState* state = new BlockProviderState;
+    state->m_best_known_block = pindex;
+    return state;
+}
+
+void rusty_ProviderStateFree(void* providerindexvoid) {
+    BlockProviderState* state = (BlockProviderState*) providerindexvoid;
+    delete state;
+}
+
+void rusty_ProviderStateSetBest(void* providerindexvoid, const void* pindexvoid) {
+    BlockProviderState* state = (BlockProviderState*) providerindexvoid;
+    const CBlockIndex *pindex = (const CBlockIndex*) pindexvoid;
+    state->m_best_known_block = pindex;
+}
+
+const void* rusty_ProviderStateGetNextDownloads(void* providerindexvoid, bool has_witness) {
+    BlockProviderState* state = (BlockProviderState*) providerindexvoid;
+    std::vector<const CBlockIndex*> blocks;
+    LOCK(cs_main);
+    state->FindNextBlocksToDownload(has_witness, 1, blocks, ::Params().GetConsensus(), [] (const uint256& block_hash) { return false; });
+    return blocks.empty() ? nullptr : blocks[0];
+}
+
+}