From ef7c8228fd5cf45526518ae2bd5ebdd483e65525 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Thu, 28 Nov 2019 22:39:12 -0500 Subject: [PATCH 1/2] Expose block filters over REST. This adds a new rest endpoint: /rest/blockfilter/filtertype/requesttype/blockhash (eg /rest/blockfilter/basic/header/000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f.hex) which exposes either the filter "header" or the filter data itself. Most of the code is cribbed from the equivalent RPC. --- src/rest.cpp | 215 +++++++++++++++++++++++++++++- test/functional/interface_rest.py | 7 +- 2 files changed, 217 insertions(+), 5 deletions(-) diff --git a/src/rest.cpp b/src/rest.cpp index f6e34c2d81d8e..18ae3506306ad 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -3,10 +3,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include #include #include #include #include +#include #include #include #include @@ -30,6 +32,7 @@ #include static const size_t MAX_GETUTXOS_OUTPOINTS = 15; //allow a max of 15 outpoints to be queried at once +static constexpr unsigned int MAX_REST_HEADERS_RESULTS = 2000; enum class RetFormat { UNDEF, @@ -190,8 +193,8 @@ static bool rest_headers(const std::any& context, return RESTERR(req, HTTP_BAD_REQUEST, "No header count specified. Use /rest/headers//.."); const auto parsed_count{ToIntegral(path[0])}; - if (!parsed_count.has_value() || *parsed_count < 1 || *parsed_count > 2000) { - return RESTERR(req, HTTP_BAD_REQUEST, "Header count out of range: " + path[0]); + if (!parsed_count.has_value() || *parsed_count < 1 || *parsed_count > MAX_REST_HEADERS_RESULTS) { + return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Header count out of acceptable range (1-%u): %s", MAX_REST_HEADERS_RESULTS, path[0])); } std::string hashStr = path[1]; @@ -254,7 +257,7 @@ static bool rest_headers(const std::any& context, return true; } default: { - return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: .bin, .hex, .json)"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); } } } @@ -337,6 +340,210 @@ static bool rest_block_notxdetails(const std::any& context, HTTPRequest* req, co return rest_block(context, req, strURIPart, false); } + +static bool rest_filter_header(const std::any& context, HTTPRequest* req, const std::string& strURIPart) +{ + if (!CheckWarmup(req)) + return false; + std::string param; + const RetFormat rf = ParseDataFormat(param, strURIPart); + + std::vector uri_parts; + boost::split(uri_parts, param, boost::is_any_of("/")); + if (uri_parts.size() != 3) { + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/blockfilterheaders///"); + } + + uint256 block_hash; + if (!ParseHashStr(uri_parts[2], block_hash)) { + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + uri_parts[2]); + } + + BlockFilterType filtertype; + if (!BlockFilterTypeByName(uri_parts[0], filtertype)) { + return RESTERR(req, HTTP_BAD_REQUEST, "Unknown filtertype " + uri_parts[0]); + } + + BlockFilterIndex* index = GetBlockFilterIndex(filtertype); + if (!index) { + return RESTERR(req, HTTP_BAD_REQUEST, "Index is not enabled for filtertype " + uri_parts[0]); + } + + const auto parsed_count{ToIntegral(uri_parts[1])}; + if (!parsed_count.has_value() || *parsed_count < 1 || *parsed_count > MAX_REST_HEADERS_RESULTS) { + return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Header count out of acceptable range (1-%u): %s", MAX_REST_HEADERS_RESULTS, uri_parts[1])); + } + + std::vector headers; + headers.reserve(*parsed_count); + { + ChainstateManager* maybe_chainman = GetChainman(context, req); + if (!maybe_chainman) return false; + ChainstateManager& chainman = *maybe_chainman; + LOCK(cs_main); + CChain& active_chain = chainman.ActiveChain(); + const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(block_hash); + while (pindex != nullptr && active_chain.Contains(pindex)) { + headers.push_back(pindex); + if (headers.size() == *parsed_count) + break; + pindex = active_chain.Next(pindex); + } + } + + bool index_ready = index->BlockUntilSyncedToCurrentChain(); + + std::vector filter_headers; + filter_headers.reserve(*parsed_count); + for (const CBlockIndex *pindex : headers) { + uint256 filter_header; + if (!index->LookupFilterHeader(pindex, filter_header)) { + std::string errmsg = "Filter not found."; + + if (!index_ready) { + errmsg += " Block filters are still in the process of being indexed."; + } else { + errmsg += " This error is unexpected and indicates index corruption."; + } + + return RESTERR(req, HTTP_NOT_FOUND, errmsg); + } + filter_headers.push_back(filter_header); + } + + switch (rf) { + case RetFormat::BINARY: { + CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); + for (const uint256& header : filter_headers) { + ssHeader << header; + } + + std::string binaryHeader = ssHeader.str(); + req->WriteHeader("Content-Type", "application/octet-stream"); + req->WriteReply(HTTP_OK, binaryHeader); + return true; + } + case RetFormat::HEX: { + CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); + for (const uint256& header : filter_headers) { + ssHeader << header; + } + + std::string strHex = HexStr(ssHeader) + "\n"; + req->WriteHeader("Content-Type", "text/plain"); + req->WriteReply(HTTP_OK, strHex); + return true; + } + case RetFormat::JSON: { + UniValue jsonHeaders(UniValue::VARR); + for (const uint256& header : filter_headers) { + jsonHeaders.push_back(header.GetHex()); + } + + std::string strJSON = jsonHeaders.write() + "\n"; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); + return true; + } + default: { + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + } + } +} + +static bool rest_block_filter(const std::any& context, HTTPRequest* req, const std::string& strURIPart) +{ + if (!CheckWarmup(req)) + return false; + std::string param; + const RetFormat rf = ParseDataFormat(param, strURIPart); + + //request is sent over URI scheme /rest/blockfilter/filtertype/blockhash + std::vector uri_parts; + boost::split(uri_parts, param, boost::is_any_of("/")); + if (uri_parts.size() != 2) { + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/blockfilter//"); + } + + uint256 block_hash; + if (!ParseHashStr(uri_parts[1], block_hash)) { + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + uri_parts[1]); + } + + BlockFilterType filtertype; + if (!BlockFilterTypeByName(uri_parts[0], filtertype)) { + return RESTERR(req, HTTP_BAD_REQUEST, "Unknown filtertype " + uri_parts[0]); + } + + BlockFilterIndex* index = GetBlockFilterIndex(filtertype); + if (!index) { + return RESTERR(req, HTTP_BAD_REQUEST, "Index is not enabled for filtertype " + uri_parts[0]); + } + + const CBlockIndex* block_index; + bool block_was_connected; + { + ChainstateManager* maybe_chainman = GetChainman(context, req); + if (!maybe_chainman) return false; + ChainstateManager& chainman = *maybe_chainman; + LOCK(cs_main); + block_index = chainman.m_blockman.LookupBlockIndex(block_hash); + if (!block_index) { + return RESTERR(req, HTTP_NOT_FOUND, uri_parts[1] + " not found"); + } + block_was_connected = block_index->IsValid(BLOCK_VALID_SCRIPTS); + } + + bool index_ready = index->BlockUntilSyncedToCurrentChain(); + + BlockFilter filter; + if (!index->LookupFilter(block_index, filter)) { + std::string errmsg = "Filter not found."; + + if (!block_was_connected) { + errmsg += " Block was not connected to active chain."; + } else if (!index_ready) { + errmsg += " Block filters are still in the process of being indexed."; + } else { + errmsg += " This error is unexpected and indicates index corruption."; + } + + return RESTERR(req, HTTP_NOT_FOUND, errmsg); + } + + switch (rf) { + case RetFormat::BINARY: { + CDataStream ssResp(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); + ssResp << filter; + + std::string binaryResp = ssResp.str(); + req->WriteHeader("Content-Type", "application/octet-stream"); + req->WriteReply(HTTP_OK, binaryResp); + return true; + } + case RetFormat::HEX: { + CDataStream ssResp(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); + ssResp << filter; + + std::string strHex = HexStr(ssResp) + "\n"; + req->WriteHeader("Content-Type", "text/plain"); + req->WriteReply(HTTP_OK, strHex); + return true; + } + case RetFormat::JSON: { + UniValue ret(UniValue::VOBJ); + ret.pushKV("filter", HexStr(filter.GetEncodedFilter())); + std::string strJSON = ret.write() + "\n"; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); + return true; + } + default: { + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + } + } +} + // A bit of a hack - dependency on a function defined in rpc/blockchain.cpp RPCHelpMan getblockchaininfo(); @@ -717,6 +924,8 @@ static const struct { {"/rest/tx/", rest_tx}, {"/rest/block/notxdetails/", rest_block_notxdetails}, {"/rest/block/", rest_block_extended}, + {"/rest/blockfilter/", rest_block_filter}, + {"/rest/blockfilterheaders/", rest_filter_header}, {"/rest/chaininfo", rest_chaininfo}, {"/rest/mempool/info", rest_mempool_info}, {"/rest/mempool/contents", rest_mempool_contents}, diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index adc33bd9df420..3490d8808ccdb 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -41,7 +41,7 @@ class RESTTest (BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 - self.extra_args = [["-rest"], []] + self.extra_args = [["-rest", "-blockfilterindex=1"], []] self.supports_cli = False def skip_test_if_missing_module(self): @@ -278,11 +278,14 @@ def run_test(self): self.sync_all() json_obj = self.test_rest_request(f"/headers/5/{bb_hash}") assert_equal(len(json_obj), 5) # now we should have 5 header objects + json_obj = self.test_rest_request(f"/blockfilterheaders/basic/5/{bb_hash}") + assert_equal(len(json_obj), 5) # now we should have 5 filter header objects + self.test_rest_request(f"/blockfilter/basic/{bb_hash}", req_type=ReqType.BIN, ret_type=RetType.OBJ) # Test number parsing for num in ['5a', '-5', '0', '2001', '99999999999999999999999999999999999']: assert_equal( - bytes(f'Header count out of range: {num}\r\n', 'ascii'), + bytes(f'Header count out of acceptable range (1-2000): {num}\r\n', 'ascii'), self.test_rest_request(f"/headers/{num}/{bb_hash}", ret_type=RetType.BYTES, status=400), ) From 2b64fa3251ac5ff4b4d174f1f0be7226490dce87 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Tue, 12 Oct 2021 02:56:48 +0000 Subject: [PATCH 2/2] Update REST docs with new accessors --- doc/REST-interface.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 8b281acca7732..51a73b89fcecf 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -52,6 +52,20 @@ With the /notxdetails/ option JSON response will only contain the transaction ha Given a block hash: returns amount of blockheaders in upward direction. Returns empty if the block doesn't exist or it isn't in the active chain. +#### Blockfilter Headers +`GET /rest/blockfilterheaders///.` + +Given a block hash: returns amount of blockfilter headers in upward +direction for the filter type . +Returns empty if the block doesn't exist or it isn't in the active chain. + +#### Blockfilters +`GET /rest/blockfilter//.` + +Given a block hash: returns the block filter of the given block of type +. +Responds with 404 if the block doesn't exist. + #### Blockhash by height `GET /rest/blockhashbyheight/.`