Skip to content

Commit

Permalink
Merge #17631: Expose block filters over REST
Browse files Browse the repository at this point in the history 
2b64fa3 Update REST docs with new accessors (Matt Corallo)
ef7c822 Expose block filters over REST. (Matt Corallo)

Pull request description:

  This adds a new rest endpoint:
  /rest/blockfilter/filtertype/requesttype/blockhash (eg
  /rest/blockfilter/basic/header/000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f.hex)
  which exposes either the filter "header" or the filter data itself.
  Most of the code is cribbed from the equivalent RPC.

  You can test it at http://bitcoin-rest.bitcoin.ninja/rest//blockfilter/basic/header/000000005b7a58a939b2636f61fa4ddd62258c5fed57667a35d23f2334c4f86d.hex

ACKs for top commit:
  dergoegge:
    ACK 2b64fa3 - Adding blockfilters to the REST interface is analogous to serving other public data such as transactions or blocks.

Tree-SHA512: d487bc694266375c94d6fcf2e9d788a8a42a3b94e8d3290e46335a64cbcde55084ce5ea6119b79a4065888d94d7c3ae25a59a901fa46e3711f0eb296add12696
  • Loading branch information
MarcoFalke committed Dec 20, 2021
2 parents 8eeb4e9 + 2b64fa3 commit 70d6a09
Show file tree
Hide file tree
Showing 3 changed files with 231 additions and 5 deletions.
14 changes: 14 additions & 0 deletions doc/REST-interface.md
View file
Expand Up @@ -52,6 +52,20 @@ With the /notxdetails/ option JSON response will only contain the transaction ha
Given a block hash: returns <COUNT> amount of blockheaders in upward direction.
Returns empty if the block doesn't exist or it isn't in the active chain.

#### Blockfilter Headers
`GET /rest/blockfilterheaders/<FILTERTYPE>/<COUNT>/<BLOCK-HASH>.<bin|hex|json>`

Given a block hash: returns <COUNT> amount of blockfilter headers in upward
direction for the filter type <FILTERTYPE>.
Returns empty if the block doesn't exist or it isn't in the active chain.

#### Blockfilters
`GET /rest/blockfilter/<FILTERTYPE>/<BLOCK-HASH>.<bin|hex|json>`

Given a block hash: returns the block filter of the given block of type
<FILTERTYPE>.
Responds with 404 if the block doesn't exist.

#### Blockhash by height
`GET /rest/blockhashbyheight/<HEIGHT>.<bin|hex|json>`

Expand Down
215 changes: 212 additions & 3 deletions src/rest.cpp
View file
Expand Up @@ -3,10 +3,12 @@
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <blockfilter.h>
#include <chain.h>
#include <chainparams.h>
#include <core_io.h>
#include <httpserver.h>
#include <index/blockfilterindex.h>
#include <index/txindex.h>
#include <node/blockstorage.h>
#include <node/context.h>
Expand All @@ -31,6 +33,7 @@
#include <univalue.h>

static const size_t MAX_GETUTXOS_OUTPOINTS = 15; //allow a max of 15 outpoints to be queried at once
static constexpr unsigned int MAX_REST_HEADERS_RESULTS = 2000;

enum class RetFormat {
UNDEF,
Expand Down Expand Up @@ -191,8 +194,8 @@ static bool rest_headers(const std::any& context,
return RESTERR(req, HTTP_BAD_REQUEST, "No header count specified. Use /rest/headers/<count>/<hash>.<ext>.");

const auto parsed_count{ToIntegral<size_t>(path[0])};
if (!parsed_count.has_value() || *parsed_count < 1 || *parsed_count > 2000) {
return RESTERR(req, HTTP_BAD_REQUEST, "Header count out of range: " + path[0]);
if (!parsed_count.has_value() || *parsed_count < 1 || *parsed_count > MAX_REST_HEADERS_RESULTS) {
return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Header count out of acceptable range (1-%u): %s", MAX_REST_HEADERS_RESULTS, path[0]));
}

std::string hashStr = path[1];
Expand Down Expand Up @@ -255,7 +258,7 @@ static bool rest_headers(const std::any& context,
return true;
}
default: {
return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: .bin, .hex, .json)");
return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")");
}
}
}
Expand Down Expand Up @@ -338,6 +341,210 @@ static bool rest_block_notxdetails(const std::any& context, HTTPRequest* req, co
return rest_block(context, req, strURIPart, TxVerbosity::SHOW_TXID);
}


static bool rest_filter_header(const std::any& context, HTTPRequest* req, const std::string& strURIPart)
{
if (!CheckWarmup(req))
return false;
std::string param;
const RetFormat rf = ParseDataFormat(param, strURIPart);

std::vector<std::string> uri_parts;
boost::split(uri_parts, param, boost::is_any_of("/"));
if (uri_parts.size() != 3) {
return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/blockfilterheaders/<filtertype>/<count>/<blockhash>");
}

uint256 block_hash;
if (!ParseHashStr(uri_parts[2], block_hash)) {
return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + uri_parts[2]);
}

BlockFilterType filtertype;
if (!BlockFilterTypeByName(uri_parts[0], filtertype)) {
return RESTERR(req, HTTP_BAD_REQUEST, "Unknown filtertype " + uri_parts[0]);
}

BlockFilterIndex* index = GetBlockFilterIndex(filtertype);
if (!index) {
return RESTERR(req, HTTP_BAD_REQUEST, "Index is not enabled for filtertype " + uri_parts[0]);
}

const auto parsed_count{ToIntegral<size_t>(uri_parts[1])};
if (!parsed_count.has_value() || *parsed_count < 1 || *parsed_count > MAX_REST_HEADERS_RESULTS) {
return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Header count out of acceptable range (1-%u): %s", MAX_REST_HEADERS_RESULTS, uri_parts[1]));
}

std::vector<const CBlockIndex *> headers;
headers.reserve(*parsed_count);
{
ChainstateManager* maybe_chainman = GetChainman(context, req);
if (!maybe_chainman) return false;
ChainstateManager& chainman = *maybe_chainman;
LOCK(cs_main);
CChain& active_chain = chainman.ActiveChain();
const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(block_hash);
while (pindex != nullptr && active_chain.Contains(pindex)) {
headers.push_back(pindex);
if (headers.size() == *parsed_count)
break;
pindex = active_chain.Next(pindex);
}
}

bool index_ready = index->BlockUntilSyncedToCurrentChain();

std::vector<uint256> filter_headers;
filter_headers.reserve(*parsed_count);
for (const CBlockIndex *pindex : headers) {
uint256 filter_header;
if (!index->LookupFilterHeader(pindex, filter_header)) {
std::string errmsg = "Filter not found.";

if (!index_ready) {
errmsg += " Block filters are still in the process of being indexed.";
} else {
errmsg += " This error is unexpected and indicates index corruption.";
}

return RESTERR(req, HTTP_NOT_FOUND, errmsg);
}
filter_headers.push_back(filter_header);
}

switch (rf) {
case RetFormat::BINARY: {
CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags());
for (const uint256& header : filter_headers) {
ssHeader << header;
}

std::string binaryHeader = ssHeader.str();
req->WriteHeader("Content-Type", "application/octet-stream");
req->WriteReply(HTTP_OK, binaryHeader);
return true;
}
case RetFormat::HEX: {
CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags());
for (const uint256& header : filter_headers) {
ssHeader << header;
}

std::string strHex = HexStr(ssHeader) + "\n";
req->WriteHeader("Content-Type", "text/plain");
req->WriteReply(HTTP_OK, strHex);
return true;
}
case RetFormat::JSON: {
UniValue jsonHeaders(UniValue::VARR);
for (const uint256& header : filter_headers) {
jsonHeaders.push_back(header.GetHex());
}

std::string strJSON = jsonHeaders.write() + "\n";
req->WriteHeader("Content-Type", "application/json");
req->WriteReply(HTTP_OK, strJSON);
return true;
}
default: {
return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")");
}
}
}

static bool rest_block_filter(const std::any& context, HTTPRequest* req, const std::string& strURIPart)
{
if (!CheckWarmup(req))
return false;
std::string param;
const RetFormat rf = ParseDataFormat(param, strURIPart);

//request is sent over URI scheme /rest/blockfilter/filtertype/blockhash
std::vector<std::string> uri_parts;
boost::split(uri_parts, param, boost::is_any_of("/"));
if (uri_parts.size() != 2) {
return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/blockfilter/<filtertype>/<blockhash>");
}

uint256 block_hash;
if (!ParseHashStr(uri_parts[1], block_hash)) {
return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + uri_parts[1]);
}

BlockFilterType filtertype;
if (!BlockFilterTypeByName(uri_parts[0], filtertype)) {
return RESTERR(req, HTTP_BAD_REQUEST, "Unknown filtertype " + uri_parts[0]);
}

BlockFilterIndex* index = GetBlockFilterIndex(filtertype);
if (!index) {
return RESTERR(req, HTTP_BAD_REQUEST, "Index is not enabled for filtertype " + uri_parts[0]);
}

const CBlockIndex* block_index;
bool block_was_connected;
{
ChainstateManager* maybe_chainman = GetChainman(context, req);
if (!maybe_chainman) return false;
ChainstateManager& chainman = *maybe_chainman;
LOCK(cs_main);
block_index = chainman.m_blockman.LookupBlockIndex(block_hash);
if (!block_index) {
return RESTERR(req, HTTP_NOT_FOUND, uri_parts[1] + " not found");
}
block_was_connected = block_index->IsValid(BLOCK_VALID_SCRIPTS);
}

bool index_ready = index->BlockUntilSyncedToCurrentChain();

BlockFilter filter;
if (!index->LookupFilter(block_index, filter)) {
std::string errmsg = "Filter not found.";

if (!block_was_connected) {
errmsg += " Block was not connected to active chain.";
} else if (!index_ready) {
errmsg += " Block filters are still in the process of being indexed.";
} else {
errmsg += " This error is unexpected and indicates index corruption.";
}

return RESTERR(req, HTTP_NOT_FOUND, errmsg);
}

switch (rf) {
case RetFormat::BINARY: {
CDataStream ssResp(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags());
ssResp << filter;

std::string binaryResp = ssResp.str();
req->WriteHeader("Content-Type", "application/octet-stream");
req->WriteReply(HTTP_OK, binaryResp);
return true;
}
case RetFormat::HEX: {
CDataStream ssResp(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags());
ssResp << filter;

std::string strHex = HexStr(ssResp) + "\n";
req->WriteHeader("Content-Type", "text/plain");
req->WriteReply(HTTP_OK, strHex);
return true;
}
case RetFormat::JSON: {
UniValue ret(UniValue::VOBJ);
ret.pushKV("filter", HexStr(filter.GetEncodedFilter()));
std::string strJSON = ret.write() + "\n";
req->WriteHeader("Content-Type", "application/json");
req->WriteReply(HTTP_OK, strJSON);
return true;
}
default: {
return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")");
}
}
}

// A bit of a hack - dependency on a function defined in rpc/blockchain.cpp
RPCHelpMan getblockchaininfo();

Expand Down Expand Up @@ -718,6 +925,8 @@ static const struct {
{"/rest/tx/", rest_tx},
{"/rest/block/notxdetails/", rest_block_notxdetails},
{"/rest/block/", rest_block_extended},
{"/rest/blockfilter/", rest_block_filter},
{"/rest/blockfilterheaders/", rest_filter_header},
{"/rest/chaininfo", rest_chaininfo},
{"/rest/mempool/info", rest_mempool_info},
{"/rest/mempool/contents", rest_mempool_contents},
Expand Down
7 changes: 5 additions & 2 deletions test/functional/interface_rest.py
View file
Expand Up @@ -41,7 +41,7 @@ class RESTTest (BitcoinTestFramework):
def set_test_params(self):
self.setup_clean_chain = True
self.num_nodes = 2
self.extra_args = [["-rest"], []]
self.extra_args = [["-rest", "-blockfilterindex=1"], []]
self.supports_cli = False

def skip_test_if_missing_module(self):
Expand Down Expand Up @@ -272,11 +272,14 @@ def run_test(self):
self.generate(self.nodes[1], 5)
json_obj = self.test_rest_request(f"/headers/5/{bb_hash}")
assert_equal(len(json_obj), 5) # now we should have 5 header objects
json_obj = self.test_rest_request(f"/blockfilterheaders/basic/5/{bb_hash}")
assert_equal(len(json_obj), 5) # now we should have 5 filter header objects
self.test_rest_request(f"/blockfilter/basic/{bb_hash}", req_type=ReqType.BIN, ret_type=RetType.OBJ)

# Test number parsing
for num in ['5a', '-5', '0', '2001', '99999999999999999999999999999999999']:
assert_equal(
bytes(f'Header count out of range: {num}\r\n', 'ascii'),
bytes(f'Header count out of acceptable range (1-2000): {num}\r\n', 'ascii'),
self.test_rest_request(f"/headers/{num}/{bb_hash}", ret_type=RetType.BYTES, status=400),
)

Expand Down

0 comments on commit 70d6a09

Please sign in to comment.