Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
index: Implement block filter index with write operations.
- Loading branch information
Showing
3 changed files
with
327 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,272 @@ | ||
// Copyright (c) 2018 The Bitcoin Core developers | ||
// Distributed under the MIT software license, see the accompanying | ||
// file COPYING or http://www.opensource.org/licenses/mit-license.php. | ||
|
||
#include <map> | ||
|
||
#include <index/blockfilterindex.h> | ||
#include <util/system.h> | ||
#include <validation.h> | ||
|
||
/* The index database stores three items for each block: the disk location of the encoded filter, | ||
* its dSHA256 hash, and the header. Those belonging to blocks on the active chain are indexed by | ||
* height, and those belonging to blocks that have been reorganized out of the active chain are | ||
* indexed by block hash. This ensures that filter data for any block that becomes part of the | ||
* active chain can always be retrieved, alleviating timing concerns. | ||
* | ||
* The filters themselves are stored in flat files and referenced by the LevelDB entries. This | ||
* minimizes the amount of data written to LevelDB and keeps the database values constant size. | ||
* | ||
* Keys for the height index have the type [DB_BLOCK_HEIGHT, uint32 (BE)]. The height is represented | ||
* as big-endian so that sequential reads of filters by height are fast. | ||
* Keys for the hash index have the type [DB_BLOCK_HASH, uint256]. | ||
*/ | ||
constexpr char DB_BLOCK_HASH = 's'; | ||
constexpr char DB_BLOCK_HEIGHT = 't'; | ||
constexpr char DB_FILTER_POS = 'P'; | ||
|
||
constexpr unsigned int MAX_FILE_SIZE = 0x1000000; // 16 MiB | ||
/** The pre-allocation chunk size for fltr?????.dat files */ | ||
constexpr unsigned int FILE_CHUNK_SIZE = 0x100000; // 1 MiB | ||
|
||
namespace { | ||
struct DBVal | ||
{ | ||
uint256 hash; | ||
uint256 header; | ||
FlatFilePos pos; | ||
|
||
ADD_SERIALIZE_METHODS; | ||
|
||
template <typename Stream, typename Operation> | ||
inline void SerializationOp(Stream& s, Operation ser_action) { | ||
READWRITE(hash); | ||
READWRITE(header); | ||
READWRITE(pos); | ||
} | ||
}; | ||
|
||
struct DBHeightKey | ||
{ | ||
int height; | ||
|
||
DBHeightKey() : height(0) {} | ||
DBHeightKey(int height_in) : height(height_in) {} | ||
|
||
template<typename Stream> | ||
void Serialize(Stream& s) const | ||
{ | ||
ser_writedata8(s, DB_BLOCK_HEIGHT); | ||
ser_writedata32be(s, height); | ||
} | ||
|
||
template<typename Stream> | ||
void Unserialize(Stream& s) | ||
{ | ||
char prefix = ser_readdata8(s); | ||
if (prefix != DB_BLOCK_HEIGHT) { | ||
throw std::ios_base::failure("Invalid format for DB key"); | ||
} | ||
height = ser_readdata32be(s); | ||
} | ||
}; | ||
|
||
struct DBHashKey | ||
{ | ||
uint256 hash; | ||
|
||
DBHashKey(const uint256& hash_in) : hash(hash_in) {} | ||
|
||
ADD_SERIALIZE_METHODS; | ||
|
||
template <typename Stream, typename Operation> | ||
inline void SerializationOp(Stream& s, Operation ser_action) { | ||
char prefix = DB_BLOCK_HASH; | ||
READWRITE(prefix); | ||
if (prefix != DB_BLOCK_HASH) { | ||
throw std::ios_base::failure("Invalid format for DB key"); | ||
} | ||
|
||
READWRITE(hash); | ||
} | ||
}; | ||
}; | ||
|
||
BlockFilterIndex::BlockFilterIndex(BlockFilterType filter_type, | ||
size_t n_cache_size, bool f_memory, bool f_wipe) | ||
: m_filter_type(filter_type) | ||
{ | ||
const std::string& filter_name = BlockFilterTypeName(filter_type); | ||
if (filter_name == "") throw std::invalid_argument("unknown filter_type"); | ||
|
||
fs::path path = GetDataDir() / "indexes" / "blockfilter" / filter_name; | ||
fs::create_directories(path); | ||
|
||
m_name = filter_name + " block filter index"; | ||
m_db = MakeUnique<BaseIndex::DB>(path / "db", n_cache_size, f_memory, f_wipe); | ||
m_filter_fileseq = MakeUnique<FlatFileSeq>(std::move(path), "fltr", FILE_CHUNK_SIZE); | ||
} | ||
|
||
bool BlockFilterIndex::Init() | ||
{ | ||
if (!m_db->Read(DB_FILTER_POS, m_next_filter_pos)) { | ||
m_next_filter_pos.nFile = 0; | ||
m_next_filter_pos.nPos = 0; | ||
} | ||
return BaseIndex::Init(); | ||
} | ||
|
||
bool BlockFilterIndex::Commit(CDBBatch& batch) | ||
{ | ||
const FlatFilePos& pos = m_next_filter_pos; | ||
|
||
// Flush current filter file to disk. | ||
CAutoFile file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION); | ||
if (file.IsNull()) { | ||
return error("%s: Failed to open filter file %d", __func__, pos.nFile); | ||
} | ||
if (!FileCommit(file.Get())) { | ||
return error("%s: Failed to commit filter file %d", __func__, pos.nFile); | ||
} | ||
|
||
batch.Write(DB_FILTER_POS, pos); | ||
return BaseIndex::Commit(batch); | ||
} | ||
|
||
size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter) | ||
{ | ||
assert(filter.GetFilterType() == GetFilterType()); | ||
|
||
size_t data_size = | ||
GetSerializeSize(filter.GetBlockHash(), CLIENT_VERSION) + | ||
GetSerializeSize(filter.GetEncodedFilter(), CLIENT_VERSION); | ||
|
||
// If writing the filter would overflow the file, flush and move to the next one. | ||
if (pos.nPos + data_size > MAX_FILE_SIZE) { | ||
CAutoFile last_file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION); | ||
if (last_file.IsNull()) { | ||
LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile); | ||
return 0; | ||
} | ||
if (!TruncateFile(last_file.Get(), pos.nPos)) { | ||
LogPrintf("%s: Failed to truncate filter file %d\n", __func__, pos.nFile); | ||
return 0; | ||
} | ||
if (!FileCommit(last_file.Get())) { | ||
LogPrintf("%s: Failed to commit filter file %d\n", __func__, pos.nFile); | ||
return 0; | ||
} | ||
|
||
pos.nFile++; | ||
pos.nPos = 0; | ||
} | ||
|
||
// Pre-allocate sufficient space for filter data. | ||
bool out_of_space; | ||
m_filter_fileseq->Allocate(pos, data_size, out_of_space); | ||
if (out_of_space) { | ||
LogPrintf("%s: out of disk space\n", __func__); | ||
return 0; | ||
} | ||
|
||
CAutoFile fileout(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION); | ||
if (fileout.IsNull()) { | ||
LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile); | ||
return 0; | ||
} | ||
|
||
fileout << filter.GetBlockHash() << filter.GetEncodedFilter(); | ||
return data_size; | ||
} | ||
|
||
bool BlockFilterIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) | ||
{ | ||
CBlockUndo block_undo; | ||
uint256 prev_header; | ||
|
||
if (pindex->nHeight > 0) { | ||
if (!UndoReadFromDisk(block_undo, pindex)) { | ||
return false; | ||
} | ||
|
||
std::pair<uint256, DBVal> read_out; | ||
if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { | ||
return false; | ||
} | ||
|
||
uint256 expected_block_hash = pindex->pprev->GetBlockHash(); | ||
if (read_out.first != expected_block_hash) { | ||
return error("%s: previous block header belongs to unexpected block %s; expected %s", | ||
__func__, read_out.first.ToString(), expected_block_hash.ToString()); | ||
} | ||
|
||
prev_header = read_out.second.header; | ||
} | ||
|
||
BlockFilter filter(m_filter_type, block, block_undo); | ||
|
||
size_t bytes_written = WriteFilterToDisk(m_next_filter_pos, filter); | ||
if (bytes_written == 0) return false; | ||
|
||
std::pair<uint256, DBVal> value; | ||
value.first = pindex->GetBlockHash(); | ||
value.second.hash = filter.GetHash(); | ||
value.second.header = filter.ComputeHeader(prev_header); | ||
value.second.pos = m_next_filter_pos; | ||
|
||
if (!m_db->Write(DBHeightKey(pindex->nHeight), value)) { | ||
return false; | ||
} | ||
|
||
m_next_filter_pos += bytes_written; | ||
return true; | ||
} | ||
|
||
static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch, | ||
const std::string& index_name, int start_height, | ||
const CBlockIndex* stop_index) | ||
{ | ||
DBHeightKey key(start_height); | ||
db_it.Seek(key); | ||
|
||
for (int height = start_height; height <= stop_index->nHeight; ++height) { | ||
if (!db_it.GetKey(key) || key.height != height) { | ||
return error("%s: unexpected key in %s: expected (%c, %d)", | ||
__func__, index_name, DB_BLOCK_HEIGHT, height); | ||
} | ||
|
||
std::pair<uint256, DBVal> value; | ||
if (!db_it.GetValue(value)) { | ||
return error("%s: unable to read value in %s at key (%c, %d)", | ||
__func__, index_name, DB_BLOCK_HEIGHT, height); | ||
} | ||
|
||
batch.Write(DBHashKey(value.first), std::move(value.second)); | ||
|
||
db_it.Next(); | ||
} | ||
return true; | ||
} | ||
|
||
bool BlockFilterIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) | ||
{ | ||
assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); | ||
|
||
CDBBatch batch(*m_db); | ||
std::unique_ptr<CDBIterator> db_it(m_db->NewIterator()); | ||
|
||
// During a reorg, we need to copy all filters for blocks that are getting disconnected from the | ||
// height index to the hash index so we can still find them when the height index entries are | ||
// overwritten. | ||
if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight, current_tip)) { | ||
return false; | ||
} | ||
|
||
// The latest filter position gets written in Commit by the call to the BaseIndex::Rewind. | ||
// But since this creates new references to the filter, the position should get updated here | ||
// atomically as well in case Commit fails. | ||
batch.Write(DB_FILTER_POS, m_next_filter_pos); | ||
if (!m_db->WriteBatch(batch)) return false; | ||
|
||
return BaseIndex::Rewind(current_tip, new_tip); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
// Copyright (c) 2018 The Bitcoin Core developers | ||
// Distributed under the MIT software license, see the accompanying | ||
// file COPYING or http://www.opensource.org/licenses/mit-license.php. | ||
|
||
#ifndef BITCOIN_INDEX_BLOCKFILTERINDEX_H | ||
#define BITCOIN_INDEX_BLOCKFILTERINDEX_H | ||
|
||
#include <blockfilter.h> | ||
#include <chain.h> | ||
#include <flatfile.h> | ||
#include <index/base.h> | ||
|
||
/** | ||
* BlockFilterIndex is used to store and retrieve block filters, hashes, and headers for a range of | ||
* blocks by height. An index is constructed for each supported filter type with its own database | ||
* (ie. filter data for different types are stored in separate databases). | ||
* | ||
* This index is used to serve BIP 157 net requests. | ||
*/ | ||
class BlockFilterIndex final : public BaseIndex | ||
{ | ||
private: | ||
BlockFilterType m_filter_type; | ||
std::string m_name; | ||
std::unique_ptr<BaseIndex::DB> m_db; | ||
|
||
FlatFilePos m_next_filter_pos; | ||
std::unique_ptr<FlatFileSeq> m_filter_fileseq; | ||
|
||
size_t WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter); | ||
|
||
protected: | ||
bool Init() override; | ||
|
||
bool Commit(CDBBatch& batch) override; | ||
|
||
bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; | ||
|
||
bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) override; | ||
|
||
BaseIndex::DB& GetDB() const override { return *m_db; } | ||
|
||
const char* GetName() const override { return m_name.c_str(); } | ||
|
||
public: | ||
/** Constructs the index, which becomes available to be queried. */ | ||
explicit BlockFilterIndex(BlockFilterType filter_type, | ||
size_t n_cache_size, bool f_memory = false, bool f_wipe = false); | ||
|
||
BlockFilterType GetFilterType() const { return m_filter_type; } | ||
}; | ||
|
||
#endif // BITCOIN_INDEX_BLOCKFILTERINDEX_H |