Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
faster -reindex by first deserializing headers
  • Loading branch information
LarryRuane committed Feb 21, 2020
1 parent eb3c6b0 commit a50f956
Show file tree
Hide file tree
Showing 3 changed files with 100 additions and 17 deletions.
11 changes: 9 additions & 2 deletions src/streams.h
Expand Up @@ -785,13 +785,20 @@ class CBufferedFile
nNow = vchBuf.size() - pos;
if (nNow + nReadPos > nSrcPos)
nNow = nSrcPos - nReadPos;
memcpy(pch, &vchBuf[pos], nNow);
if (pch) {
memcpy(pch, &vchBuf[pos], nNow);
pch += nNow;
}
nReadPos += nNow;
pch += nNow;
nSize -= nNow;
}
}

//! move ahead in the stream
void Skip(size_t nSize) {
read(nullptr, nSize);
}

//! return the current reading position
uint64_t GetPos() const {
return nReadPos;
Expand Down
72 changes: 69 additions & 3 deletions src/test/streams_tests.cpp
Expand Up @@ -305,7 +305,11 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file)
}
BOOST_CHECK_EQUAL(bf.GetPos(), 40);

// We've read the entire file, the next read should throw.
// Even though we're at the end of the file, the EOF indicator isn't set
// yet because we haven't tried to read beyond the end.
BOOST_CHECK(!bf.eof());

// At end of file, the next read should throw.
try {
bf >> i;
BOOST_CHECK(false);
Expand Down Expand Up @@ -335,6 +339,57 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file)
fs::remove("streams_test_tmp");
}

BOOST_AUTO_TEST_CASE(streams_buffered_file_skip)
{
FILE* file = fsbridge::fopen("streams_test_tmp", "w+b");
// The value at each offset is the offset.
for (uint8_t j = 0; j < 40; ++j) {
fwrite(&j, 1, 1, file);
}
rewind(file);

// The buffer is 25 bytes, allow rewinding 10 bytes.
CBufferedFile bf(file, 25, 10, 222, 333);

uint8_t i;
// This is like bf >> (7-byte-variable), in that it will cause data
// to be read from the file into memory, but it's not copied to us.
bf.Skip(7);
BOOST_CHECK_EQUAL(bf.GetPos(), 7);
bf >> i;
BOOST_CHECK_EQUAL(i, 7);

// The bytes in the buffer up to offset 7 are valid and can be read.
BOOST_CHECK(bf.SetPos(0));
bf >> i;
BOOST_CHECK_EQUAL(i, 0);
bf >> i;
BOOST_CHECK_EQUAL(i, 1);

// Skip()'s argument is the number of bytes to move forward in the
// file, not the absolute file position. Since bf is currently
// positioned to 2, this will advance it to 11.
bf.Skip(9);
bf >> i;
BOOST_CHECK_EQUAL(i, 11);

// Skip() honors the transfer limit; this allows only one byte
// to be skipped (or read) since we're at position 12.
bf.SetLimit(13);
try {
bf.Skip(2);
BOOST_CHECK(false);
} catch (const std::exception& e) {
BOOST_CHECK(strstr(e.what(),
"Read attempted past buffer limit") != nullptr);
}
bf.Skip(1);
BOOST_CHECK_EQUAL(bf.GetPos(), 13);

bf.fclose();
fs::remove("streams_test_tmp");
}

BOOST_AUTO_TEST_CASE(streams_buffered_file_rand)
{
// Make this test deterministic.
Expand Down Expand Up @@ -365,7 +420,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand)
// sizes; the boundaries of the objects can interact arbitrarily
// with the CBufferFile's internal buffer. These first three
// cases simulate objects of various sizes (1, 2, 5 bytes).
switch (InsecureRandRange(5)) {
switch (InsecureRandRange(6)) {
case 0: {
uint8_t a[1];
if (currentPos + 1 > fileSize)
Expand Down Expand Up @@ -403,6 +458,17 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand)
break;
}
case 3: {
// Skip is similar to the "read" cases above, except
// we don't receive the data.
size_t nSize = InsecureRandRange(5);
if (currentPos + nSize > fileSize)
continue;
bf.SetLimit(currentPos + nSize);
bf.Skip(nSize);
currentPos += nSize;
break;
}
case 4: {
// Find a byte value (that is at or ahead of the current position).
size_t find = currentPos + InsecureRandRange(8);
if (find >= fileSize)
Expand All @@ -419,7 +485,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand)
currentPos++;
break;
}
case 4: {
case 5: {
size_t requestPos = InsecureRandRange(maxPos + 4);
bool okay = bf.SetPos(requestPos);
// The new position may differ from the requested position
Expand Down
34 changes: 22 additions & 12 deletions src/validation.cpp
Expand Up @@ -4674,29 +4674,39 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi
break;
}
try {
// read block
// read block header
uint64_t nBlockPos = blkdat.GetPos();
if (dbp)
dbp->nPos = nBlockPos;
blkdat.SetLimit(nBlockPos + nSize);
blkdat.SetPos(nBlockPos);
std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>();
CBlock& block = *pblock;
blkdat >> block;
nRewind = blkdat.GetPos();

uint256 hash = block.GetHash();
blkdat.SetLimit(nBlockPos + 80);
CBlockHeader header;
blkdat >> header;

const uint256 hash = header.GetHash();
{
LOCK(cs_main);
// detect out of order blocks, and store them for later
if (hash != chainparams.GetConsensus().hashGenesisBlock && !LookupBlockIndex(block.hashPrevBlock)) {
if (hash != chainparams.GetConsensus().hashGenesisBlock && !LookupBlockIndex(header.hashPrevBlock)) {
LogPrint(BCLog::REINDEX, "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(),
block.hashPrevBlock.ToString());
header.hashPrevBlock.ToString());
if (dbp)
mapBlocksUnknownParent.insert(std::make_pair(block.hashPrevBlock, *dbp));
mapBlocksUnknownParent.insert(std::make_pair(header.hashPrevBlock, *dbp));

// Position to the start of the next block.
nRewind = nBlockPos + nSize;
blkdat.SetLimit(nRewind);
blkdat.Skip(nRewind - blkdat.GetPos());
continue;
}

// This block can be processed immediately; rewind to its start then read it.
blkdat.SetPos(nBlockPos);
blkdat.SetLimit(nBlockPos + nSize);
std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>();
CBlock& block = *pblock;
blkdat >> block;
nRewind = blkdat.GetPos();

// process in case the block isn't known yet
CBlockIndex* pindex = LookupBlockIndex(hash);
if (!pindex || (pindex->nStatus & BLOCK_HAVE_DATA) == 0) {
Expand Down

0 comments on commit a50f956

Please sign in to comment.