Skip to content

Commit

Permalink
Merge pull request cryptonomex#339 from pmconrad/quickstart
Browse files Browse the repository at this point in the history 
Improve startup time after crash
  • Loading branch information
@oxarbitrage
oxarbitrage committed Sep 8, 2017
2 parents 2134eab + 4822c21 commit ea23881
Show file tree
Hide file tree
Showing 14 changed files with 186 additions and 169 deletions.
65 changes: 8 additions & 57 deletions libraries/app/application.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,6 @@ namespace detail {

~application_impl()
{
fc::remove_all(_data_dir / "blockchain/dblock");
}

void set_dbg_init_key( genesis_state_type& genesis, const std::string& init_key )
Expand All @@ -323,8 +322,7 @@ namespace detail {

void startup()
{ try {
bool clean = !fc::exists(_data_dir / "blockchain/dblock");
fc::create_directories(_data_dir / "blockchain/dblock");
fc::create_directories(_data_dir / "blockchain");

auto initial_state = [&] {
ilog("Initializing database...");
Expand Down Expand Up @@ -390,64 +388,17 @@ namespace detail {
bool replay = false;
std::string replay_reason = "reason not provided";

// never replay if data dir is empty
if( fc::exists( _data_dir ) && fc::directory_iterator( _data_dir ) != fc::directory_iterator() )
{
if( _options->count("replay-blockchain") )
{
replay = true;
replay_reason = "replay-blockchain argument specified";
}
else if( !clean )
{
replay = true;
replay_reason = "unclean shutdown detected";
}
else if( !fc::exists( _data_dir / "db_version" ) )
{
replay = true;
replay_reason = "db_version file not found";
}
else
{
std::string version_string;
fc::read_file_contents( _data_dir / "db_version", version_string );
if( _options->count("replay-blockchain") )
_chain_db->wipe( _data_dir / "blockchain", false );

if( version_string != GRAPHENE_CURRENT_DB_VERSION )
{
replay = true;
replay_reason = "db_version file content mismatch";
}
}
}

if( !replay )
try
{
try
{
_chain_db->open( _data_dir / "blockchain", initial_state );
}
catch( const fc::exception& e )
{
ilog( "Caught exception ${e} in open()", ("e", e.to_detail_string()) );

replay = true;
replay_reason = "exception in open()";
}
_chain_db->open( _data_dir / "blockchain", initial_state, GRAPHENE_CURRENT_DB_VERSION );
}

if( replay )
catch( const fc::exception& e )
{
ilog( "Replaying blockchain due to: ${reason}", ("reason", replay_reason) );

fc::remove_all( _data_dir / "db_version" );
_chain_db->reindex( _data_dir / "blockchain", initial_state() );

const auto mode = std::ios::out | std::ios::binary | std::ios::trunc;
std::ofstream db_version( (_data_dir / "db_version").generic_string().c_str(), mode );
std::string version_string = GRAPHENE_CURRENT_DB_VERSION;
db_version.write( version_string.c_str(), version_string.size() );
db_version.close();
elog( "Caught exception ${e} in open(), you might want to force a replay", ("e", e.to_detail_string()) );
throw;
}

if( _options->count("force-validate") )
Expand Down
95 changes: 44 additions & 51 deletions libraries/chain/block_database.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,14 +45,15 @@ void block_database::open( const fc::path& dbdir )
_block_num_to_pos.exceptions(std::ios_base::failbit | std::ios_base::badbit);
_blocks.exceptions(std::ios_base::failbit | std::ios_base::badbit);

if( !fc::exists( dbdir/"index" ) )
_index_filename = dbdir / "index";
if( !fc::exists( _index_filename ) )
{
_block_num_to_pos.open( (dbdir/"index").generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out | std::fstream::trunc);
_block_num_to_pos.open( _index_filename.generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out | std::fstream::trunc);
_blocks.open( (dbdir/"blocks").generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out | std::fstream::trunc);
}
else
{
_block_num_to_pos.open( (dbdir/"index").generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out );
_block_num_to_pos.open( _index_filename.generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out );
_blocks.open( (dbdir/"blocks").generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out );
}
} FC_CAPTURE_AND_RETHROW( (dbdir) ) }
Expand Down Expand Up @@ -121,7 +122,7 @@ bool block_database::contains( const block_id_type& id )const
index_entry e;
auto index_pos = sizeof(e)*block_header::num_from_id(id);
_block_num_to_pos.seekg( 0, _block_num_to_pos.end );
if ( _block_num_to_pos.tellg() <= index_pos )
if ( _block_num_to_pos.tellg() < index_pos + sizeof(e) )
return false;
_block_num_to_pos.seekg( index_pos );
_block_num_to_pos.read( (char*)&e, sizeof(e) );
Expand Down Expand Up @@ -206,77 +207,69 @@ optional<signed_block> block_database::fetch_by_number( uint32_t block_num )cons
return optional<signed_block>();
}

optional<signed_block> block_database::last()const
{
optional<index_entry> block_database::last_index_entry()const {
try
{
index_entry e;

_block_num_to_pos.seekg( 0, _block_num_to_pos.end );
std::streampos pos = _block_num_to_pos.tellg();
if( pos < sizeof(index_entry) )
return optional<index_entry>();

if( _block_num_to_pos.tellp() < sizeof(index_entry) )
return optional<signed_block>();
pos -= pos % sizeof(index_entry);

_block_num_to_pos.seekg( -sizeof(index_entry), _block_num_to_pos.end );
_block_num_to_pos.read( (char*)&e, sizeof(e) );
uint64_t pos = _block_num_to_pos.tellg();
while( e.block_size == 0 && pos > 0 )
_blocks.seekg( 0, _block_num_to_pos.end );
const std::streampos blocks_size = _blocks.tellg();
while( pos > 0 )
{
pos -= sizeof(index_entry);
_block_num_to_pos.seekg( pos );
_block_num_to_pos.read( (char*)&e, sizeof(e) );
if( _block_num_to_pos.gcount() == sizeof(e) && e.block_size > 0
&& e.block_pos + e.block_size <= blocks_size )
try
{
vector<char> data( e.block_size );
_blocks.seekg( e.block_pos );
_blocks.read( data.data(), e.block_size );
if( _blocks.gcount() == e.block_size )
{
const signed_block block = fc::raw::unpack<signed_block>(data);
if( block.id() == e.block_id )
return e;
}
}
catch (const fc::exception&)
{
}
catch (const std::exception&)
{
}
fc::resize_file( _index_filename, pos );
}

if( e.block_size == 0 )
return optional<signed_block>();

vector<char> data( e.block_size );
_blocks.seekg( e.block_pos );
_blocks.read( data.data(), e.block_size );
auto result = fc::raw::unpack<signed_block>(data);
return result;
}
catch (const fc::exception&)
{
}
catch (const std::exception&)
{
}
return optional<index_entry>();
}

optional<signed_block> block_database::last()const
{
optional<index_entry> entry = last_index_entry();
if( entry.valid() ) return fetch_by_number( block_header::num_from_id(entry->block_id) );
return optional<signed_block>();
}

optional<block_id_type> block_database::last_id()const
{
try
{
index_entry e;
_block_num_to_pos.seekg( 0, _block_num_to_pos.end );

if( _block_num_to_pos.tellp() < sizeof(index_entry) )
return optional<block_id_type>();

_block_num_to_pos.seekg( -sizeof(index_entry), _block_num_to_pos.end );
_block_num_to_pos.read( (char*)&e, sizeof(e) );
uint64_t pos = _block_num_to_pos.tellg();
while( e.block_size == 0 && pos > 0 )
{
pos -= sizeof(index_entry);
_block_num_to_pos.seekg( pos );
_block_num_to_pos.read( (char*)&e, sizeof(e) );
}

if( e.block_size == 0 )
return optional<block_id_type>();

return e.block_id;
}
catch (const fc::exception&)
{
}
catch (const std::exception&)
{
}
optional<index_entry> entry = last_index_entry();
if( entry.valid() ) return entry->block_id;
return optional<block_id_type>();
}


} }
1 change: 0 additions & 1 deletion libraries/chain/db_block.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -423,7 +423,6 @@ void database::pop_block()
GRAPHENE_ASSERT( head_block.valid(), pop_empty_chain, "there are no blocks to pop" );

_fork_db.pop_block();
_block_id_to_block.remove( head_id );
pop_undo();

_popped_tx.insert( _popped_tx.begin(), head_block->transactions.begin(), head_block->transactions.end() );
Expand Down
96 changes: 63 additions & 33 deletions libraries/chain/db_management.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,27 +46,39 @@ database::~database()
clear_pending();
}

void database::reindex(fc::path data_dir, const genesis_state_type& initial_allocation)
void database::reindex( fc::path data_dir )
{ try {
ilog( "reindexing blockchain" );
wipe(data_dir, false);
open(data_dir, [&initial_allocation]{return initial_allocation;});

auto start = fc::time_point::now();
auto last_block = _block_id_to_block.last();
if( !last_block ) {
elog( "!no last block" );
edump((last_block));
return;
}
if( last_block->block_num() <= head_block_num()) return;

ilog( "reindexing blockchain" );
auto start = fc::time_point::now();
const auto last_block_num = last_block->block_num();
uint32_t flush_point = last_block_num < 10000 ? 0 : last_block_num - 10000;
uint32_t undo_point = last_block_num < 50 ? 0 : last_block_num - 50;

ilog( "Replaying blocks..." );
_undo_db.disable();
for( uint32_t i = 1; i <= last_block_num; ++i )
ilog( "Replaying blocks, starting at ${next}...", ("next",head_block_num() + 1) );
if( head_block_num() >= undo_point )
{
if( head_block_num() > 0 )
_fork_db.start_block( *fetch_block_by_number( head_block_num() ) );
}
else
_undo_db.disable();
for( uint32_t i = head_block_num() + 1; i <= last_block_num; ++i )
{
if( i % 10000 == 0 ) std::cerr << " " << double(i*100)/last_block_num << "% "<<i << " of " <<last_block_num<<" \n";
if( i == flush_point )
{
ilog( "Writing database to disk at block ${i}", ("i",i) );
flush();
ilog( "Done" );
}
fc::optional< signed_block > block = _block_id_to_block.fetch_by_number(i);
if( !block.valid() )
{
Expand All @@ -87,12 +99,23 @@ void database::reindex(fc::path data_dir, const genesis_state_type& initial_allo
wlog( "Dropped ${n} blocks from after the gap", ("n", dropped_count) );
break;
}
apply_block(*block, skip_witness_signature |
skip_transaction_signatures |
skip_transaction_dupe_check |
skip_tapos_check |
skip_witness_schedule_check |
skip_authority_check);
if( i < undo_point )
apply_block(*block, skip_witness_signature |
skip_transaction_signatures |
skip_transaction_dupe_check |
skip_tapos_check |
skip_witness_schedule_check |
skip_authority_check);
else
{
_undo_db.enable();
push_block(*block, skip_witness_signature |
skip_transaction_signatures |
skip_transaction_dupe_check |
skip_tapos_check |
skip_witness_schedule_check |
skip_authority_check);
}
}
_undo_db.enable();
auto end = fc::time_point::now();
Expand All @@ -110,28 +133,43 @@ void database::wipe(const fc::path& data_dir, bool include_blocks)

void database::open(
const fc::path& data_dir,
std::function<genesis_state_type()> genesis_loader)
std::function<genesis_state_type()> genesis_loader,
const std::string& db_version)
{
try
{
bool wipe_object_db = false;
if( !fc::exists( data_dir / "db_version" ) )
wipe_object_db = true;
else
{
std::string version_string;
fc::read_file_contents( data_dir / "db_version", version_string );
wipe_object_db = ( version_string != db_version );
}
if( wipe_object_db ) {
ilog("Wiping object_database due to missing or wrong version");
object_database::wipe( data_dir );
std::ofstream version_file( (data_dir / "db_version").generic_string().c_str(),
std::ios::out | std::ios::binary | std::ios::trunc );
version_file.write( db_version.c_str(), db_version.size() );
version_file.close();
}

object_database::open(data_dir);

_block_id_to_block.open(data_dir / "database" / "block_num_to_block");

if( !find(global_property_id_type()) )
init_genesis(genesis_loader());

fc::optional<signed_block> last_block = _block_id_to_block.last();
fc::optional<block_id_type> last_block = _block_id_to_block.last_id();
if( last_block.valid() )
{
_fork_db.start_block( *last_block );
idump((last_block->id())(last_block->block_num()));
idump((head_block_id())(head_block_num()));
if( last_block->id() != head_block_id() )
{
FC_ASSERT( head_block_num() == 0, "last block ID does not match current chain state",
("last_block->id", last_block->id())("head_block_num",head_block_num()) );
}
FC_ASSERT( *last_block >= head_block_id(),
"last block ID does not match current chain state",
("last_block->id", last_block)("head_block_id",head_block_num()) );
reindex( data_dir );
}
}
FC_CAPTURE_LOG_AND_RETHROW( (data_dir) )
Expand All @@ -153,17 +191,9 @@ void database::close(bool rewind)
ilog( "Rewinding from ${head} to ${cutoff}", ("head",head_block_num())("cutoff",cutoff) );
while( head_block_num() > cutoff )
{
// elog("pop");
block_id_type popped_block_id = head_block_id();
pop_block();
_fork_db.remove(popped_block_id); // doesn't throw on missing
try
{
_block_id_to_block.remove(popped_block_id);
}
catch (const fc::key_not_found_exception&)
{
}
}
}
catch ( const fc::exception& e )
Expand Down
Loading

0 comments on commit ea23881

Please sign in to comment.