Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow to optional specify the directory for the blocks storage #12653

Merged
merged 3 commits into from Mar 27, 2018
Merged
Changes from all commits
Commits
File filter...
Filter file types
Jump to…
Jump to file or symbol
Failed to load files and symbols.
+85 −11
Diff settings

Always

Just for now

Copy path View file
@@ -333,6 +333,7 @@ std::string HelpMessage(HelpMessageMode mode)
strUsage += HelpMessageOpt("-version", _("Print version and exit"));
strUsage += HelpMessageOpt("-alertnotify=<cmd>", _("Execute command when a relevant alert is received or we see a really long fork (%s in cmd is replaced by message)"));
strUsage +=HelpMessageOpt("-assumevalid=<hex>", strprintf(_("If this block is in the chain assume that it and its ancestors are valid and potentially skip their script verification (0 to verify all, default: %s, testnet: %s)"), defaultChainParams->GetConsensus().defaultAssumeValid.GetHex(), testnetChainParams->GetConsensus().defaultAssumeValid.GetHex()));
strUsage += HelpMessageOpt("-blocksdir=<dir>", _("Specify blocks directory (default: <datadir>/blocks)"));
strUsage += HelpMessageOpt("-blocknotify=<cmd>", _("Execute command when the best block changes (%s in cmd is replaced by block hash)"));
strUsage += HelpMessageOpt("-blockreconstructionextratxn=<n>", strprintf(_("Extra transactions to keep in memory for compact block reconstructions (default: %u)"), DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN));
if (showDebug)
@@ -596,7 +597,7 @@ void CleanupBlockRevFiles()
// Remove the rev files immediately and insert the blk file paths into an
// ordered map keyed by block file index.
LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n");
fs::path blocksdir = GetDataDir() / "blocks";
fs::path blocksdir = GetBlocksDir();
for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) {
if (fs::is_regular_file(*it) &&
it->path().filename().string().length() == 12 &&
@@ -908,6 +909,10 @@ bool AppInitParameterInteraction()

// also see: InitParameterInteraction()

if (!fs::is_directory(GetBlocksDir(false))) {
return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist.\n"), gArgs.GetArg("-blocksdir", "").c_str()));
}

// if using block pruning, then disallow txindex
if (gArgs.GetArg("-prune", 0)) {
if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX))
@@ -1630,7 +1635,7 @@ bool AppInitMain()

// ********************************************************* Step 10: import blocks

if (!CheckDiskSpace())
if (!CheckDiskSpace() && !CheckDiskSpace(0, true))
return false;

// Either install a handler to notify us when genesis activates, or set fHaveGenesis directly.
Copy path View file
@@ -617,7 +617,7 @@ int main(int argc, char *argv[])
if (!Intro::pickDataDirectory())
return EXIT_SUCCESS;

/// 6. Determine availability of data directory and parse bitcoin.conf
/// 6. Determine availability of data and blocks directory and parse bitcoin.conf
/// - Do not call GetDataDir(true) before this step finishes
if (!fs::is_directory(GetDataDir(false)))
{
Copy path View file
@@ -147,7 +147,7 @@ size_t CCoinsViewDB::EstimateSize() const
return db.EstimateSize(DB_COIN, (char)(DB_COIN+1));
}

CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) {
CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(gArgs.IsArgSet("-blocksdir") ? GetDataDir() / "blocks" / "index" : GetBlocksDir() / "index", nCacheSize, fMemory, fWipe) {
}

bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) {
Copy path View file
@@ -598,10 +598,41 @@ fs::path GetDefaultDataDir()
#endif
}

static fs::path g_blocks_path_cached;
static fs::path g_blocks_path_cache_net_specific;
static fs::path pathCached;
static fs::path pathCachedNetSpecific;
static CCriticalSection csPathCached;

const fs::path &GetBlocksDir(bool fNetSpecific)
{

LOCK(csPathCached);

fs::path &path = fNetSpecific ? g_blocks_path_cache_net_specific : g_blocks_path_cached;

// This can be called during exceptions by LogPrintf(), so we cache the
// value so we don't have to do memory allocations after that.
if (!path.empty())
return path;

if (gArgs.IsArgSet("-blocksdir")) {
path = fs::system_complete(gArgs.GetArg("-blocksdir", ""));
if (!fs::is_directory(path)) {
path = "";
return path;
}
} else {
path = GetDataDir(false);
}
if (fNetSpecific)
path /= BaseParams().DataDir();

path /= "blocks";
fs::create_directories(path);
return path;
}

const fs::path &GetDataDir(bool fNetSpecific)
{

@@ -640,6 +671,8 @@ void ClearDatadirCache()

pathCached = fs::path();
pathCachedNetSpecific = fs::path();
g_blocks_path_cached = fs::path();
g_blocks_path_cache_net_specific = fs::path();
}

fs::path GetConfigFile(const std::string& confPath)
Copy path View file
@@ -182,6 +182,7 @@ void ReleaseDirectoryLocks();

bool TryCreateDirectories(const fs::path& p);
fs::path GetDefaultDataDir();
const fs::path &GetBlocksDir(bool fNetSpecific = true);

This comment has been minimized.

Copy link
@MarcoFalke

MarcoFalke Mar 11, 2018

Member

I am wondering why this should be net specific, considering that we have the same concept for wallets and the GetWalletDir is not net specific.

This comment has been minimized.

Copy link
@laanwj

laanwj Mar 15, 2018

Member

Isn't the blocks directory by definition net specific?

const fs::path &GetDataDir(bool fNetSpecific = true);
void ClearDatadirCache();
fs::path GetConfigFile(const std::string& confPath);
Copy path View file
@@ -2056,7 +2056,7 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState &
// Write blocks and block index to disk.
if (fDoFullFlush || fPeriodicWrite) {
// Depend on nMinDiskSpace to ensure we can write block index
if (!CheckDiskSpace(0))
if (!CheckDiskSpace(0, true))
return state.Error("out of disk space");
// First make sure all block and undo data is flushed to disk.
FlushBlockFile();
@@ -2895,7 +2895,7 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int
if (nNewChunks > nOldChunks) {
if (fPruneMode)
fCheckForPruning = true;
if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos)) {
if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos, true)) {
FILE *file = OpenBlockFile(pos);
if (file) {
LogPrintf("Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile);
@@ -2928,7 +2928,7 @@ static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos,
if (nNewChunks > nOldChunks) {
if (fPruneMode)
fCheckForPruning = true;
if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos)) {
if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos, true)) {
FILE *file = OpenUndoFile(pos);
if (file) {
LogPrintf("Pre-allocating up to position 0x%x in rev%05u.dat\n", nNewChunks * UNDOFILE_CHUNK_SIZE, pos.nFile);
@@ -3604,9 +3604,9 @@ static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfte
nLastBlockWeCanPrune, count);
}

bool CheckDiskSpace(uint64_t nAdditionalBytes)
bool CheckDiskSpace(uint64_t nAdditionalBytes, bool blocks_dir)
{
uint64_t nFreeBytesAvailable = fs::space(GetDataDir()).available;
uint64_t nFreeBytesAvailable = fs::space(blocks_dir ? GetBlocksDir() : GetDataDir()).available;

// Check for nMinDiskSpace bytes (currently 50MB)
if (nFreeBytesAvailable < nMinDiskSpace + nAdditionalBytes)
@@ -3649,7 +3649,7 @@ static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) {

fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix)
{
return GetDataDir() / "blocks" / strprintf("%s%05u.dat", prefix, pos.nFile);
return GetBlocksDir() / strprintf("%s%05u.dat", prefix, pos.nFile);
}

CBlockIndex * CChainState::InsertBlockIndex(const uint256& hash)
Copy path View file
@@ -254,7 +254,7 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons
bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex=nullptr, CBlockHeader *first_invalid=nullptr);

/** Check whether enough disk space is available for an incoming block */
bool CheckDiskSpace(uint64_t nAdditionalBytes = 0);
bool CheckDiskSpace(uint64_t nAdditionalBytes = 0, bool blocks_dir = false);
/** Open a block file (blk?????.dat) */
FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly = false);
/** Translation to a filesystem path */
@@ -0,0 +1,34 @@
#!/usr/bin/env python3
# Copyright (c) 2018 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test the blocksdir option.
"""

from test_framework.test_framework import BitcoinTestFramework, initialize_datadir

import shutil
import os

class BlocksdirTest(BitcoinTestFramework):
def set_test_params(self):
self.setup_clean_chain = True
self.num_nodes = 1

def run_test(self):
self.stop_node(0)
node0path = os.path.join(self.options.tmpdir, "node0")
shutil.rmtree(node0path)
initialize_datadir(self.options.tmpdir, 0)
self.log.info("Starting with non exiting blocksdir ...")
self.assert_start_raises_init_error(0, ["-blocksdir="+self.options.tmpdir+ "/blocksdir"], "Specified blocks director")
os.mkdir(self.options.tmpdir+ "/blocksdir")
self.log.info("Starting with exiting blocksdir ...")
self.start_node(0, ["-blocksdir="+self.options.tmpdir+ "/blocksdir"])
self.log.info("mining blocks..")
self.nodes[0].generate(10)
assert(os.path.isfile(os.path.join(self.options.tmpdir, "blocksdir", "regtest", "blocks", "blk00000.dat")))
assert(os.path.isdir(os.path.join(self.options.tmpdir, "node0", "regtest", "blocks", "index")))

if __name__ == '__main__':
BlocksdirTest().main()
@@ -133,6 +133,7 @@
'p2p_unrequested_blocks.py',
'feature_logging.py',
'p2p_node_network_limited.py',
'feature_blocksdir.py',
'feature_config_args.py',
# Don't append tests at the end to avoid merge conflicts
# Put them in a random line within the section that fits their approximate run-time
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.