db: verify the tail of the chain on startup and back out of a bad tip

src/db.cpp

@@ -435,6 +435,47 @@ bool CTxDB::LoadBlockIndex()
     nBestHeight = pindexBest->nHeight;
     printf("LoadBlockIndex(): hashBestChain=%s  height=%d\n", hashBestChain.ToString().substr(0,14).c_str(), nBestHeight);
 
+    // Verify the tail of the best chain, and fall back to the last good block
+    // if any of it fails.
+    //
+    // A node that accepted blocks its current rules would reject -- because it
+    // was running an older build, or because the tip it landed on was never
+    // valid -- has no way to notice on its own. It keeps the branch, keeps
+    // building on it, and keeps mining coins that the rest of the network
+    // throws away. The only cure used to be deleting blk*.dat by hand, which
+    // requires knowing something is wrong in the first place. That is how a
+    // node here once mined a 323-block fork for five days.
+    //
+    // Checking every block on every start would mean a RandomX hash per block,
+    // so only the tail is checked by default. /checkblocks=0 does the lot.
+    int nCheckBlocks = nCheckBlocksOnLoad;
+    CBlockIndex* pindexFork = NULL;
+    int nChecked = 0;
+    for (CBlockIndex* pindex = pindexBest; pindex && pindex->pprev; pindex = pindex->pprev)
+    {
+        if (nCheckBlocks > 0 && nChecked >= nCheckBlocks)
+            break;
+        nChecked++;
+        CBlock block;
+        if (!block.ReadFromDisk(pindex->nFile, pindex->nBlockPos, true))
+            return error("LoadBlockIndex() : ReadFromDisk failed at height %d", pindex->nHeight);
+        if (!block.CheckBlock())
+        {
+            printf("LoadBlockIndex() : *** bad block at height %d, hash=%s\n",
+                   pindex->nHeight, pindex->GetBlockHash().ToString().substr(0,14).c_str());
+            // Keep going: an older bad block further back wins, because
+            // everything after it has to come off too.
+            pindexFork = pindex->pprev;
+        }
+    }
+    printf("LoadBlockIndex(): verified %d block(s)\n", nChecked);
+
+    // The repair itself happens in the caller, after this handle is closed.
+    // The cursor above is still open and holds read locks, so a write
+    // transaction opened here waits on it forever -- measured: the node came
+    // up, printed the diagnosis, and hung with no error anywhere.
+    pindexBadChainFork = pindexFork;
+
     return true;
 }
 

src/main.cpp

@@ -49,6 +49,15 @@ uint256 hashGenesisMerkleRoot("0x1a45b4482532abb29b10e234d3f13132230525a339ecea9
 CBlockIndex* pindexGenesisBlock = NULL;
 int nBestHeight = -1;
 uint256 hashBestChain = 0;
+// Blocks at the tip to re-verify on startup. 0 means the whole chain. 288 is
+// about ten hours at a two-minute target -- far enough back to catch a bad tip,
+// short enough that startup does not pay a RandomX hash per block for the whole
+// history. /checkblocks=N overrides it.
+int nCheckBlocksOnLoad = 288;
+// Set by CTxDB::LoadBlockIndex when the tail of the chain fails verification.
+// The repair runs in LoadBlockIndex() below, once that read-only handle and its
+// cursor are closed.
+CBlockIndex* pindexBadChainFork = NULL;
 CBlockIndex* pindexBest = NULL;
 
 map<uint256, CBlock*> mapOrphanBlocks;
@@ -1577,6 +1586,72 @@ bool LoadBlockIndex(bool fAllowNew)
         return false;
     txdb.Close();
 
+    // Repair a bad tip found during verification.
+    //
+    // This has to happen after the handle above is closed. CTxDB::LoadBlockIndex
+    // walks the index with a cursor it never closes, so it is still holding read
+    // locks -- a write transaction opened alongside it waits on them forever.
+    // Measured, not guessed: the node came up, printed the diagnosis, and hung
+    // there with nothing in any log.
+    if (pindexBadChainFork)
+    {
+        CBlockIndex* pindexFork = pindexBadChainFork;
+        pindexBadChainFork = NULL;
+
+        printf("LoadBlockIndex() : *** moving best chain back to height %d, discarding %d block(s)\n",
+               pindexFork->nHeight, nBestHeight - pindexFork->nHeight);
+
+        vector<CBlockIndex*> vDisconnect;
+        for (CBlockIndex* pindex = pindexBest; pindex && pindex != pindexFork; pindex = pindex->pprev)
+            vDisconnect.push_back(pindex);
+
+        bool fRolledBack = false;
+        try
+        {
+            CTxDB txdbWrite;
+            txdbWrite.TxnBegin();
+            foreach(CBlockIndex* pindex, vDisconnect)
+            {
+                CBlock block;
+                if (!block.ReadFromDisk(pindex->nFile, pindex->nBlockPos, true))
+                    throw runtime_error("ReadFromDisk for disconnect failed");
+                if (!block.DisconnectBlock(txdbWrite, pindex))
+                    throw runtime_error("DisconnectBlock failed");
+            }
+            if (!txdbWrite.WriteHashBestChain(pindexFork->GetBlockHash()))
+                throw runtime_error("WriteHashBestChain failed");
+            txdbWrite.TxnCommit();
+            txdbWrite.Close();
+            fRolledBack = true;
+        }
+        catch (const std::exception& e)
+        {
+            printf("LoadBlockIndex() : rollback failed: %s\n", e.what());
+        }
+        catch (...)
+        {
+            printf("LoadBlockIndex() : rollback failed\n");
+        }
+
+        if (fRolledBack)
+        {
+            foreach(CBlockIndex* pindex, vDisconnect)
+                if (pindex->pprev)
+                    pindex->pprev->pnext = NULL;
+
+            pindexBest    = pindexFork;
+            hashBestChain = pindexBest->GetBlockHash();
+            nBestHeight   = pindexBest->nHeight;
+            printf("LoadBlockIndex() : now at height %d, will re-download from peers\n", nBestHeight);
+        }
+        else
+        {
+            // Start anyway. The operator needs a running node to read this from.
+            printf("LoadBlockIndex() : *** still on the bad chain at height %d -- back up the data directory and report this\n",
+                   nBestHeight);
+        }
+    }
+
     //
     // Init with genesis block
     //

src/main.h

@@ -52,6 +52,8 @@ extern unsigned int GENESIS_NONCE;
 extern CBlockIndex* pindexGenesisBlock;
 extern int nBestHeight;
 extern uint256 hashBestChain;
+extern int nCheckBlocksOnLoad;
+extern CBlockIndex* pindexBadChainFork;
 extern CBlockIndex* pindexBest;
 extern unsigned int nTransactionsUpdated;
 extern string strSetDataDir;

src/main_gui.cpp

@@ -54,6 +54,7 @@ static void PrintUsage()
     printf("  /participant=POOL_BTF_ADDRESS\n");
     printf("  /solomine\n");
     printf("  /genproclimit=N            (mining threads; 0 or absent = every core but one)\n");
+    printf("  /checkblocks=N             (blocks re-verified at startup, default 288, 0 = all)\n");
     printf("\n");
     printf("Pool operator announcement:\n");
     printf("  /poolname=NAME\n");
@@ -101,6 +102,10 @@ static void ParseStartupArguments(int argc, char* argv[])
     if (arg(argc,argv,"/solomine") || arg(argc,argv,"-solomine"))
         fSoloMineTest = true;
 
+    string strCheckBlocks = argval2(argc, argv, "/checkblocks", "-checkblocks");
+    if (!strCheckBlocks.empty())
+        nCheckBlocksOnLoad = atoi(strCheckBlocks.c_str());
+
     // /genproclimit=N -- threads to hash with. 0 or absent means automatic,
     // which is every core but one. Named after Bitcoin's own option so it reads
     // familiarly to anyone who has run one of these before.