WIP: Memory optimized combined num tx

src/chain.cpp

@@ -128,6 +128,24 @@ void CBlockIndex::BuildSkip()
         pskip = pprev->GetAncestor(GetSkipHeight(nHeight));
 }
 
+void CBlockIndex::SetChainTx(const uint64_t chain_tx) {
+    m_num_tx &= (static_cast<uint64_t>(0xFFFF));
+    m_num_tx |= (chain_tx << 16);
+}
+
+void CBlockIndex::SetBlockTx(const uint16_t block_tx) {
+    m_num_tx &= ~static_cast<uint64_t>(0xFFFF);
+    m_num_tx |= block_tx;
+}
+
+uint16_t CBlockIndex::GetBlockTx() const {
+    return static_cast<uint16_t>(m_num_tx & 0xFFFF);
+}
+
+uint64_t CBlockIndex::GetChainTx() const {
+    return m_num_tx >> 16;
+}
+
 arith_uint256 GetBlockProof(const CBlockIndex& block)
 {
     arith_uint256 bnTarget;

src/chain.h

@@ -148,6 +148,10 @@ enum BlockStatus : uint32_t {
  */
 class CBlockIndex
 {
+private:
+    //! TODO Docs
+    uint64_t m_num_tx{0};
+
 public:
     //! pointer to the hash of the block, if any. Memory is owned by this CBlockIndex
     const uint256* phashBlock{nullptr};
@@ -178,7 +182,7 @@ class CBlockIndex
     //! Note: this value is faked during UTXO snapshot load to ensure that
     //! LoadBlockIndex() will load index entries for blocks that we lack data for.
     //! @sa ActivateSnapshot
-    unsigned int nTx{0};
+    // unsigned int nTx{0};
 
     //! (memory only) Number of transactions in the chain up to and including this block.
     //! This value will be non-zero only if and only if transactions for this block and all its parents are available.
@@ -187,7 +191,7 @@ class CBlockIndex
     //! have the underlying block data available during snapshot load.
     //! @sa AssumeutxoData
     //! @sa ActivateSnapshot
-    uint64_t nChainTx{0};
+    // uint64_t nChainTx{0};
 
     //! Verification status of this block. See enum BlockStatus
     //!
@@ -271,7 +275,7 @@ class CBlockIndex
      * all subsequent assumed-valid blocks) since its nChainTx value will have been set
      * manually based on the related AssumeutxoData entry.
      */
-    bool HaveNumChainTxs() const { return nChainTx != 0; }
+    bool HaveNumChainTxs() const { return GetChainTx() != 0; }
 
     NodeSeconds Time() const
     {
@@ -353,6 +357,11 @@ class CBlockIndex
     CBlockIndex* GetAncestor(int height);
     const CBlockIndex* GetAncestor(int height) const;
 
+    void SetChainTx(const uint64_t chain_tx);
+    void SetBlockTx(const uint16_t block_tx);
+    uint16_t GetBlockTx() const;
+    uint64_t GetChainTx() const;
+
     CBlockIndex() = default;
     ~CBlockIndex() = default;
 
@@ -411,7 +420,10 @@ class CDiskBlockIndex : public CBlockIndex
 
         READWRITE(VARINT_MODE(obj.nHeight, VarIntMode::NONNEGATIVE_SIGNED));
         READWRITE(VARINT(obj.nStatus));
-        READWRITE(VARINT(obj.nTx));
+        uint16_t block_tx{0};
+        SER_WRITE(obj, block_tx = obj.GetBlockTx());
+        READWRITE(VARINT(block_tx));
+        SER_READ(obj, obj.SetBlockTx(block_tx));
         if (obj.nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO)) READWRITE(VARINT_MODE(obj.nFile, VarIntMode::NONNEGATIVE_SIGNED));
         if (obj.nStatus & BLOCK_HAVE_DATA) READWRITE(VARINT(obj.nDataPos));
         if (obj.nStatus & BLOCK_HAVE_UNDO) READWRITE(VARINT(obj.nUndoPos));

src/net_processing.cpp

@@ -4516,7 +4516,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type,
         }
 
         if (pindex->nChainWork <= m_chainman.ActiveChain().Tip()->nChainWork || // We know something better
-                pindex->nTx != 0) { // We had this block at some point, but pruned it
+                pindex->GetBlockTx() != 0) { // We had this block at some point, but pruned it
             if (requested_block_from_this_peer) {
                 // We requested this block for some reason, but our mempool will probably be useless
                 // so we just grab the block via normal getdata

src/node/blockstorage.cpp

@@ -128,7 +128,7 @@ bool BlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, s
                 pindexNew->nBits          = diskindex.nBits;
                 pindexNew->nNonce         = diskindex.nNonce;
                 pindexNew->nStatus        = diskindex.nStatus;
-                pindexNew->nTx            = diskindex.nTx;
+                pindexNew->SetBlockTx(diskindex.GetBlockTx());
 
                 if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, consensusParams)) {
                     LogError("%s: CheckProofOfWork failed: %s\n", __func__, pindexNew->ToString());
@@ -414,7 +414,7 @@ bool BlockManager::LoadBlockIndex(const std::optional<uint256>& snapshot_blockha
         // Since nChainTx (responsible for estimated progress) isn't persisted
         // to disk, we must bootstrap the value for assumedvalid chainstates
         // from the hardcoded assumeutxo chainparams.
-        base->nChainTx = au_data.nChainTx;
+        base->SetChainTx(au_data.nChainTx);
         LogPrintf("[snapshot] set nChainTx=%d for %s\n", au_data.nChainTx, snapshot_blockhash->ToString());
     } else {
         // If this isn't called with a snapshot blockhash, make sure the cached snapshot height
@@ -445,20 +445,20 @@ bool BlockManager::LoadBlockIndex(const std::optional<uint256>& snapshot_blockha
         // blocks that are assumed-valid on the basis of snapshot load (see
         // PopulateAndValidateSnapshot()).
         // Pruned nodes may have deleted the block.
-        if (pindex->nTx > 0) {
+        if (pindex->GetBlockTx() > 0) {
             if (pindex->pprev) {
                 if (m_snapshot_height && pindex->nHeight == *m_snapshot_height &&
                         pindex->GetBlockHash() == *snapshot_blockhash) {
                     // Should have been set above; don't disturb it with code below.
-                    Assert(pindex->nChainTx > 0);
-                } else if (pindex->pprev->nChainTx > 0) {
-                    pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx;
+                    Assert(pindex->GetChainTx() > 0);
+                } else if (pindex->pprev->GetChainTx() > 0) {
+                    pindex->SetChainTx(pindex->pprev->GetChainTx() + pindex->GetBlockTx());
                 } else {
-                    pindex->nChainTx = 0;
+                    pindex->SetChainTx(0);
                     m_blocks_unlinked.insert(std::make_pair(pindex->pprev, pindex));
                 }
             } else {
-                pindex->nChainTx = pindex->nTx;
+                pindex->SetChainTx(pindex->GetBlockTx());
             }
         }
         if (!(pindex->nStatus & BLOCK_FAILED_MASK) && pindex->pprev && (pindex->pprev->nStatus & BLOCK_FAILED_MASK)) {
@@ -592,7 +592,7 @@ const CBlockIndex* BlockManager::GetLastCheckpoint(const CCheckpointData& data)
 bool BlockManager::IsBlockPruned(const CBlockIndex& block)
 {
     AssertLockHeld(::cs_main);
-    return m_have_pruned && !(block.nStatus & BLOCK_HAVE_DATA) && (block.nTx > 0);
+    return m_have_pruned && !(block.nStatus & BLOCK_HAVE_DATA) && (block.GetBlockTx() > 0);
 }
 
 const CBlockIndex* BlockManager::GetFirstStoredBlock(const CBlockIndex& upper_block, const CBlockIndex* lower_block)

src/node/interfaces.cpp

@@ -534,7 +534,7 @@ class ChainImpl : public Chain
     {
         LOCK(::cs_main);
         const CBlockIndex* block{chainman().ActiveChain()[height]};
-        return block && ((block->nStatus & BLOCK_HAVE_DATA) != 0) && block->nTx > 0;
+        return block && ((block->nStatus & BLOCK_HAVE_DATA) != 0) && block->GetBlockTx() > 0;
     }
     CBlockLocator getTipLocator() override
     {

src/rpc/blockchain.cpp

@@ -152,7 +152,7 @@ UniValue blockheaderToJSON(const CBlockIndex& tip, const CBlockIndex& blockindex
     result.pushKV("bits", strprintf("%08x", blockindex.nBits));
     result.pushKV("difficulty", GetDifficulty(blockindex));
     result.pushKV("chainwork", blockindex.nChainWork.GetHex());
-    result.pushKV("nTx", blockindex.nTx);
+    result.pushKV("nTx", blockindex.GetBlockTx());
 
     if (blockindex.pprev)
         result.pushKV("previousblockhash", blockindex.pprev->GetBlockHash().GetHex());
@@ -1680,11 +1680,11 @@ static RPCHelpMan getchaintxstats()
 
     const CBlockIndex& past_block{*CHECK_NONFATAL(pindex->GetAncestor(pindex->nHeight - blockcount))};
     const int64_t nTimeDiff{pindex->GetMedianTimePast() - past_block.GetMedianTimePast()};
-    const uint64_t nTxDiff = pindex->nChainTx - past_block.nChainTx;
+    const uint64_t nTxDiff = pindex->GetChainTx() - past_block.GetChainTx();
 
     UniValue ret(UniValue::VOBJ);
     ret.pushKV("time", (int64_t)pindex->nTime);
-    ret.pushKV("txcount", pindex->nChainTx);
+    ret.pushKV("txcount", pindex->GetChainTx());
     ret.pushKV("window_final_block_hash", pindex->GetBlockHash().GetHex());
     ret.pushKV("window_final_block_height", pindex->nHeight);
     ret.pushKV("window_block_count", blockcount);
@@ -2716,7 +2716,7 @@ UniValue CreateUTXOSnapshot(
     result.pushKV("base_height", tip->nHeight);
     result.pushKV("path", path.utf8string());
     result.pushKV("txoutset_hash", maybe_stats->hashSerialized.ToString());
-    result.pushKV("nchaintx", tip->nChainTx);
+    result.pushKV("nchaintx", tip->GetChainTx());
     return result;
 }
 

src/rpc/txoutproof.cpp

@@ -152,12 +152,12 @@ static RPCHelpMan verifytxoutproof()
             LOCK(cs_main);
 
             const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(merkleBlock.header.GetHash());
-            if (!pindex || !chainman.ActiveChain().Contains(pindex) || pindex->nTx == 0) {
+            if (!pindex || !chainman.ActiveChain().Contains(pindex) || pindex->GetBlockTx() == 0) {
                 throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found in chain");
             }
 
             // Check if proof is valid, only add results if so
-            if (pindex->nTx == merkleBlock.txn.GetNumTransactions()) {
+            if (pindex->GetBlockTx() == merkleBlock.txn.GetNumTransactions()) {
                 for (const uint256& hash : vMatch) {
                     res.push_back(hash.GetHex());
                 }

src/test/blockchain_tests.cpp

@@ -77,8 +77,10 @@ BOOST_AUTO_TEST_CASE(get_difficulty_for_very_high_target)
 BOOST_AUTO_TEST_CASE(num_chain_tx_max)
 {
     CBlockIndex* block_index = CreateBlockIndexWithNbits(0);
-    block_index->nChainTx = std::numeric_limits<uint64_t>::max();
-    BOOST_CHECK_EQUAL(block_index->nChainTx, std::numeric_limits<uint64_t>::max());
+    block_index->SetChainTx(std::numeric_limits<uint64_t>::max() >> 16);
+    block_index->SetBlockTx(std::numeric_limits<uint16_t>::max());
+    BOOST_CHECK_EQUAL(block_index->GetChainTx(), std::numeric_limits<uint64_t>::max() >> 16);
+    BOOST_CHECK_EQUAL(block_index->GetBlockTx(), std::numeric_limits<uint16_t>::max());
 }
 
 BOOST_AUTO_TEST_SUITE_END()

src/test/fuzz/utxo_snapshot.cpp

@@ -76,12 +76,12 @@ FUZZ_TARGET(utxo_snapshot, .init = initialize_chain)
         for (const auto& block : *g_chain) {
             Assert(coinscache.HaveCoin(COutPoint{block->vtx.at(0)->GetHash(), 0}));
             const auto* index{chainman.m_blockman.LookupBlockIndex(block->GetHash())};
-            const auto num_tx{Assert(index)->nTx};
+            const auto num_tx{Assert(index)->GetBlockTx()};
             Assert(num_tx == 1);
             chain_tx += num_tx;
         }
         Assert(g_chain->size() == coinscache.GetCacheSize());
-        Assert(chain_tx == chainman.ActiveTip()->nChainTx);
+        Assert(chain_tx == chainman.ActiveTip()->GetChainTx());
     } else {
         Assert(!chainman.SnapshotBlockhash());
         Assert(!chainman.ActiveChainstate().m_from_snapshot_blockhash);

src/test/validation_chainstatemanager_tests.cpp

@@ -284,7 +284,7 @@ struct SnapshotTestSetup : TestChain100Setup {
         const auto& au_data = ::Params().AssumeutxoForHeight(snapshot_height);
         const CBlockIndex* tip = WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip());
 
-        BOOST_CHECK_EQUAL(tip->nChainTx, au_data->nChainTx);
+        BOOST_CHECK_EQUAL(tip->GetChainTx(), au_data->nChainTx);
 
         // To be checked against later when we try loading a subsequent snapshot.
         uint256 loaded_snapshot_blockhash{*chainman.SnapshotBlockhash()};

src/validation.cpp

@@ -2777,7 +2777,7 @@ static void UpdateTipLog(
     LogPrintf("%s%s: new best=%s height=%d version=0x%08x log2_work=%f tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)%s\n",
         prefix, func_name,
         tip->GetBlockHash().ToString(), tip->nHeight, tip->nVersion,
-        log(tip->nChainWork.getdouble()) / log(2.0), tip->nChainTx,
+        log(tip->nChainWork.getdouble()) / log(2.0), tip->GetChainTx(),
         FormatISO8601DateTime(tip->GetBlockTime()),
         GuessVerificationProgress(params.TxData(), tip),
         coins_tip.DynamicMemoryUsage() * (1.0 / (1 << 20)),
@@ -3659,8 +3659,8 @@ void Chainstate::TryAddBlockIndexCandidate(CBlockIndex* pindex)
 void ChainstateManager::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos)
 {
     AssertLockHeld(cs_main);
-    pindexNew->nTx = block.vtx.size();
-    pindexNew->nChainTx = 0;
+    pindexNew->SetBlockTx(block.vtx.size());
+    pindexNew->SetChainTx(0);
     pindexNew->nFile = pos.nFile;
     pindexNew->nDataPos = pos.nPos;
     pindexNew->nUndoPos = 0;
@@ -3680,7 +3680,7 @@ void ChainstateManager::ReceivedBlockTransactions(const CBlock& block, CBlockInd
         while (!queue.empty()) {
             CBlockIndex *pindex = queue.front();
             queue.pop_front();
-            pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx;
+            pindex->SetChainTx((pindex->pprev ? pindex->pprev->GetChainTx() : 0) + pindex->GetBlockTx());
             pindex->nSequenceId = nBlockSequenceId++;
             for (Chainstate *c : GetAll()) {
                 c->TryAddBlockIndexCandidate(pindex);
@@ -4260,7 +4260,7 @@ bool ChainstateManager::AcceptBlock(const std::shared_ptr<const CBlock>& pblock,
     // and unrequested blocks.
     if (fAlreadyHave) return true;
     if (!fRequested) {  // If we didn't ask for it:
-        if (pindex->nTx != 0) return true;    // This is a previously-processed block that was pruned
+        if (pindex->GetBlockTx() != 0) return true;    // This is a previously-processed block that was pruned
         if (!fHasMoreOrSameWork) return true; // Don't process less-work chains
         if (fTooFarAhead) return true;        // Block height is too high
 
@@ -5037,7 +5037,7 @@ void ChainstateManager::CheckBlockIndex()
         if (pindexFirstMissing == nullptr && !(pindex->nStatus & BLOCK_HAVE_DATA)) {
             pindexFirstMissing = pindex;
         }
-        if (pindexFirstNeverProcessed == nullptr && pindex->nTx == 0) pindexFirstNeverProcessed = pindex;
+        if (pindexFirstNeverProcessed == nullptr && pindex->GetBlockTx() == 0) pindexFirstNeverProcessed = pindex;
         if (pindex->pprev != nullptr && pindexFirstNotTreeValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TREE) pindexFirstNotTreeValid = pindex;
 
         if (pindex->pprev != nullptr && !pindex->IsAssumedValid()) {
@@ -5077,7 +5077,7 @@ void ChainstateManager::CheckBlockIndex()
         // background chainstate.
         if (!m_blockman.m_have_pruned && !pindex->IsAssumedValid()) {
             // If we've never pruned, then HAVE_DATA should be equivalent to nTx > 0
-            assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0));
+            assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->GetBlockTx() == 0));
             if (pindexFirstAssumeValid == nullptr) {
                 // If we've got some assume valid blocks, then we might have
                 // missing blocks (not HAVE_DATA) but still treat them as
@@ -5087,18 +5087,18 @@ void ChainstateManager::CheckBlockIndex()
             }
         } else {
             // If we have pruned, then we can only say that HAVE_DATA implies nTx > 0
-            if (pindex->nStatus & BLOCK_HAVE_DATA) assert(pindex->nTx > 0);
+            if (pindex->nStatus & BLOCK_HAVE_DATA) assert(pindex->GetBlockTx() > 0);
         }
         if (pindex->nStatus & BLOCK_HAVE_UNDO) assert(pindex->nStatus & BLOCK_HAVE_DATA);
         if (pindex->IsAssumedValid()) {
             // Assumed-valid blocks should have some nTx value.
-            assert(pindex->nTx > 0);
+            assert(pindex->GetBlockTx() > 0);
             // Assumed-valid blocks should connect to the main chain.
             assert((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TREE);
         } else {
             // Otherwise there should only be an nTx value if we have
             // actually seen a block's transactions.
-            assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // This is pruning-independent.
+            assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->GetBlockTx() > 0)); // This is pruning-independent.
         }
         // All parents having had data (at some point) is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to HaveNumChainTxs().
         assert((pindexFirstNeverProcessed == nullptr) == pindex->HaveNumChainTxs());
@@ -5115,12 +5115,12 @@ void ChainstateManager::CheckBlockIndex()
             assert((pindex->nStatus & BLOCK_FAILED_MASK) == 0); // The failed mask cannot be set for blocks without invalid parents.
         }
         // Make sure nChainTx sum is correctly computed.
-        uint64_t prev_chain_tx = pindex->pprev ? pindex->pprev->nChainTx : 0;
-        assert((pindex->nChainTx == pindex->nTx + prev_chain_tx)
+        uint64_t prev_chain_tx = pindex->pprev ? pindex->pprev->GetChainTx() : 0;
+        assert((pindex->GetChainTx() == pindex->GetBlockTx() + prev_chain_tx)
                // Transaction may be completely unset - happens if only the header was accepted but the block hasn't been processed.
-               || (pindex->nChainTx == 0 && pindex->nTx == 0)
+               || (pindex->GetChainTx() == 0 && pindex->GetBlockTx() == 0)
                // nChainTx may be unset, but nTx set (if a block has been accepted, but one of its predecessors hasn't been processed yet)
-               || (pindex->nChainTx == 0 && prev_chain_tx == 0 && pindex->pprev)
+               || (pindex->GetChainTx() == 0 && prev_chain_tx == 0 && pindex->pprev)
                // Transaction counts prior to snapshot are unknown.
                || pindex->IsAssumedValid());
         // Chainstate-specific checks on setBlockIndexCandidates
@@ -5290,13 +5290,13 @@ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex *pin
 
     double fTxTotal;
 
-    if (pindex->nChainTx <= data.nTxCount) {
+    if (pindex->GetChainTx() <= data.nTxCount) {
         fTxTotal = data.nTxCount + (nNow - data.nTime) * data.dTxRate;
     } else {
-        fTxTotal = pindex->nChainTx + (nNow - pindex->GetBlockTime()) * data.dTxRate;
+        fTxTotal = pindex->GetChainTx() + (nNow - pindex->GetBlockTime()) * data.dTxRate;
     }
 
-    return std::min<double>(pindex->nChainTx / fTxTotal, 1.0);
+    return std::min<double>(pindex->GetChainTx() / fTxTotal, 1.0);
 }
 
 std::optional<uint256> ChainstateManager::SnapshotBlockhash() const
@@ -5703,11 +5703,11 @@ bool ChainstateManager::PopulateAndValidateSnapshot(
 
         // Fake nTx so that LoadBlockIndex() loads assumed-valid CBlockIndex
         // entries (among other things)
-        if (!index->nTx) {
-            index->nTx = 1;
+        if (!index->GetBlockTx()) {
+            index->SetBlockTx(1);
         }
         // Fake nChainTx so that GuessVerificationProgress reports accurately
-        index->nChainTx = index->pprev->nChainTx + index->nTx;
+        index->SetChainTx(index->pprev->GetChainTx() + index->GetBlockTx());
 
         // Mark unvalidated block index entries beneath the snapshot base block as assumed-valid.
         if (!index->IsValid(BLOCK_VALID_SCRIPTS)) {
@@ -5731,7 +5731,7 @@ bool ChainstateManager::PopulateAndValidateSnapshot(
     }
 
     assert(index);
-    index->nChainTx = au_data.nChainTx;
+    index->SetChainTx(au_data.nChainTx);
     snapshot_chainstate.setBlockIndexCandidates.insert(snapshot_start_block);
 
     LogPrintf("[snapshot] validated snapshot (%.2f MB)\n",