Block template cache improvement (#2023)

* Block template cache improvement

* Avoid concurrent calls to template builder
* Clear cache on new block event
* Move IsNearlySynced logic to within consensus and cache it for each template 
* Use a single consensus call for building template and checking synced

Co-authored-by: msutton <mikisiton2@gmail.com>

app/protocol/flowcontext/blocks.go

@@ -70,6 +70,9 @@ func (f *FlowContext) OnVirtualChange(virtualChangeSet *externalapi.VirtualChang
 
 // OnNewBlockTemplate calls the handler function whenever a new block template is available for miners.
 func (f *FlowContext) OnNewBlockTemplate() error {
+	// Clear current template cache. Note we call this even if the handler is nil, in order to keep the
+	// state consistent without dependency on external event registration
+	f.Domain().MiningManager().ClearBlockTemplate()
 	if f.onNewBlockTemplateHandler != nil {
 		return f.onNewBlockTemplateHandler()
 	}

app/protocol/flowcontext/flow_context.go

@@ -53,9 +53,8 @@ type FlowContext struct {
 	onPruningPointUTXOSetOverrideHandler OnPruningPointUTXOSetOverrideHandler
 	onTransactionAddedToMempoolHandler   OnTransactionAddedToMempoolHandler
 
-	expectedDAAWindowDurationInMilliseconds int64
-	lastRebroadcastTime                     time.Time
-	sharedRequestedTransactions             *SharedRequestedTransactions
+	lastRebroadcastTime         time.Time
+	sharedRequestedTransactions *SharedRequestedTransactions
 
 	sharedRequestedBlocks *SharedRequestedBlocks
 
@@ -93,8 +92,6 @@ func New(cfg *config.Config, domain domain.Domain, addressManager *addressmanage
 		transactionIDsToPropagate:        []*externalapi.DomainTransactionID{},
 		lastTransactionIDPropagationTime: time.Now(),
 		shutdownChan:                     make(chan struct{}),
-		expectedDAAWindowDurationInMilliseconds: cfg.NetParams().TargetTimePerBlock.Milliseconds() *
-			int64(cfg.NetParams().DifficultyAdjustmentWindowSize),
 	}
 }
 
@@ -109,6 +106,11 @@ func (f *FlowContext) ShutdownChan() <-chan struct{} {
 	return f.shutdownChan
 }
 
+// IsNearlySynced returns whether current consensus is considered synced or close to being synced.
+func (f *FlowContext) IsNearlySynced() (bool, error) {
+	return f.Domain().Consensus().IsNearlySynced()
+}
+
 // SetOnVirtualChangeHandler sets the onVirtualChangeHandler handler
 func (f *FlowContext) SetOnVirtualChangeHandler(onVirtualChangeHandler OnVirtualChangeHandler) {
 	f.onVirtualChangeHandler = onVirtualChangeHandler

app/protocol/flowcontext/network.go

@@ -72,3 +72,10 @@ func (f *FlowContext) Peers() []*peerpkg.Peer {
 	}
 	return peers
 }
+
+// HasPeers returns whether there are currently active peers
+func (f *FlowContext) HasPeers() bool {
+	f.peersMutex.RLock()
+	defer f.peersMutex.RUnlock()
+	return len(f.peers) > 0
+}

app/protocol/manager.go

@@ -115,12 +115,6 @@ func (m *Manager) SetOnTransactionAddedToMempoolHandler(onTransactionAddedToMemp
 	m.context.SetOnTransactionAddedToMempoolHandler(onTransactionAddedToMempoolHandler)
 }
 
-// ShouldMine returns whether it's ok to use block template from this node
-// for mining purposes.
-func (m *Manager) ShouldMine() (bool, error) {
-	return m.context.IsNearlySynced()
-}
-
 // IsIBDRunning returns true if IBD is currently marked as running
 func (m *Manager) IsIBDRunning() bool {
 	return m.context.IsIBDRunning()

app/rpc/rpchandlers/get_block_template.go

@@ -28,7 +28,8 @@ func HandleGetBlockTemplate(context *rpccontext.Context, _ *router.Router, reque
 	}
 
 	coinbaseData := &externalapi.DomainCoinbaseData{ScriptPublicKey: scriptPublicKey, ExtraData: []byte(version.Version() + "/" + getBlockTemplateRequest.ExtraData)}
-	templateBlock, err := context.Domain.MiningManager().GetBlockTemplate(coinbaseData)
+
+	templateBlock, isNearlySynced, err := context.Domain.MiningManager().GetBlockTemplate(coinbaseData)
 	if err != nil {
 		return nil, err
 	}
@@ -41,10 +42,5 @@ func HandleGetBlockTemplate(context *rpccontext.Context, _ *router.Router, reque
 
 	rpcBlock := appmessage.DomainBlockToRPCBlock(templateBlock)
 
-	isSynced, err := context.ProtocolManager.ShouldMine()
-	if err != nil {
-		return nil, err
-	}
-
-	return appmessage.NewGetBlockTemplateResponseMessage(rpcBlock, isSynced), nil
+	return appmessage.NewGetBlockTemplateResponseMessage(rpcBlock, context.ProtocolManager.Context().HasPeers() && isNearlySynced), nil
 }

app/rpc/rpchandlers/submit_block.go

@@ -15,9 +15,14 @@ import (
 func HandleSubmitBlock(context *rpccontext.Context, _ *router.Router, request appmessage.Message) (appmessage.Message, error) {
 	submitBlockRequest := request.(*appmessage.SubmitBlockRequestMessage)
 
-	isSynced, err := context.ProtocolManager.ShouldMine()
-	if err != nil {
-		return nil, err
+	var err error
+	isSynced := false
+	// The node is considered synced if it has peers and consensus state is nearly synced
+	if context.ProtocolManager.Context().HasPeers() {
+		isSynced, err = context.ProtocolManager.Context().IsNearlySynced()
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	if !context.Config.AllowSubmitBlockWhenNotSynced && !isSynced {

domain/consensus/consensus.go

@@ -1,6 +1,7 @@
 package consensus
 
 import (
+	"github.com/kaspanet/kaspad/util/mstime"
 	"math/big"
 	"sync"
 
@@ -20,6 +21,8 @@ type consensus struct {
 	genesisBlock *externalapi.DomainBlock
 	genesisHash  *externalapi.DomainHash
 
+	expectedDAAWindowDurationInMilliseconds int64
+
 	blockProcessor        model.BlockProcessor
 	blockBuilder          model.BlockBuilder
 	consensusStateManager model.ConsensusStateManager
@@ -154,15 +157,31 @@ func (s *consensus) BuildBlock(coinbaseData *externalapi.DomainCoinbaseData,
 	return block, err
 }
 
-// BuildBlockWithTemplateMetadata builds a block over the current state, with the transactions
-// selected by the given transactionSelector plus metadata information related to coinbase rewards
-func (s *consensus) BuildBlockWithTemplateMetadata(coinbaseData *externalapi.DomainCoinbaseData,
-	transactions []*externalapi.DomainTransaction) (block *externalapi.DomainBlock, coinbaseHasRedReward bool, err error) {
+// BuildBlockTemplate builds a block over the current state, with the transactions
+// selected by the given transactionSelector plus metadata information related to
+// coinbase rewards and node sync status
+func (s *consensus) BuildBlockTemplate(coinbaseData *externalapi.DomainCoinbaseData,
+	transactions []*externalapi.DomainTransaction) (*externalapi.DomainBlockTemplate, error) {
 
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
-	return s.blockBuilder.BuildBlock(coinbaseData, transactions)
+	block, hasRedReward, err := s.blockBuilder.BuildBlock(coinbaseData, transactions)
+	if err != nil {
+		return nil, err
+	}
+
+	isNearlySynced, err := s.isNearlySyncedNoLock()
+	if err != nil {
+		return nil, err
+	}
+
+	return &externalapi.DomainBlockTemplate{
+		Block:                block,
+		CoinbaseData:         coinbaseData,
+		CoinbaseHasRedReward: hasRedReward,
+		IsNearlySynced:       isNearlySynced,
+	}, nil
 }
 
 // ValidateAndInsertBlock validates the given block and, if valid, applies it
@@ -894,3 +913,42 @@ func (s *consensus) VirtualMergeDepthRoot() (*externalapi.DomainHash, error) {
 	stagingArea := model.NewStagingArea()
 	return s.mergeDepthManager.VirtualMergeDepthRoot(stagingArea)
 }
+
+// IsNearlySynced returns whether this consensus is considered synced or close to being synced. This info
+// is used to determine if it's ok to use a block template from this node for mining purposes.
+func (s *consensus) IsNearlySynced() (bool, error) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	return s.isNearlySyncedNoLock()
+}
+
+func (s *consensus) isNearlySyncedNoLock() (bool, error) {
+	stagingArea := model.NewStagingArea()
+	virtualGHOSTDAGData, err := s.ghostdagDataStores[0].Get(s.databaseContext, stagingArea, model.VirtualBlockHash, false)
+	if err != nil {
+		return false, err
+	}
+
+	if virtualGHOSTDAGData.SelectedParent().Equal(s.genesisHash) {
+		return false, nil
+	}
+
+	virtualSelectedParentHeader, err := s.blockHeaderStore.BlockHeader(s.databaseContext, stagingArea, virtualGHOSTDAGData.SelectedParent())
+	if err != nil {
+		return false, err
+	}
+
+	now := mstime.Now().UnixMilliseconds()
+	// As a heuristic, we allow the node to mine if he is likely to be within the current DAA window of fully synced nodes.
+	// Such blocks contribute to security by maintaining the current difficulty despite possibly being slightly out of sync.
+	if now-virtualSelectedParentHeader.TimeInMilliseconds() < s.expectedDAAWindowDurationInMilliseconds {
+		log.Debugf("The selected tip timestamp is recent (%d), so IsNearlySynced returns true",
+			virtualSelectedParentHeader.TimeInMilliseconds())
+		return true, nil
+	}
+
+	log.Debugf("The selected tip timestamp is old (%d), so IsNearlySynced returns false",
+		virtualSelectedParentHeader.TimeInMilliseconds())
+	return false, nil
+}

domain/consensus/factory.go

@@ -472,6 +472,9 @@ func (f *factory) NewConsensus(config *Config, db infrastructuredatabase.Databas
 		genesisBlock: config.GenesisBlock,
 		genesisHash:  config.GenesisHash,
 
+		expectedDAAWindowDurationInMilliseconds: config.TargetTimePerBlock.Milliseconds() *
+			int64(config.DifficultyAdjustmentWindowSize),
+
 		blockProcessor:        blockProcessor,
 		blockBuilder:          blockBuilder,
 		consensusStateManager: consensusStateManager,

domain/consensus/model/externalapi/blocktemplate.go

@@ -5,6 +5,7 @@ type DomainBlockTemplate struct {
 	Block                *DomainBlock
 	CoinbaseData         *DomainCoinbaseData
 	CoinbaseHasRedReward bool
+	IsNearlySynced       bool
 }
 
 // Clone returns a clone of DomainBlockTemplate
@@ -13,5 +14,6 @@ func (bt *DomainBlockTemplate) Clone() *DomainBlockTemplate {
 		Block:                bt.Block.Clone(),
 		CoinbaseData:         bt.CoinbaseData.Clone(),
 		CoinbaseHasRedReward: bt.CoinbaseHasRedReward,
+		IsNearlySynced:       bt.IsNearlySynced,
 	}
 }

domain/consensus/model/externalapi/consensus.go

@@ -4,7 +4,7 @@ package externalapi
 type Consensus interface {
 	Init(skipAddingGenesis bool) error
 	BuildBlock(coinbaseData *DomainCoinbaseData, transactions []*DomainTransaction) (*DomainBlock, error)
-	BuildBlockWithTemplateMetadata(coinbaseData *DomainCoinbaseData, transactions []*DomainTransaction) (block *DomainBlock, coinbaseHasRedReward bool, err error)
+	BuildBlockTemplate(coinbaseData *DomainCoinbaseData, transactions []*DomainTransaction) (*DomainBlockTemplate, error)
 	ValidateAndInsertBlock(block *DomainBlock, shouldValidateAgainstUTXO bool) (*VirtualChangeSet, error)
 	ValidateAndInsertBlockWithTrustedData(block *BlockWithTrustedData, validateUTXO bool) (*VirtualChangeSet, error)
 	ValidateTransactionAndPopulateWithConsensusData(transaction *DomainTransaction) error
@@ -54,4 +54,5 @@ type Consensus interface {
 	TrustedGHOSTDAGData(blockHash *DomainHash) (*BlockGHOSTDAGData, error)
 	IsChainBlock(blockHash *DomainHash) (bool, error)
 	VirtualMergeDepthRoot() (*DomainHash, error)
+	IsNearlySynced() (bool, error)
 }

domain/miningmanager/blocktemplatebuilder/blocktemplatebuilder.go

@@ -142,7 +142,7 @@ func (btb *blockTemplateBuilder) BuildBlockTemplate(
 		len(candidateTxs))
 
 	blockTxs := btb.selectTransactions(candidateTxs)
-	blk, coinbaseHasRedReward, err := btb.consensusReference.Consensus().BuildBlockWithTemplateMetadata(coinbaseData, blockTxs.selectedTxs)
+	blockTemplate, err := btb.consensusReference.Consensus().BuildBlockTemplate(coinbaseData, blockTxs.selectedTxs)
 
 	invalidTxsErr := ruleerrors.ErrInvalidTransactionsInNewBlock{}
 	if errors.As(err, &invalidTxsErr) {
@@ -167,13 +167,9 @@ func (btb *blockTemplateBuilder) BuildBlockTemplate(
 	}
 
 	log.Debugf("Created new block template (%d transactions, %d in fees, %d mass, target difficulty %064x)",
-		len(blk.Transactions), blockTxs.totalFees, blockTxs.totalMass, difficulty.CompactToBig(blk.Header.Bits()))
+		len(blockTemplate.Block.Transactions), blockTxs.totalFees, blockTxs.totalMass, difficulty.CompactToBig(blockTemplate.Block.Header.Bits()))
 
-	return &consensusexternalapi.DomainBlockTemplate{
-		Block:                blk,
-		CoinbaseData:         coinbaseData,
-		CoinbaseHasRedReward: coinbaseHasRedReward,
-	}, nil
+	return blockTemplate, nil
 }
 
 // ModifyBlockTemplate modifies an existing block template to the requested coinbase data and updates the timestamp

domain/miningmanager/factory.go

@@ -27,7 +27,7 @@ func (f *factory) NewMiningManager(consensusReference consensusreference.Consens
 		consensusReference:   consensusReference,
 		mempool:              mempool,
 		blockTemplateBuilder: blockTemplateBuilder,
-		cachingTime:          time.Now(),
+		cachingTime:          time.Time{},
 		cacheLock:            &sync.Mutex{},
 	}
 }