txscript: Proactively evict SigCache entries.

This adds functionality to proactively evict SigCache entries when they
are nearly guaranteed to no longer be useful.  It accomplishes this by
evicting entries related to transactions in the block that is 2 levels
deep from a newly processed block.

Proactively evicting entries reduces the likelihood of the SigCache
reaching maximum capacity quickly and then relying on random eviction,
which may randomly evict entries that are still useful.

blockmanager.go

@@ -27,6 +27,7 @@ import (
 	"github.com/decred/dcrd/internal/mining"
 	"github.com/decred/dcrd/internal/rpcserver"
 	peerpkg "github.com/decred/dcrd/peer/v2"
+	"github.com/decred/dcrd/txscript/v3"
 	"github.com/decred/dcrd/wire"
 )
 
@@ -269,6 +270,9 @@ type blockManagerConfig struct {
 	ChainParams  *chaincfg.Params
 	SubsidyCache *standalone.SubsidyCache
 
+	// SigCache defines the signature cache to use.
+	SigCache *txscript.SigCache
+
 	// The following fields provide access to the fee estimator, mempool and
 	// the background block template generator.
 	FeeEstimator       *fees.Estimator
@@ -1163,6 +1167,9 @@ func (b *blockManager) handleBlockMsg(bmsg *blockMsg) {
 
 			// Clear the rejected transactions.
 			b.rejectedTxns = make(map[chainhash.Hash]struct{})
+
+			// Proactively evict SigCache entries.
+			b.proactivelyEvictSigCacheEntries(best.Height)
 		}
 	}
 
@@ -1232,6 +1239,26 @@ func (b *blockManager) handleBlockMsg(bmsg *blockMsg) {
 	}
 }
 
+// proactivelyEvictSigCacheEntries fetches the block that is
+// txscript.ProactiveEvictionDepth levels deep from bestHeight and passes it to
+// SigCache to evict the entries associated with the transactions in that block.
+func (b *blockManager) proactivelyEvictSigCacheEntries(bestHeight int64) {
+	// Nothing to do before the eviction depth is reached.
+	if bestHeight <= txscript.ProactiveEvictionDepth {
+		return
+	}
+
+	evictHeight := bestHeight - txscript.ProactiveEvictionDepth
+	block, err := b.cfg.Chain.BlockByHeight(evictHeight)
+	if err != nil {
+		bmgrLog.Warnf("Failed to retrieve the block at height %d: %v",
+			evictHeight, err)
+		return
+	}
+
+	b.cfg.SigCache.EvictEntries(block.MsgBlock())
+}
+
 // fetchHeaderBlocks creates and sends a request to the syncPeer for the next
 // list of blocks to be downloaded based on the current list of headers.
 func (b *blockManager) fetchHeaderBlocks() {

server.go

@@ -3251,6 +3251,7 @@ func newServer(ctx context.Context, listenAddrs []string, db database.DB, chainP
 		PeerNotifier:       &s,
 		Chain:              s.chain,
 		ChainParams:        s.chainParams,
+		SigCache:           s.sigCache,
 		SubsidyCache:       s.subsidyCache,
 		TimeSource:         s.timeSource,
 		FeeEstimator:       s.feeEstimator,

txscript/sigcache.go

@@ -17,6 +17,11 @@ import (
 	"github.com/decred/dcrd/wire"
 )
 
+// ProactiveEvictionDepth is the depth of the block at which the signatures for
+// the transactions within the block are nearly guaranteed to no longer be
+// useful.
+const ProactiveEvictionDepth = 2
+
 // shortTxHashKeySize is the size of the byte array required for key material
 // for the SipHash keyed shortTxHash function.
 const shortTxHashKeySize = 16
@@ -139,3 +144,60 @@ func shortTxHash(msg *wire.MsgTx, key [shortTxHashKeySize]byte) uint64 {
 	txHash := msg.TxHash()
 	return siphash.Hash(k0, k1, txHash[:])
 }
+
+// EvictEntries removes all entries from the SigCache that correspond to the
+// transactions in the given block.  The block that is passed should be
+// ProactiveEvictionDepth blocks deep, which is the depth at which the
+// signatures for the transactions within the block are nearly guaranteed to no
+// longer be useful.
+//
+// EvictEntries wraps the unexported evictEntries method, which is run from a
+// goroutine.  evictEntries is only invoked if validSigs is not empty.  This
+// avoids starting a new goroutine when there is nothing to evict, such as when
+// syncing is ongoing.
+func (s *SigCache) EvictEntries(block *wire.MsgBlock) {
+	s.RLock()
+	if len(s.validSigs) == 0 {
+		s.RUnlock()
+		return
+	}
+	s.RUnlock()
+
+	go s.evictEntries(block)
+}
+
+// evictEntries removes all entries from the SigCache that correspond to the
+// transactions in the given block.  The block that is passed should be
+// ProactiveEvictionDepth blocks deep, which is the depth at which the
+// signatures for the transactions within the block are nearly guaranteed to no
+// longer be useful.
+//
+// Proactively evicting entries reduces the likelihood of the SigCache reaching
+// maximum capacity quickly and then relying on random eviction, which may
+// randomly evict entries that are still useful.
+//
+// This method must be run from a goroutine and should not be run during block
+// validation.
+func (s *SigCache) evictEntries(block *wire.MsgBlock) {
+	// Create a set consisting of the short tx hashes that are in the block.
+	numTxns := len(block.Transactions) + len(block.STransactions)
+	shortTxHashSet := make(map[uint64]struct{}, numTxns)
+	for _, tx := range block.Transactions {
+		shortTxHashSet[shortTxHash(tx, s.shortTxHashKey)] = struct{}{}
+	}
+	for _, stx := range block.STransactions {
+		shortTxHashSet[shortTxHash(stx, s.shortTxHashKey)] = struct{}{}
+	}
+
+	// Iterate through the entries in validSigs and remove any that are associated
+	// with a transaction in the block.  This is done by iterating through every
+	// entry in validSigs, since the alternative of also keying the map by the
+	// shortTxHash would take extra space.
+	s.Lock()
+	for sigHash, sigEntry := range s.validSigs {
+		if _, ok := shortTxHashSet[sigEntry.shortTxHash]; ok {
+			delete(s.validSigs, sigHash)
+		}
+	}
+	s.Unlock()
+}

txscript/sigcache_test.go

@@ -6,7 +6,10 @@
 package txscript
 
 import (
+	"compress/bzip2"
 	"crypto/rand"
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/decred/dcrd/chaincfg/chainhash"
@@ -15,6 +18,27 @@ import (
 	"github.com/decred/dcrd/wire"
 )
 
+// testDataPath is the path where txscript test fixtures reside.
+const testDataPath = "data"
+
+// block432100 mocks block 432,100 of the block chain.  It is loaded and
+// deserialized immediately here and then can be used throughout the tests.
+var block432100 = func() wire.MsgBlock {
+	// Load and deserialize the test block.
+	blockDataFile := filepath.Join(testDataPath, "block432100.bz2")
+	fi, err := os.Open(blockDataFile)
+	if err != nil {
+		panic(err)
+	}
+	defer fi.Close()
+	var block wire.MsgBlock
+	err = block.Deserialize(bzip2.NewReader(fi))
+	if err != nil {
+		panic(err)
+	}
+	return block
+}()
+
 // msgTx113875_1 mocks the first transaction from block 113875.
 func msgTx113875_1() *wire.MsgTx {
 	msgTx := wire.NewMsgTx()
@@ -47,20 +71,22 @@ func msgTx113875_1() *wire.MsgTx {
 // genRandomSig returns a random message, a signature of the message under the
 // public key and the public key. This function is used to generate randomized
 // test data.
-func genRandomSig() (*chainhash.Hash, *ecdsa.Signature, *secp256k1.PublicKey, error) {
+func genRandomSig(t *testing.T) (*chainhash.Hash, *ecdsa.Signature, *secp256k1.PublicKey) {
+	t.Helper()
+
 	privKey, err := secp256k1.GeneratePrivateKey()
 	if err != nil {
-		return nil, nil, nil, err
+		t.Fatalf("error generating private key: %v", err)
 	}
 	pub := privKey.PubKey()
 
 	var msgHash chainhash.Hash
 	if _, err := rand.Read(msgHash[:]); err != nil {
-		return nil, nil, nil, err
+		t.Fatalf("error reading random hash: %v", err)
 	}
 
 	sig := ecdsa.Sign(privKey, msgHash[:])
-	return &msgHash, sig, pub, nil
+	return &msgHash, sig, pub
 }
 
 // TestSigCacheAddExists tests the ability to add, and later check the
@@ -72,10 +98,7 @@ func TestSigCacheAddExists(t *testing.T) {
 	}
 
 	// Generate a random sigCache entry triplet.
-	msg1, sig1, key1, err := genRandomSig()
-	if err != nil {
-		t.Errorf("unable to generate random signature test data")
-	}
+	msg1, sig1, key1 := genRandomSig(t)
 
 	// Add the triplet to the signature cache.
 	sigCache.Add(*msg1, sig1, key1, msgTx113875_1())
@@ -104,10 +127,7 @@ func TestSigCacheAddEvictEntry(t *testing.T) {
 
 	// Fill the sigcache up with some random sig triplets.
 	for i := uint(0); i < sigCacheSize; i++ {
-		msg, sig, key, err := genRandomSig()
-		if err != nil {
-			t.Fatalf("unable to generate random signature test data")
-		}
+		msg, sig, key := genRandomSig(t)
 
 		sigCache.Add(*msg, sig, key, tx)
 		sigCopy, _ := ecdsa.ParseDERSignature(sig.Serialize())
@@ -126,10 +146,7 @@ func TestSigCacheAddEvictEntry(t *testing.T) {
 
 	// Add a new entry, this should cause eviction of a randomly chosen
 	// previous entry.
-	msgNew, sigNew, keyNew, err := genRandomSig()
-	if err != nil {
-		t.Fatalf("unable to generate random signature test data")
-	}
+	msgNew, sigNew, keyNew := genRandomSig(t)
 	sigCache.Add(*msgNew, sigNew, keyNew, tx)
 
 	// The sigcache should still have sigCache entries.
@@ -156,10 +173,7 @@ func TestSigCacheAddMaxEntriesZeroOrNegative(t *testing.T) {
 	}
 
 	// Generate a random sigCache entry triplet.
-	msg1, sig1, key1, err := genRandomSig()
-	if err != nil {
-		t.Errorf("unable to generate random signature test data")
-	}
+	msg1, sig1, key1 := genRandomSig(t)
 
 	// Add the triplet to the signature cache.
 	sigCache.Add(*msg1, sig1, key1, msgTx113875_1())
@@ -208,3 +222,55 @@ func TestShortTxHash(t *testing.T) {
 		t.Errorf("shortTxHash: wanted different hash, but got same hash %d", got)
 	}
 }
+
+// TestEvictEntries tests that evictEntries properly removes all SigCache
+// entries related to the given block.
+func TestEvictEntries(t *testing.T) {
+	// Create a SigCache instance.
+	numTxns := len(block432100.Transactions) + len(block432100.STransactions)
+	sigCache, err := NewSigCache(uint(numTxns + 1))
+	if err != nil {
+		t.Fatalf("error creating NewSigCache: %v", err)
+	}
+
+	// Add random signatures to the SigCache for each transaction in block432100.
+	for _, tx := range block432100.Transactions {
+		msg, sig, key := genRandomSig(t)
+		sigCache.Add(*msg, sig, key, tx)
+	}
+	for _, stx := range block432100.STransactions {
+		msg, sig, key := genRandomSig(t)
+		sigCache.Add(*msg, sig, key, stx)
+	}
+
+	// Add another random signature that is not related to a transaction in
+	// block432100.
+	msg, sig, key := genRandomSig(t)
+	sigCache.Add(*msg, sig, key, msgTx113875_1())
+
+	// Validate the number of entries that should exist in the SigCache before
+	// eviction.
+	wantLength := numTxns + 1
+	gotLength := len(sigCache.validSigs)
+	if gotLength != wantLength {
+		t.Fatalf("Incorrect number of entries before eviction: "+
+			"gotLength: %d, wantLength: %d", gotLength, wantLength)
+	}
+
+	// Evict entries for block432100.
+	sigCache.evictEntries(&block432100)
+
+	// Validate that entries related to block432100 have been removed and that
+	// entries unrelated to block432100 have not been removed.
+	wantLength = 1
+	gotLength = len(sigCache.validSigs)
+	if gotLength != wantLength {
+		t.Errorf("Incorrect number of entries after eviction: "+
+			"gotLength: %d, wantLength: %d", gotLength, wantLength)
+	}
+	sigCopy, _ := ecdsa.ParseDERSignature(sig.Serialize())
+	keyCopy, _ := secp256k1.ParsePubKey(key.SerializeCompressed())
+	if !sigCache.Exists(*msg, sigCopy, keyCopy) {
+		t.Errorf("previously added item not found in signature cache")
+	}
+}