Background Transaction Generation

sei-db/state_db/bench/cryptosim/block.go

@@ -0,0 +1,94 @@
+package cryptosim
+
+import "iter"
+
+// A simulated block of transactions.
+type block struct {
+	config *CryptoSimConfig
+
+	// The transactions in the block.
+	transactions []*transaction
+
+	// The block number. This is not currently preserved across benchmark restarts, but otherwise monotonically
+	// increases as you'd expect.
+	blockNumber int64
+
+	// The next account ID to be used when creating a new account, as of the end of this block.
+	nextAccountID int64
+
+	// The number of cold accounts, as of the end of this block.
+	numberOfColdAccounts int64
+
+	// The next ERC20 contract ID to be used when creating a new ERC20 contract, as of the end of this block.
+	nextErc20ContractID int64
+
+	metrics *CryptosimMetrics
+}
+
+// Creates a new block with the given capacity.
+func NewBlock(
+	config *CryptoSimConfig,
+	metrics *CryptosimMetrics,
+	blockNumber int64,
+	capacity int,
+) *block {
+	return &block{
+		config:       config,
+		blockNumber:  blockNumber,
+		transactions: make([]*transaction, 0, capacity),
+		metrics:      metrics,
+	}
+}
+
+// Returns an iterator over the transactions in the block.
+func (b *block) Iterator() iter.Seq[*transaction] {
+	return func(yield func(*transaction) bool) {
+		for _, txn := range b.transactions {
+			if !yield(txn) {
+				return
+			}
+		}
+	}
+}
+
+// Adds a transaction to the block.
+func (b *block) AddTransaction(txn *transaction) {
+	b.transactions = append(b.transactions, txn)
+}
+
+// Returns the block number.
+func (b *block) BlockNumber() int64 {
+	return b.blockNumber
+}
+
+// Sets information about account state as of the end of this block.
+func (b *block) SetBlockAccountStats(
+	nextAccountID int64,
+	numberOfColdAccounts int64,
+	nextErc20ContractID int64,
+) {
+	b.nextAccountID = nextAccountID
+	b.numberOfColdAccounts = numberOfColdAccounts
+	b.nextErc20ContractID = nextErc20ContractID
+}
+
+// This method should be called after a block is finished executing and finalized.
+// Reports metrics about the block.
+func (b *block) ReportBlockMetrics() {
+	b.metrics.SetTotalNumberOfAccounts(b.nextAccountID, int64(b.config.NumberOfHotAccounts), b.numberOfColdAccounts)
+}
+
+// Returns the next account ID to be used when creating a new account, as of the end of this block.
+func (b *block) NextAccountID() int64 {
+	return b.nextAccountID
+}
+
+// Returns the next ERC20 contract ID to be used when creating a new ERC20 contract, as of the end of this block.
+func (b *block) NextErc20ContractID() int64 {
+	return b.nextErc20ContractID
+}
+
+// Returns the number of transactions in the block.
+func (b *block) TransactionCount() int64 {
+	return int64(len(b.transactions))
+}

sei-db/state_db/bench/cryptosim/block_builder.go

@@ -0,0 +1,82 @@
+package cryptosim
+
+import (
+	"context"
+	"fmt"
+)
+
+// A builder for blocks of transactions.
+type blockBuilder struct {
+	ctx context.Context
+
+	config *CryptoSimConfig
+
+	// Metrics for the benchmark.
+	metrics *CryptosimMetrics
+
+	// Produces random data.
+	dataGenerator *DataGenerator
+
+	// Blocks are sent to this channel.
+	blocksChan chan *block
+
+	// The next block number to be used.
+	nextBlockNumber int64
+}
+
+// Asyncronously produces blocks of transactions.
+func NewBlockBuilder(
+	ctx context.Context,
+	config *CryptoSimConfig,
+	metrics *CryptosimMetrics,
+	dataGenerator *DataGenerator,
+) *blockBuilder {
+	return &blockBuilder{
+		ctx:           ctx,
+		config:        config,
+		metrics:       metrics,
+		dataGenerator: dataGenerator,
+		blocksChan:    make(chan *block, config.BlockChannelCapacity),
+	}
+}
+
+// Starts the block builder. This should not be called until all other threads are done using the data generator,
+// as the data generator is not thread-safe.
+func (b *blockBuilder) Start() {
+	go b.mainLoop()
+}
+
+// Builds blocks and sends them to the blocks channel.
+func (b *blockBuilder) mainLoop() {
+	for {
+		block := b.buildBlock()
+		select {
+		case <-b.ctx.Done():
+			return
+		case b.blocksChan <- block:
+		}
+	}
+}
+
+func (b *blockBuilder) buildBlock() *block {
+	blk := NewBlock(b.config, b.metrics, b.nextBlockNumber, b.config.TransactionsPerBlock)
+	b.nextBlockNumber++
+
+	for i := 0; i < b.config.TransactionsPerBlock; i++ {
+		txn, err := BuildTransaction(b.dataGenerator)
+		if err != nil {
+			fmt.Printf("failed to build transaction: %v\n", err)
+			continue
+		}
+		blk.AddTransaction(txn)
+	}
+
+	blk.SetBlockAccountStats(
+		b.dataGenerator.NextAccountID(),
+		b.dataGenerator.NumberOfColdAccounts(),
+		b.dataGenerator.NextErc20ContractID())
+
+	b.dataGenerator.ReportEndOfBlock()
+
+	return blk
+}

sei-db/state_db/bench/cryptosim/config/basic-config.json

@@ -11,7 +11,7 @@
   "Erc20ContractSize": 2048,
   "Erc20InteractionsPerAccount": 10,
   "Erc20StorageSlotSize": 32,
-  "ExecutorQueueSize": 64,
+  "ExecutorQueueSize": 1024,
   "HotAccountProbability": 0.1,
   "HotErc20ContractProbability": 0.5,
   "HotErc20ContractSetSize": 100,
@@ -29,5 +29,6 @@
   "TransactionsPerBlock": 1024,
   "MaxRuntimeSeconds": 0,
   "TransactionMetricsSampleRate": 0.001,
-  "BackgroundMetricsScrapeInterval": 60
+  "BackgroundMetricsScrapeInterval": 60,
+  "BlockChannelCapacity": 8
 }