From 63637522bb0022dc2183e2c793f24c84b850bd47 Mon Sep 17 00:00:00 2001 From: Cody Littley <56973212+cody-littley@users.noreply.github.com> Date: Tue, 10 Mar 2026 15:51:11 -0500 Subject: [PATCH 1/6] Background Transaction Generation (#3021) ## Describe your changes and provide context Moved transaction generation off of the main benchmark thread for the cryptosim benchmark. ## Testing performed to validate your change Tested locally. --------- Signed-off-by: Cody Littley Co-authored-by: Cody Littley --- sei-db/state_db/bench/cryptosim/block.go | 94 ++++++++++++++ .../state_db/bench/cryptosim/block_builder.go | 82 +++++++++++++ .../bench/cryptosim/config/basic-config.json | 5 +- sei-db/state_db/bench/cryptosim/cryptosim.go | 115 +++++++++++------- .../bench/cryptosim/cryptosim_config.go | 9 +- .../bench/cryptosim/cryptosim_metrics.go | 11 +- .../bench/cryptosim/data_generator.go | 6 +- sei-db/state_db/bench/cryptosim/database.go | 21 +++- 8 files changed, 279 insertions(+), 64 deletions(-) create mode 100644 sei-db/state_db/bench/cryptosim/block.go create mode 100644 sei-db/state_db/bench/cryptosim/block_builder.go diff --git a/sei-db/state_db/bench/cryptosim/block.go b/sei-db/state_db/bench/cryptosim/block.go new file mode 100644 index 0000000000..a7833f957d --- /dev/null +++ b/sei-db/state_db/bench/cryptosim/block.go @@ -0,0 +1,94 @@ +package cryptosim + +import "iter" + +// A simulated block of transactions. +type block struct { + config *CryptoSimConfig + + // The transactions in the block. + transactions []*transaction + + // The block number. This is not currently preserved across benchmark restarts, but otherwise monotonically + // increases as you'd expect. + blockNumber int64 + + // The next account ID to be used when creating a new account, as of the end of this block. + nextAccountID int64 + + // The number of cold accounts, as of the end of this block. + numberOfColdAccounts int64 + + // The next ERC20 contract ID to be used when creating a new ERC20 contract, as of the end of this block. + nextErc20ContractID int64 + + metrics *CryptosimMetrics +} + +// Creates a new block with the given capacity. +func NewBlock( + config *CryptoSimConfig, + metrics *CryptosimMetrics, + blockNumber int64, + capacity int, +) *block { + return &block{ + config: config, + blockNumber: blockNumber, + transactions: make([]*transaction, 0, capacity), + metrics: metrics, + } +} + +// Returns an iterator over the transactions in the block. +func (b *block) Iterator() iter.Seq[*transaction] { + return func(yield func(*transaction) bool) { + for _, txn := range b.transactions { + if !yield(txn) { + return + } + } + } +} + +// Adds a transaction to the block. +func (b *block) AddTransaction(txn *transaction) { + b.transactions = append(b.transactions, txn) +} + +// Returns the block number. +func (b *block) BlockNumber() int64 { + return b.blockNumber +} + +// Sets information about account state as of the end of this block. +func (b *block) SetBlockAccountStats( + nextAccountID int64, + numberOfColdAccounts int64, + nextErc20ContractID int64, +) { + b.nextAccountID = nextAccountID + b.numberOfColdAccounts = numberOfColdAccounts + b.nextErc20ContractID = nextErc20ContractID +} + +// This method should be called after a block is finished executing and finalized. +// Reports metrics about the block. +func (b *block) ReportBlockMetrics() { + b.metrics.SetTotalNumberOfAccounts(b.nextAccountID, int64(b.config.NumberOfHotAccounts), b.numberOfColdAccounts) +} + +// Returns the next account ID to be used when creating a new account, as of the end of this block. +func (b *block) NextAccountID() int64 { + return b.nextAccountID +} + +// Returns the next ERC20 contract ID to be used when creating a new ERC20 contract, as of the end of this block. +func (b *block) NextErc20ContractID() int64 { + return b.nextErc20ContractID +} + +// Returns the number of transactions in the block. +func (b *block) TransactionCount() int64 { + return int64(len(b.transactions)) +} diff --git a/sei-db/state_db/bench/cryptosim/block_builder.go b/sei-db/state_db/bench/cryptosim/block_builder.go new file mode 100644 index 0000000000..eb36b5d281 --- /dev/null +++ b/sei-db/state_db/bench/cryptosim/block_builder.go @@ -0,0 +1,82 @@ +package cryptosim + +import ( + "context" + "fmt" +) + +// A builder for blocks of transactions. +type blockBuilder struct { + ctx context.Context + + config *CryptoSimConfig + + // Metrics for the benchmark. + metrics *CryptosimMetrics + + // Produces random data. + dataGenerator *DataGenerator + + // Blocks are sent to this channel. + blocksChan chan *block + + // The next block number to be used. + nextBlockNumber int64 +} + +// Asyncronously produces blocks of transactions. +func NewBlockBuilder( + ctx context.Context, + config *CryptoSimConfig, + metrics *CryptosimMetrics, + dataGenerator *DataGenerator, +) *blockBuilder { + return &blockBuilder{ + ctx: ctx, + config: config, + metrics: metrics, + dataGenerator: dataGenerator, + blocksChan: make(chan *block, config.BlockChannelCapacity), + } +} + +// Starts the block builder. This should not be called until all other threads are done using the data generator, +// as the data generator is not thread-safe. +func (b *blockBuilder) Start() { + go b.mainLoop() +} + +// Builds blocks and sends them to the blocks channel. +func (b *blockBuilder) mainLoop() { + for { + block := b.buildBlock() + select { + case <-b.ctx.Done(): + return + case b.blocksChan <- block: + } + } +} + +func (b *blockBuilder) buildBlock() *block { + blk := NewBlock(b.config, b.metrics, b.nextBlockNumber, b.config.TransactionsPerBlock) + b.nextBlockNumber++ + + for i := 0; i < b.config.TransactionsPerBlock; i++ { + txn, err := BuildTransaction(b.dataGenerator) + if err != nil { + fmt.Printf("failed to build transaction: %v\n", err) + continue + } + blk.AddTransaction(txn) + } + + blk.SetBlockAccountStats( + b.dataGenerator.NextAccountID(), + b.dataGenerator.NumberOfColdAccounts(), + b.dataGenerator.NextErc20ContractID()) + + b.dataGenerator.ReportEndOfBlock() + + return blk +} diff --git a/sei-db/state_db/bench/cryptosim/config/basic-config.json b/sei-db/state_db/bench/cryptosim/config/basic-config.json index dea1b2a229..53cdf98724 100644 --- a/sei-db/state_db/bench/cryptosim/config/basic-config.json +++ b/sei-db/state_db/bench/cryptosim/config/basic-config.json @@ -11,7 +11,7 @@ "Erc20ContractSize": 2048, "Erc20InteractionsPerAccount": 10, "Erc20StorageSlotSize": 32, - "ExecutorQueueSize": 64, + "ExecutorQueueSize": 1024, "HotAccountProbability": 0.1, "HotErc20ContractProbability": 0.5, "HotErc20ContractSetSize": 100, @@ -29,5 +29,6 @@ "TransactionsPerBlock": 1024, "MaxRuntimeSeconds": 0, "TransactionMetricsSampleRate": 0.001, - "BackgroundMetricsScrapeInterval": 60 + "BackgroundMetricsScrapeInterval": 60, + "BlockChannelCapacity": 8 } diff --git a/sei-db/state_db/bench/cryptosim/cryptosim.go b/sei-db/state_db/bench/cryptosim/cryptosim.go index 04fc441b1a..94fdd02d3a 100644 --- a/sei-db/state_db/bench/cryptosim/cryptosim.go +++ b/sei-db/state_db/bench/cryptosim/cryptosim.go @@ -48,6 +48,9 @@ type CryptoSim struct { // The data generator for the benchmark. dataGenerator *DataGenerator + // Builds blocks of transactions. + blockBuilder *blockBuilder + // The database for the benchmark. database *Database @@ -64,6 +67,14 @@ type CryptoSim struct { // benchmark, sending "false" will resume it. Suspending an already suspended benchmark will have no effect, // and resuming an already resumed benchmark will likewise have no effect. suspendChan chan bool + + // The most recent block that has been processed. + mostRecentBlock *block + + // The next ERC20 contract ID to be used when creating a new ERC20 contract. + // This is fixed after initial setup is complete, since we don't currently simulate + // the creation of new ERC20 contracts during the benchmark. + nextERC20ContractID int64 } // Creates a new cryptosim benchmark runner. @@ -142,6 +153,8 @@ func NewCryptoSim( ctx, cancel, database, dataGenerator.FeeCollectionAddress(), config.ExecutorQueueSize, metrics) } + blockBuilder := NewBlockBuilder(ctx, config, metrics, dataGenerator) + c := &CryptoSim{ ctx: ctx, cancel: cancel, @@ -151,6 +164,7 @@ func NewCryptoSim( lastConsoleUpdateTransactionCount: 0, closeChan: make(chan struct{}, 1), dataGenerator: dataGenerator, + blockBuilder: blockBuilder, database: database, executors: executors, metrics: metrics, @@ -166,7 +180,10 @@ func NewCryptoSim( c.database.ResetTransactionCount() c.startTimestamp = time.Now() - c.metrics.StartBackgroundSampling(c.startTimestamp) + + // Now that we are done generating initial data, it is thread safe to start the block builder. + // (dataGenerator is not thread safe, and is used both for initial setup and for transaction generation) + c.blockBuilder.Start() go c.run() return c, nil @@ -210,7 +227,7 @@ func (c *CryptoSim) setupAccounts() error { if err != nil { return fmt.Errorf("failed to create new account: %w", err) } - c.database.IncrementTransactionCount() + c.database.IncrementTransactionCount(1) finalized, err := c.database.MaybeFinalizeBlock( c.dataGenerator.NextAccountID(), c.dataGenerator.NextErc20ContractID()) if err != nil { @@ -218,7 +235,7 @@ func (c *CryptoSim) setupAccounts() error { } if finalized { c.dataGenerator.ReportAccountCounts() - c.dataGenerator.ReportFinalizeBlock() + c.dataGenerator.ReportEndOfBlock() } if c.dataGenerator.NextAccountID()%c.config.SetupUpdateIntervalCount == 0 { @@ -237,7 +254,7 @@ func (c *CryptoSim) setupAccounts() error { return fmt.Errorf("failed to finalize block: %w", err) } c.dataGenerator.ReportAccountCounts() - c.dataGenerator.ReportFinalizeBlock() + c.dataGenerator.ReportEndOfBlock() fmt.Printf("There are now %s accounts in the database.\n", int64Commas(c.dataGenerator.NextAccountID())) @@ -267,7 +284,7 @@ func (c *CryptoSim) setupErc20Contracts() error { break } - c.database.IncrementTransactionCount() + c.database.IncrementTransactionCount(1) _, _, err := c.dataGenerator.CreateNewErc20Contract(c.config.Erc20ContractSize, true) if err != nil { @@ -279,7 +296,7 @@ func (c *CryptoSim) setupErc20Contracts() error { return fmt.Errorf("failed to maybe commit batch: %w", err) } if finalized { - c.dataGenerator.ReportFinalizeBlock() + c.dataGenerator.ReportEndOfBlock() c.metrics.SetTotalNumberOfERC20Contracts(c.dataGenerator.NextErc20ContractID()) } @@ -301,12 +318,14 @@ func (c *CryptoSim) setupErc20Contracts() error { if err != nil { return fmt.Errorf("failed to finalize block: %w", err) } - c.dataGenerator.ReportFinalizeBlock() + c.dataGenerator.ReportEndOfBlock() c.metrics.SetTotalNumberOfERC20Contracts(c.dataGenerator.NextErc20ContractID()) fmt.Printf("There are now %s simulated ERC20 contracts in the database.\n", int64Commas(c.dataGenerator.NextErc20ContractID())) + c.nextERC20ContractID = c.dataGenerator.NextErc20ContractID() + return nil } @@ -316,10 +335,14 @@ func (c *CryptoSim) run() { defer c.teardown() haltTime := time.Now().Add(time.Duration(c.config.MaxRuntimeSeconds) * time.Second) - - c.metrics.SetMainThreadPhase("executing") + var timeoutChan <-chan time.Time + if c.config.MaxRuntimeSeconds > 0 { + timeoutChan = time.After(time.Until(haltTime)) + } for { + c.metrics.SetMainThreadPhase("get_block") + select { case <-c.ctx.Done(): if c.database.TransactionCount() > 0 { @@ -331,55 +354,52 @@ func (c *CryptoSim) run() { if isSuspended { c.suspend() } - default: - c.handleNextCycle(haltTime) + case <-timeoutChan: + fmt.Printf("\nBenchmark timed out after %s.\n", formatDuration(time.Since(c.startTimestamp), 1)) + c.cancel() + return + case blk := <-c.blockBuilder.blocksChan: + c.handleNextBlock(blk) } + + c.generateConsoleReport(false) } } -// Process the next benchmark cycle, creating a new transaction and executing it. -func (c *CryptoSim) handleNextCycle(haltTime time.Time) { - txn, err := BuildTransaction(c.dataGenerator) - if err != nil { - fmt.Printf("\nfailed to build transaction: %v\n", err) - c.cancel() - return - } +// Execute and finalize the next block. +func (c *CryptoSim) handleNextBlock(blk *block) { + c.mostRecentBlock = blk + c.metrics.SetMainThreadPhase("send_to_executors") - c.executors[c.nextExecutorIndex].ScheduleForExecution(txn) - c.nextExecutorIndex = (c.nextExecutorIndex + 1) % len(c.executors) + c.database.IncrementTransactionCount(blk.TransactionCount()) - finalized, err := c.database.MaybeFinalizeBlock( - c.dataGenerator.NextAccountID(), c.dataGenerator.NextErc20ContractID()) - if err != nil { - fmt.Printf("error finalizing block: %v\n", err) - c.cancel() - return + for txn := range blk.Iterator() { + c.executors[c.nextExecutorIndex].ScheduleForExecution(txn) + c.nextExecutorIndex = (c.nextExecutorIndex + 1) % len(c.executors) } - if finalized { - c.dataGenerator.ReportAccountCounts() - c.dataGenerator.ReportFinalizeBlock() - if c.config.MaxRuntimeSeconds > 0 && time.Now().After(haltTime) { - c.cancel() - } + if err := c.database.FinalizeBlock(blk.NextAccountID(), blk.NextErc20ContractID(), false); err != nil { + fmt.Printf("failed to finalize block: %v\n", err) + c.cancel() + return } - - c.database.IncrementTransactionCount() - c.generateConsoleReport(false) + blk.ReportBlockMetrics() } // Suspends the benchmark. This method blocks until the benchmark is resumed or shut down. func (c *CryptoSim) suspend() { - err := c.database.FinalizeBlock(c.dataGenerator.NextAccountID(), c.dataGenerator.NextErc20ContractID(), true) - if err != nil { - fmt.Printf("failed to finalize block: %v\n", err) - c.cancel() - return + if c.mostRecentBlock != nil { + err := c.database.FinalizeBlock(c.mostRecentBlock.nextAccountID, c.nextERC20ContractID, true) + if err != nil { + fmt.Printf("failed to finalize block: %v\n", err) + c.cancel() + return + } } fmt.Printf("Benchmark suspended.\n") + c.metrics.SetMainThreadPhase("suspended") for { select { @@ -403,9 +423,16 @@ func (c *CryptoSim) suspend() { // Clean up the benchmark and release any resources. func (c *CryptoSim) teardown() { - err := c.database.Close(c.dataGenerator.NextAccountID(), c.dataGenerator.NextErc20ContractID()) - if err != nil { - fmt.Printf("failed to close database: %v\n", err) + if c.mostRecentBlock == nil { + err := c.database.CloseWithoutFinalizing() + if err != nil { + fmt.Printf("failed to close database: %v\n", err) + } + } else { + err := c.database.Close(c.mostRecentBlock.nextAccountID, c.nextERC20ContractID) + if err != nil { + fmt.Printf("failed to close database: %v\n", err) + } } c.dataGenerator.Close() diff --git a/sei-db/state_db/bench/cryptosim/cryptosim_config.go b/sei-db/state_db/bench/cryptosim/cryptosim_config.go index a33e5a42ff..a6434c269e 100644 --- a/sei-db/state_db/bench/cryptosim/cryptosim_config.go +++ b/sei-db/state_db/bench/cryptosim/cryptosim_config.go @@ -132,6 +132,9 @@ type CryptoSimConfig struct { // If true, pressing Enter in the terminal will toggle suspend/resume of the benchmark. // If false, Enter has no effect. EnableSuspension bool + + // The capacity of the channel that holds blocks awaiting execution. + BlockChannelCapacity int } // Returns the default configuration for the cryptosim benchmark. @@ -164,12 +167,13 @@ func DefaultCryptoSimConfig() *CryptoSimConfig { SetupUpdateIntervalCount: 100_000, ThreadsPerCore: 2.0, ConstantThreadCount: 0, - ExecutorQueueSize: 64, + ExecutorQueueSize: 1024, MaxRuntimeSeconds: 0, MetricsAddr: ":9090", TransactionMetricsSampleRate: 0.001, BackgroundMetricsScrapeInterval: 60, EnableSuspension: true, + BlockChannelCapacity: 8, } } @@ -242,6 +246,9 @@ func (c *CryptoSimConfig) Validate() error { if c.BackgroundMetricsScrapeInterval < 0 { return fmt.Errorf("BackgroundMetricsScrapeInterval must be non-negative (got %d)", c.BackgroundMetricsScrapeInterval) } + if c.BlockChannelCapacity < 1 { + return fmt.Errorf("BlockChannelCapacity must be at least 1 (got %d)", c.BlockChannelCapacity) + } return nil } diff --git a/sei-db/state_db/bench/cryptosim/cryptosim_metrics.go b/sei-db/state_db/bench/cryptosim/cryptosim_metrics.go index a25ac9ebc6..c5b16408b8 100644 --- a/sei-db/state_db/bench/cryptosim/cryptosim_metrics.go +++ b/sei-db/state_db/bench/cryptosim/cryptosim_metrics.go @@ -172,20 +172,11 @@ func NewCryptosimMetrics( m.startDataDirSizeSampling(dataDir, config.BackgroundMetricsScrapeInterval) } m.startProcessIOSampling(config.BackgroundMetricsScrapeInterval) + m.startUptimeSampling(time.Now()) } return m } -// StartBackgroundSampling starts goroutines that periodically update gauges -// (uptime, etc.). Call this when the benchmark is about to run, after initial -// state is loaded. Does not start any HTTP server; the caller configures export. -func (m *CryptosimMetrics) StartBackgroundSampling(startTime time.Time) { - if m == nil { - return - } - m.startUptimeSampling(startTime) -} - func (m *CryptosimMetrics) startUptimeSampling(startTime time.Time) { if m == nil || m.uptimeSeconds == nil { return diff --git a/sei-db/state_db/bench/cryptosim/data_generator.go b/sei-db/state_db/bench/cryptosim/data_generator.go index 5920366814..8347d6bd2d 100644 --- a/sei-db/state_db/bench/cryptosim/data_generator.go +++ b/sei-db/state_db/bench/cryptosim/data_generator.go @@ -138,7 +138,6 @@ func (d *DataGenerator) CreateNewAccount( accountID := d.nextAccountID d.nextAccountID++ - // Use EVMKeyCode for account data (balance+padding); EVMKeyNonce only accepts 8-byte values. addr := d.rand.Address(accountPrefix, accountID, AddressLen) address = evm.BuildMemIAVLEVMKey(evm.EVMKeyCode, addr) @@ -289,7 +288,8 @@ func (d *DataGenerator) FeeCollectionAddress() []byte { return d.feeCollectionAddress } -// This method should be called after a block is finalized. -func (d *DataGenerator) ReportFinalizeBlock() { +// Call this to signal that we have reached the end of a block. This is a signal that it is now safe to use +// recently created accounts as read/write targets. +func (d *DataGenerator) ReportEndOfBlock() { d.highestSafeAccountIDInBlock = d.nextAccountID - 1 } diff --git a/sei-db/state_db/bench/cryptosim/database.go b/sei-db/state_db/bench/cryptosim/database.go index 2b723fbdbc..21d9a48937 100644 --- a/sei-db/state_db/bench/cryptosim/database.go +++ b/sei-db/state_db/bench/cryptosim/database.go @@ -80,10 +80,10 @@ func (d *Database) Get(key []byte) ([]byte, bool, error) { return nil, false, nil } -// Signal that a transaction has been executed. -func (d *Database) IncrementTransactionCount() { - d.transactionCount++ - d.transactionsInCurrentBlock++ +// Signal that transactions have been added to the current block. +func (d *Database) IncrementTransactionCount(count int64) { + d.transactionCount += count + d.transactionsInCurrentBlock += count } // Reset the transaction count. Useful for when changing test phases. @@ -120,6 +120,8 @@ func (d *Database) FinalizeBlock( forceCommit bool, ) error { + d.metrics.SetMainThreadPhase("execute_block") + // Wait for all transactions in the current block to be executed. if d.flushFunc != nil { d.flushFunc() @@ -204,6 +206,17 @@ func (d *Database) Close(nextAccountID int64, nextErc20ContractID int64) error { return nil } +// Close the database and release any resources without finalizing the last batch. +func (d *Database) CloseWithoutFinalizing() error { + fmt.Printf("Closing database.\n") + err := d.db.Close() + if err != nil { + return fmt.Errorf("failed to close database: %w", err) + } + + return nil +} + // Set the function that flushes the executors. This setter is required to break a circular dependency. func (d *Database) SetFlushFunc(flushFunc func()) { d.flushFunc = flushFunc From 91d9968112b0e8bd7f63dae1ecec4cf2b9d79993 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 13 Mar 2026 10:00:03 -0400 Subject: [PATCH 2/6] Restore fix(giga): check whether txs follow Giga ordering (#2810) This commit was inadvertently reverted by the squash-merge of #3039. Restores the tx ordering check (firstCosmosSeen), len(evmEntries)>0 and len(v2Entries)>0 guards in ProcessTXsWithOCCGiga. Conflicts with #3050's slog migration resolved by using package-level logger instead of ctx.Logger(). Made-with: Cursor --- app/app.go | 116 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 67 insertions(+), 49 deletions(-) diff --git a/app/app.go b/app/app.go index 75f8c4ca8f..338891e445 100644 --- a/app/app.go +++ b/app/app.go @@ -758,7 +758,7 @@ func New( if gigaExecutorConfig.OCCEnabled { logger.Info("benchmark: Giga Executor with OCC is ENABLED - using new EVM execution path with parallel execution") } else { - logger.Info("benchmark: Giga Executor (evmone-based) is ENABLED - using new EVM execution path (sequential)") + logger.Info("benchmark: Giga Executor is ENABLED - using new EVM execution path (sequential)") } } else { logger.Info("benchmark: Giga Executor is DISABLED - using default GETH interpreter") @@ -1560,71 +1560,89 @@ func (app *App) ProcessTXsWithOCCV2(ctx sdk.Context, txs [][]byte, typedTxs []sd func (app *App) ProcessTXsWithOCCGiga(ctx sdk.Context, txs [][]byte, typedTxs []sdk.Tx) ([]*abci.ExecTxResult, sdk.Context) { evmEntries := make([]*sdk.DeliverTxEntry, 0, len(txs)) v2Entries := make([]*sdk.DeliverTxEntry, 0, len(txs)) + firstCosmosSeen := false for txIndex, tx := range txs { if app.GetEVMMsg(typedTxs[txIndex]) != nil { + if firstCosmosSeen { + logger.Error("Giga OCC cannot execute block due to tx ordering, falling back to V2") + // Oops! This isn't "all EVM txs, then all Cosmos txs" - we need to fallback to V2. + return app.ProcessTXsWithOCCV2(ctx, txs, typedTxs) + } + evmEntries = append(evmEntries, app.GetDeliverTxEntry(ctx, txIndex, tx, typedTxs[txIndex])) } else { + if !firstCosmosSeen { + firstCosmosSeen = true + } v2Entries = append(v2Entries, app.GetDeliverTxEntry(ctx, txIndex, tx, typedTxs[txIndex])) } } - // Run EVM txs against a cache so we can discard all changes on fallback. - evmCtx, evmCache := app.CacheContext(ctx) + var evmBatchResult []abci.ResponseDeliverTx + fallbackToV2 := false - // Cache block-level constants (identical for all txs in this block). - // Must use evmCtx (not ctx) because giga KV stores are registered in CacheContext. - cache, cacheErr := newGigaBlockCache(evmCtx, &app.GigaEvmKeeper) - if cacheErr != nil { - logger.Error("failed to build giga block cache", "error", cacheErr, "height", ctx.BlockHeight()) - return nil, ctx - } + if len(evmEntries) > 0 { + // Run EVM txs against a cache so we can discard all changes on fallback. + evmCtx, evmCache := app.CacheContext(ctx) - // Create OCC scheduler with giga executor deliverTx capturing the cache. - evmScheduler := tasks.NewScheduler( - app.ConcurrencyWorkers(), - app.TracingInfo, - app.makeGigaDeliverTx(cache), - ) + // Cache block-level constants (identical for all txs in this block). + // Must use evmCtx (not ctx) because giga KV stores are registered in CacheContext. + cache, cacheErr := newGigaBlockCache(evmCtx, &app.GigaEvmKeeper) + if cacheErr != nil { + logger.Error("failed to build giga block cache", "error", cacheErr, "height", ctx.BlockHeight()) + return nil, ctx + } - evmBatchResult, evmSchedErr := evmScheduler.ProcessAll(evmCtx, evmEntries) - if evmSchedErr != nil { - // TODO: DeliverTxBatch panics in this case - // TODO: detect if it was interop, and use v2 if so - logger.Error("benchmark OCC scheduler error (EVM txs)", "error", evmSchedErr, "height", ctx.BlockHeight(), "txCount", len(evmEntries)) - return nil, ctx - } + // Create OCC scheduler with giga executor deliverTx capturing the cache. + evmScheduler := tasks.NewScheduler( + app.ConcurrencyWorkers(), + app.TracingInfo, + app.makeGigaDeliverTx(cache), + ) - fallbackToV2 := false - for _, r := range evmBatchResult { - if r.Code == gigautils.GigaAbortCode && r.Codespace == gigautils.GigaAbortCodespace { - fallbackToV2 = true - break + var evmSchedErr error + evmBatchResult, evmSchedErr = evmScheduler.ProcessAll(evmCtx, evmEntries) + if evmSchedErr != nil { + logger.Error("benchmark OCC scheduler error (EVM txs)", "error", evmSchedErr, "height", ctx.BlockHeight(), "txCount", len(evmEntries)) + return nil, ctx } - } - if fallbackToV2 { - metrics.IncrGigaFallbackToV2Counter() - // Discard all EVM changes by skipping cache writes, then re-run all txs via DeliverTx. - evmBatchResult = nil - v2Entries = make([]*sdk.DeliverTxEntry, len(txs)) - for txIndex, tx := range txs { - v2Entries[txIndex] = app.GetDeliverTxEntry(ctx, txIndex, tx, typedTxs[txIndex]) + for _, r := range evmBatchResult { + if r.Code == gigautils.GigaAbortCode && r.Codespace == gigautils.GigaAbortCodespace { + fallbackToV2 = true + break + } + } + + if fallbackToV2 { + metrics.IncrGigaFallbackToV2Counter() + // Discard all EVM changes by skipping cache writes, then re-run all txs via DeliverTx. + evmBatchResult = nil + v2Entries = make([]*sdk.DeliverTxEntry, len(txs)) + for txIndex, tx := range txs { + v2Entries[txIndex] = app.GetDeliverTxEntry(ctx, txIndex, tx, typedTxs[txIndex]) + } + } else { + // Commit EVM cache to main store before processing non-EVM txs. + evmCache.Write() + evmCtx.GigaMultiStore().WriteGiga() } - } else { - // Commit EVM cache to main store before processing non-EVM txs. - evmCache.Write() - evmCtx.GigaMultiStore().WriteGiga() } - v2Scheduler := tasks.NewScheduler( - app.ConcurrencyWorkers(), - app.TracingInfo, - app.DeliverTx, - ) - v2BatchResult, v2SchedErr := v2Scheduler.ProcessAll(ctx, v2Entries) - if v2SchedErr != nil { - logger.Error("benchmark OCC scheduler error", "error", v2SchedErr, "height", ctx.BlockHeight(), "txCount", len(v2Entries)) - return nil, ctx + var v2BatchResult []abci.ResponseDeliverTx + + if len(v2Entries) > 0 { + v2Scheduler := tasks.NewScheduler( + app.ConcurrencyWorkers(), + app.TracingInfo, + app.DeliverTx, + ) + var v2SchedErr error + v2BatchResult, v2SchedErr = v2Scheduler.ProcessAll(ctx, v2Entries) + if v2SchedErr != nil { + logger.Error("benchmark OCC scheduler error", "error", v2SchedErr, "height", ctx.BlockHeight(), "txCount", len(v2Entries)) + return nil, ctx + } } execResults := make([]*abci.ExecTxResult, 0, len(evmBatchResult)+len(v2BatchResult)) From 389a409e6d56f7118c5deb98ed8791b8c9587416 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 13 Mar 2026 10:02:09 -0400 Subject: [PATCH 3/6] Restore receiptdb config option in app.toml (#3035) This commit was inadvertently reverted by the squash-merge of #3039. Restores ReceiptStoreConfig in app.toml, readReceiptStoreConfig(), BackendTypeName(), receipt config tests, and init_test.go. Conflicts resolved: - app_config.go: kept both ReceiptStore (from #3035) and Admin (from #3062) - receipt_store.go: kept BackendTypeName but used #3050's slog convention - parquet_store_test.go: kept #3053's deterministic pruning test fix Made-with: Cursor --- app/app.go | 10 +-- app/receipt_store_config.go | 26 ++++++ app/seidb_test.go | 81 +++++++++++++++++ app/test_helpers.go | 59 ++++++++++++- cmd/seid/cmd/app_config.go | 28 +++--- cmd/seid/cmd/init_test.go | 33 +++++++ cmd/seid/cmd/root.go | 1 + sei-db/config/receipt_config.go | 86 ++++++++++++++++--- sei-db/config/receipt_config_test.go | 23 +++++ sei-db/config/toml.go | 18 ++++ sei-db/config/toml_test.go | 33 +++++++ sei-db/ledger_db/parquet/store_config_test.go | 13 +++ .../ledger_db/receipt/parquet_store_test.go | 25 ++++++ sei-db/ledger_db/receipt/receipt_store.go | 20 +++++ 14 files changed, 424 insertions(+), 32 deletions(-) create mode 100644 app/receipt_store_config.go create mode 100644 sei-db/config/receipt_config_test.go diff --git a/app/app.go b/app/app.go index 338891e445..e9e645b2f9 100644 --- a/app/app.go +++ b/app/app.go @@ -36,7 +36,6 @@ import ( "github.com/sei-protocol/sei-chain/sei-cosmos/codec" "github.com/sei-protocol/sei-chain/sei-cosmos/codec/types" cryptotypes "github.com/sei-protocol/sei-chain/sei-cosmos/crypto/types" - "github.com/sei-protocol/sei-chain/sei-cosmos/server" "github.com/sei-protocol/sei-chain/sei-cosmos/server/api" "github.com/sei-protocol/sei-chain/sei-cosmos/server/config" servertypes "github.com/sei-protocol/sei-chain/sei-cosmos/server/types" @@ -121,7 +120,6 @@ import ( gigautils "github.com/sei-protocol/sei-chain/giga/executor/utils" "github.com/sei-protocol/sei-chain/precompiles" putils "github.com/sei-protocol/sei-chain/precompiles/utils" - ssconfig "github.com/sei-protocol/sei-chain/sei-db/config" "github.com/sei-protocol/sei-chain/sei-ibc-go/modules/apps/transfer" ibctransferkeeper "github.com/sei-protocol/sei-chain/sei-ibc-go/modules/apps/transfer/keeper" ibctransfertypes "github.com/sei-protocol/sei-chain/sei-ibc-go/modules/apps/transfer/types" @@ -683,10 +681,10 @@ func New( wasmOpts..., ) - receiptStorePath := filepath.Join(homePath, "data", "receipt.db") - receiptConfig := ssconfig.DefaultReceiptStoreConfig() - receiptConfig.DBDirectory = receiptStorePath - receiptConfig.KeepRecent = cast.ToInt(appOpts.Get(server.FlagMinRetainBlocks)) + receiptConfig, err := readReceiptStoreConfig(homePath, appOpts) + if err != nil { + panic(fmt.Sprintf("error reading receipt store config: %s", err)) + } if app.receiptStore == nil { receiptStore, err := receipt.NewReceiptStore(receiptConfig, keys[evmtypes.StoreKey]) if err != nil { diff --git a/app/receipt_store_config.go b/app/receipt_store_config.go new file mode 100644 index 0000000000..8d2acfacf2 --- /dev/null +++ b/app/receipt_store_config.go @@ -0,0 +1,26 @@ +package app + +import ( + "path/filepath" + + seidbconfig "github.com/sei-protocol/sei-chain/sei-db/config" +) + +const ( + receiptStoreBackendKey = "receipt-store.rs-backend" + receiptStoreDBDirectoryKey = "receipt-store.db-directory" + receiptStoreAsyncWriteBufferKey = "receipt-store.async-write-buffer" + receiptStoreKeepRecentKey = "receipt-store.keep-recent" + receiptStorePruneIntervalSecondsKey = "receipt-store.prune-interval-seconds" +) + +func readReceiptStoreConfig(homePath string, appOpts seidbconfig.AppOptions) (seidbconfig.ReceiptStoreConfig, error) { + receiptConfig, err := seidbconfig.ReadReceiptConfig(appOpts) + if err != nil { + return receiptConfig, err + } + if receiptConfig.DBDirectory == "" { + receiptConfig.DBDirectory = filepath.Join(homePath, "data", "receipt.db") + } + return receiptConfig, nil +} diff --git a/app/seidb_test.go b/app/seidb_test.go index 94c7db5257..fa91e1436e 100644 --- a/app/seidb_test.go +++ b/app/seidb_test.go @@ -1,10 +1,14 @@ package app import ( + "path/filepath" "testing" + "github.com/sei-protocol/sei-chain/sei-cosmos/server" "github.com/sei-protocol/sei-chain/sei-db/config" + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/receipt" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type TestSeiDBAppOpts struct { @@ -13,6 +17,7 @@ type TestSeiDBAppOpts struct { func (t TestSeiDBAppOpts) Get(s string) interface{} { defaultSCConfig := config.DefaultStateCommitConfig() defaultSSConfig := config.DefaultStateStoreConfig() + defaultReceiptConfig := config.DefaultReceiptStoreConfig() switch s { case FlagSCEnable: return defaultSCConfig.Enable @@ -46,6 +51,8 @@ func (t TestSeiDBAppOpts) Get(s string) interface{} { return defaultSSConfig.PruneIntervalSeconds case FlagSSImportNumWorkers: return defaultSSConfig.ImportNumWorkers + case receiptStoreBackendKey: + return defaultReceiptConfig.Backend case FlagEVMSSDirectory: return defaultSSConfig.EVMDBDirectory case FlagEVMSSWriteMode: @@ -61,8 +68,11 @@ func TestNewDefaultConfig(t *testing.T) { appOpts := TestSeiDBAppOpts{} scConfig := parseSCConfigs(appOpts) ssConfig := parseSSConfigs(appOpts) + receiptConfig, err := config.ReadReceiptConfig(appOpts) + assert.NoError(t, err) assert.Equal(t, scConfig, config.DefaultStateCommitConfig()) assert.Equal(t, ssConfig, config.DefaultStateStoreConfig()) + assert.Equal(t, receiptConfig, config.DefaultReceiptStoreConfig()) } type mapAppOpts map[string]interface{} @@ -85,3 +95,74 @@ func TestParseSCConfigs_HistoricalProofFlags(t *testing.T) { assert.Equal(t, 12.5, scConfig.HistoricalProofRateLimit) assert.Equal(t, 3, scConfig.HistoricalProofBurst) } + +func TestParseReceiptConfigs_DefaultsToPebbleWhenUnset(t *testing.T) { + receiptConfig, err := config.ReadReceiptConfig(mapAppOpts{}) + assert.NoError(t, err) + assert.Equal(t, config.DefaultReceiptStoreConfig(), receiptConfig) +} + +func TestParseReceiptConfigs_UsesConfiguredBackend(t *testing.T) { + receiptConfig, err := config.ReadReceiptConfig(mapAppOpts{ + receiptStoreBackendKey: "parquet", + }) + assert.NoError(t, err) + assert.Equal(t, "parquet", receiptConfig.Backend) + assert.Equal(t, config.DefaultReceiptStoreConfig().AsyncWriteBuffer, receiptConfig.AsyncWriteBuffer) + assert.Equal(t, config.DefaultReceiptStoreConfig().KeepRecent, receiptConfig.KeepRecent) +} + +func TestParseReceiptConfigs_UsesConfiguredValues(t *testing.T) { + receiptConfig, err := config.ReadReceiptConfig(mapAppOpts{ + receiptStoreDBDirectoryKey: "/tmp/custom-receipt-db", + receiptStoreBackendKey: "parquet", + receiptStoreAsyncWriteBufferKey: 7, + receiptStoreKeepRecentKey: 42, + receiptStorePruneIntervalSecondsKey: 9, + }) + assert.NoError(t, err) + assert.Equal(t, "/tmp/custom-receipt-db", receiptConfig.DBDirectory) + assert.Equal(t, "parquet", receiptConfig.Backend) + assert.Equal(t, 7, receiptConfig.AsyncWriteBuffer) + assert.Equal(t, 42, receiptConfig.KeepRecent) + assert.Equal(t, 9, receiptConfig.PruneIntervalSeconds) +} + +func TestParseReceiptConfigs_RejectsInvalidBackend(t *testing.T) { + _, err := config.ReadReceiptConfig(mapAppOpts{ + receiptStoreBackendKey: "rocksdb", + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "unsupported receipt-store backend") + assert.Contains(t, err.Error(), "rocksdb") +} + +func TestReadReceiptStoreConfigUsesConfiguredValues(t *testing.T) { + homePath := t.TempDir() + receiptConfig, err := readReceiptStoreConfig(homePath, mapAppOpts{ + receiptStoreDBDirectoryKey: "/tmp/custom-receipt-db", + receiptStoreKeepRecentKey: 5, + server.FlagMinRetainBlocks: 100, + }) + require.NoError(t, err) + assert.Equal(t, "/tmp/custom-receipt-db", receiptConfig.DBDirectory) + assert.Equal(t, 5, receiptConfig.KeepRecent) +} + +func TestReadReceiptStoreConfigUsesDefaultDirectoryWhenUnset(t *testing.T) { + homePath := t.TempDir() + receiptConfig, err := readReceiptStoreConfig(homePath, mapAppOpts{}) + require.NoError(t, err) + assert.Equal(t, filepath.Join(homePath, "data", "receipt.db"), receiptConfig.DBDirectory) +} + +// TestFullAppPathWithParquetReceiptStore exercises the full app.New path with rs-backend = "parquet" +// and asserts the parquet receipt store is actually instantiated (not pebble). +func TestFullAppPathWithParquetReceiptStore(t *testing.T) { + app := SetupWithScReceiptFromOpts(t, false, false, TestAppOpts{ + UseSc: true, + ReceiptBackend: "parquet", + }) + require.NotNil(t, app.receiptStore, "receipt store should be created") + assert.Equal(t, "parquet", receipt.BackendTypeName(app.receiptStore), "receipt store backend should be parquet") +} diff --git a/app/test_helpers.go b/app/test_helpers.go index a080373bae..f5cef8f22c 100644 --- a/app/test_helpers.go +++ b/app/test_helpers.go @@ -74,9 +74,10 @@ func (t TestTx) GetGasEstimate() uint64 { } type TestAppOpts struct { - UseSc bool - EnableGiga bool - EnableGigaOCC bool + UseSc bool + EnableGiga bool + EnableGigaOCC bool + ReceiptBackend string // e.g. "parquet" to use parquet receipt store; empty = default (pebble) } func (t TestAppOpts) Get(s string) interface{} { @@ -97,6 +98,9 @@ func (t TestAppOpts) Get(s string) interface{} { if s == gigaconfig.FlagOCCEnabled { return t.EnableGigaOCC } + if s == receiptStoreBackendKey && t.ReceiptBackend != "" { + return t.ReceiptBackend + } // Disable EVM HTTP and WebSocket servers in tests to avoid port conflicts // when multiple tests run in parallel (all would try to bind to port 8545) if s == "evm.http_enabled" || s == "evm.ws_enabled" { @@ -468,6 +472,55 @@ func SetupWithDB(tb testing.TB, db dbm.DB, isCheckTx bool, enableEVMCustomPrecom return res } +// SetupWithScReceiptFromOpts is like SetupWithSc but does not inject a receipt store via AppOption. +// The receipt store is created inside New() from testAppOpts (e.g. testAppOpts.ReceiptBackend = "parquet"). +// Use this to test the full app path with rs-backend from config. +func SetupWithScReceiptFromOpts(t *testing.T, isCheckTx bool, enableEVMCustomPrecompiles bool, testAppOpts TestAppOpts, baseAppOptions ...func(*bam.BaseApp)) (res *App) { + db := dbm.NewMemDB() + encodingConfig := MakeEncodingConfig() + cdc := encodingConfig.Marshaler + + res = New( + log.NewNopLogger(), + db, + nil, + true, + map[int64]bool{}, + t.TempDir(), + 1, + enableEVMCustomPrecompiles, + config.TestConfig(), + encodingConfig, + wasm.EnableAllProposals, + testAppOpts, + EmptyWasmOpts, + nil, // no options: receipt store is created from testAppOpts inside New() + baseAppOptions..., + ) + if !isCheckTx { + genesisState := NewDefaultGenesisState(cdc) + stateBytes, err := json.MarshalIndent(genesisState, "", " ") + if err != nil { + panic(err) + } + + defer func() { _ = recover() }() + + _, err = res.InitChain( + context.Background(), &abci.RequestInitChain{ + Validators: []abci.ValidatorUpdate{}, + ConsensusParams: DefaultConsensusParams, + AppStateBytes: stateBytes, + }, + ) + if err != nil { + panic(err) + } + } + + return res +} + func SetupWithSc(t *testing.T, isCheckTx bool, enableEVMCustomPrecompiles bool, testAppOpts TestAppOpts, baseAppOptions ...func(*bam.BaseApp)) (res *App) { db := dbm.NewMemDB() encodingConfig := MakeEncodingConfig() diff --git a/cmd/seid/cmd/app_config.go b/cmd/seid/cmd/app_config.go index d4d1e53218..b984c12c2a 100644 --- a/cmd/seid/cmd/app_config.go +++ b/cmd/seid/cmd/app_config.go @@ -23,24 +23,26 @@ type WASMConfig struct { type CustomAppConfig struct { srvconfig.Config - StateCommit seidbconfig.StateCommitConfig `mapstructure:"state-commit"` - StateStore seidbconfig.StateStoreConfig `mapstructure:"state-store"` - WASM WASMConfig `mapstructure:"wasm"` - EVM evmrpcconfig.Config `mapstructure:"evm"` - GigaExecutor gigaconfig.Config `mapstructure:"giga_executor"` - ETHReplay replay.Config `mapstructure:"eth_replay"` - ETHBlockTest blocktest.Config `mapstructure:"eth_block_test"` - EvmQuery querier.Config `mapstructure:"evm_query"` - LightInvariance seiapp.LightInvarianceConfig `mapstructure:"light_invariance"` - Admin admin.Config `mapstructure:"admin_server"` + StateCommit seidbconfig.StateCommitConfig `mapstructure:"state-commit"` + StateStore seidbconfig.StateStoreConfig `mapstructure:"state-store"` + ReceiptStore seidbconfig.ReceiptStoreConfig `mapstructure:"receipt-store"` + WASM WASMConfig `mapstructure:"wasm"` + EVM evmrpcconfig.Config `mapstructure:"evm"` + GigaExecutor gigaconfig.Config `mapstructure:"giga_executor"` + ETHReplay replay.Config `mapstructure:"eth_replay"` + ETHBlockTest blocktest.Config `mapstructure:"eth_block_test"` + EvmQuery querier.Config `mapstructure:"evm_query"` + LightInvariance seiapp.LightInvarianceConfig `mapstructure:"light_invariance"` + Admin admin.Config `mapstructure:"admin_server"` } // NewCustomAppConfig creates a CustomAppConfig with the given base config and EVM config func NewCustomAppConfig(baseConfig *srvconfig.Config, evmConfig evmrpcconfig.Config) CustomAppConfig { return CustomAppConfig{ - Config: *baseConfig, - StateCommit: seidbconfig.DefaultStateCommitConfig(), - StateStore: seidbconfig.DefaultStateStoreConfig(), + Config: *baseConfig, + StateCommit: seidbconfig.DefaultStateCommitConfig(), + StateStore: seidbconfig.DefaultStateStoreConfig(), + ReceiptStore: seidbconfig.DefaultReceiptStoreConfig(), WASM: WASMConfig{ QueryGasLimit: 300000, LruSize: 1, diff --git a/cmd/seid/cmd/init_test.go b/cmd/seid/cmd/init_test.go index f6c47add8f..c96845106f 100644 --- a/cmd/seid/cmd/init_test.go +++ b/cmd/seid/cmd/init_test.go @@ -1,11 +1,13 @@ package cmd import ( + "bytes" "encoding/json" "fmt" "os" "path/filepath" "testing" + "text/template" "github.com/sei-protocol/sei-chain/app" "github.com/sei-protocol/sei-chain/app/genesis" @@ -148,6 +150,7 @@ func TestInitModeConfiguration(t *testing.T) { // Build custom template with all sections customAppTemplate := srvconfig.ManualConfigTemplate + seidbconfig.StateCommitConfigTemplate + seidbconfig.StateStoreConfigTemplate + + seidbconfig.ReceiptStoreConfigTemplate + srvconfig.AutoManagedConfigTemplate // Simplified - just need the pruning config srvconfig.SetConfigTemplate(customAppTemplate) @@ -166,10 +169,40 @@ func TestInitModeConfiguration(t *testing.T) { if tt.validateApp != nil { tt.validateApp(t, configDir) } + + v := viper.New() + v.SetConfigFile(appTomlPath) + err = v.ReadInConfig() + require.NoError(t, err) + require.Equal(t, "pebbledb", v.GetString("receipt-store.rs-backend")) + require.Equal(t, "", v.GetString("receipt-store.db-directory")) + require.Equal(t, seidbconfig.DefaultReceiptStoreConfig().AsyncWriteBuffer, v.GetInt("receipt-store.async-write-buffer")) + require.Equal(t, seidbconfig.DefaultReceiptStoreConfig().KeepRecent, v.GetInt("receipt-store.keep-recent")) + require.Equal(t, seidbconfig.DefaultReceiptStoreConfig().PruneIntervalSeconds, v.GetInt("receipt-store.prune-interval-seconds")) }) } } +func TestInitAppConfigIncludesReceiptStoreDefaults(t *testing.T) { + customAppTemplate, customAppConfig := initAppConfig() + + tmpl, err := template.New("app").Parse(customAppTemplate) + require.NoError(t, err) + + var buf bytes.Buffer + err = tmpl.Execute(&buf, customAppConfig) + require.NoError(t, err) + + output := buf.String() + require.Contains(t, output, "[receipt-store]") + require.Contains(t, output, `rs-backend = "pebbledb"`) + require.Contains(t, output, `db-directory = ""`) + require.Contains(t, output, "async-write-buffer =") + require.Contains(t, output, "keep-recent =") + require.Contains(t, output, "prune-interval-seconds =") + require.NotContains(t, output, "use-default-comparer") +} + // TestInitModeFlag verifies the mode flag validation func TestInitModeFlag(t *testing.T) { validModes := []string{"validator", "full", "seed", "archive"} diff --git a/cmd/seid/cmd/root.go b/cmd/seid/cmd/root.go index 977f6c8528..c90b953bdf 100644 --- a/cmd/seid/cmd/root.go +++ b/cmd/seid/cmd/root.go @@ -429,6 +429,7 @@ func initAppConfig() (string, interface{}) { customAppTemplate := serverconfig.ManualConfigTemplate + seidbconfig.StateCommitConfigTemplate + seidbconfig.StateStoreConfigTemplate + + seidbconfig.ReceiptStoreConfigTemplate + evmrpcconfig.ConfigTemplate + gigaconfig.ConfigTemplate + admin.ConfigTemplate + diff --git a/sei-db/config/receipt_config.go b/sei-db/config/receipt_config.go index 87b540251e..0d2887ed91 100644 --- a/sei-db/config/receipt_config.go +++ b/sei-db/config/receipt_config.go @@ -1,5 +1,27 @@ package config +import ( + "fmt" + "strings" + + "github.com/spf13/cast" +) + +// AppOptions is a minimal interface for reading config (e.g. from Viper). +// Implemented by sei-cosmos server/types.AppOptions; defined here to avoid import cycles. +type AppOptions interface { + Get(string) interface{} +} + +const ( + flagRSDBDirectory = "receipt-store.db-directory" + flagRSBackend = "receipt-store.rs-backend" + flagRSMisnamedBackend = "receipt-store.backend" + flagRSAsyncWriteBuffer = "receipt-store.async-write-buffer" + flagRSKeepRecent = "receipt-store.keep-recent" + flagRSPruneIntervalSeconds = "receipt-store.prune-interval-seconds" +) + // ReceiptStoreConfig defines configuration for the receipt store database. type ReceiptStoreConfig struct { // DBDirectory defines the directory to store the receipt store db files @@ -7,12 +29,13 @@ type ReceiptStoreConfig struct { // default to empty DBDirectory string `mapstructure:"db-directory"` - // Backend defines the backend database used for receipt-store - // Supported backends: pebbledb, rocksdb + // Backend defines the backend database used for receipt-store. + // Supported backends: pebbledb (aka pebble), parquet // defaults to pebbledb - Backend string `mapstructure:"backend"` + Backend string `mapstructure:"rs-backend"` // AsyncWriteBuffer defines the async queue length for commits to be applied to receipt store + // Applies only to the pebbledb backend. // Set <= 0 for synchronous writes. // defaults to 100 AsyncWriteBuffer int `mapstructure:"async-write-buffer"` @@ -25,12 +48,6 @@ type ReceiptStoreConfig struct { // PruneIntervalSeconds defines the interval in seconds to trigger pruning // default to every 600 seconds PruneIntervalSeconds int `mapstructure:"prune-interval-seconds"` - - // UseDefaultComparer uses Pebble's default lexicographic byte comparer instead of - // the custom MVCCComparer. This is NOT backwards compatible with existing databases - // that were created with MVCCComparer - only use this for NEW databases. - // defaults to false (use MVCCComparer for backwards compatibility) - UseDefaultComparer bool `mapstructure:"use-default-comparer"` } // DefaultReceiptStoreConfig returns the default ReceiptStoreConfig @@ -40,6 +57,55 @@ func DefaultReceiptStoreConfig() ReceiptStoreConfig { AsyncWriteBuffer: DefaultSSAsyncBuffer, KeepRecent: DefaultSSKeepRecent, PruneIntervalSeconds: DefaultSSPruneInterval, - UseDefaultComparer: false, } } + +// ReadReceiptConfig reads receipt store config from app options (e.g. TOML / Viper). +func ReadReceiptConfig(opts AppOptions) (ReceiptStoreConfig, error) { + cfg := DefaultReceiptStoreConfig() + if v := opts.Get(flagRSMisnamedBackend); v != nil { + return cfg, fmt.Errorf("unsupported receipt-store config key %q; use %q instead", flagRSMisnamedBackend, flagRSBackend) + } + if v := opts.Get(flagRSDBDirectory); v != nil { + dbDirectory, err := cast.ToStringE(v) + if err != nil { + return cfg, err + } + cfg.DBDirectory = strings.TrimSpace(dbDirectory) + } + if v := opts.Get(flagRSBackend); v != nil { + backend, err := cast.ToStringE(v) + if err != nil { + return cfg, err + } + backend = strings.ToLower(strings.TrimSpace(backend)) + switch backend { + case "pebbledb", "pebble", "parquet": + cfg.Backend = backend + default: + return cfg, fmt.Errorf("unsupported receipt-store backend %q; supported: pebbledb, parquet", backend) + } + } + if v := opts.Get(flagRSAsyncWriteBuffer); v != nil { + asyncWriteBuffer, err := cast.ToIntE(v) + if err != nil { + return cfg, err + } + cfg.AsyncWriteBuffer = asyncWriteBuffer + } + if v := opts.Get(flagRSKeepRecent); v != nil { + keepRecent, err := cast.ToIntE(v) + if err != nil { + return cfg, err + } + cfg.KeepRecent = keepRecent + } + if v := opts.Get(flagRSPruneIntervalSeconds); v != nil { + pruneIntervalSeconds, err := cast.ToIntE(v) + if err != nil { + return cfg, err + } + cfg.PruneIntervalSeconds = pruneIntervalSeconds + } + return cfg, nil +} diff --git a/sei-db/config/receipt_config_test.go b/sei-db/config/receipt_config_test.go new file mode 100644 index 0000000000..68af52c524 --- /dev/null +++ b/sei-db/config/receipt_config_test.go @@ -0,0 +1,23 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +type mapAppOpts map[string]interface{} + +func (m mapAppOpts) Get(key string) interface{} { + return m[key] +} + +func TestReadReceiptConfigRejectsMisnamedBackendKey(t *testing.T) { + _, err := ReadReceiptConfig(mapAppOpts{ + "receipt-store.backend": "parquet", + }) + + require.Error(t, err) + require.ErrorContains(t, err, "receipt-store.backend") + require.ErrorContains(t, err, "receipt-store.rs-backend") +} diff --git a/sei-db/config/toml.go b/sei-db/config/toml.go index 8d972cf97f..7588e7b434 100644 --- a/sei-db/config/toml.go +++ b/sei-db/config/toml.go @@ -135,6 +135,24 @@ const ReceiptStoreConfigTemplate = ` # Supported backends: pebble (aka pebbledb), parquet # defaults to pebbledb rs-backend = "{{ .ReceiptStore.Backend }}" + +# Defines the receipt store directory. If unset, defaults to /data/receipt.db +db-directory = "{{ .ReceiptStore.DBDirectory }}" + +# AsyncWriteBuffer defines the async queue length for commits to be applied to receipt store. +# Applies only when rs-backend = "pebbledb"; parquet ignores this setting. +# Set <= 0 for synchronous writes. +# defaults to 100 +async-write-buffer = {{ .ReceiptStore.AsyncWriteBuffer }} + +# KeepRecent defines the number of versions to keep in receipt store +# Setting it to 0 means keep everything. +# Default to keep the last 100,000 blocks +keep-recent = {{ .ReceiptStore.KeepRecent }} + +# PruneIntervalSeconds defines the interval in seconds to trigger pruning. +# defaults to 600 seconds +prune-interval-seconds = {{ .ReceiptStore.PruneIntervalSeconds }} ` // DefaultConfigTemplate combines both templates for backward compatibility diff --git a/sei-db/config/toml_test.go b/sei-db/config/toml_test.go index d35307766f..09a175850c 100644 --- a/sei-db/config/toml_test.go +++ b/sei-db/config/toml_test.go @@ -89,6 +89,36 @@ func TestStateStoreConfigTemplate(t *testing.T) { require.Contains(t, output, "ss-import-num-workers =", "Missing ss-import-num-workers") } +// TestReceiptStoreConfigTemplate verifies that all field paths in the receipt-store TOML template +// are valid and can be resolved against the actual config struct. +func TestReceiptStoreConfigTemplate(t *testing.T) { + type TemplateConfig struct { + ReceiptStore ReceiptStoreConfig + } + + cfg := TemplateConfig{ + ReceiptStore: DefaultReceiptStoreConfig(), + } + + tmpl, err := template.New("receipt").Parse(ReceiptStoreConfigTemplate) + require.NoError(t, err, "Failed to parse ReceiptStoreConfigTemplate") + + var buf bytes.Buffer + err = tmpl.Execute(&buf, cfg) + require.NoError(t, err, "Failed to execute ReceiptStoreConfigTemplate - field path mismatch detected") + + output := buf.String() + + require.Contains(t, output, "[receipt-store]", "Missing receipt-store section") + require.Contains(t, output, `rs-backend = "pebbledb"`, "Missing or incorrect rs-backend") + require.Contains(t, output, `db-directory = ""`, "Missing or incorrect db-directory") + require.Contains(t, output, "async-write-buffer =", "Missing async-write-buffer") + require.Contains(t, output, "keep-recent =", "Missing keep-recent") + require.Contains(t, output, "prune-interval-seconds =", "Missing prune-interval-seconds") + require.Contains(t, output, `Applies only when rs-backend = "pebbledb"`, "Missing pebble-only async-write-buffer note") + require.NotContains(t, output, "use-default-comparer", "use-default-comparer should not be in receipt-store template") +} + // TestDefaultConfigTemplate verifies the combined template works correctly func TestDefaultConfigTemplate(t *testing.T) { type TemplateConfig struct { @@ -116,6 +146,9 @@ func TestDefaultConfigTemplate(t *testing.T) { require.Contains(t, output, "[state-commit]") require.Contains(t, output, "[state-store]") require.Contains(t, output, "[receipt-store]") + require.Contains(t, output, "async-write-buffer =") + require.Contains(t, output, "keep-recent =") + require.Contains(t, output, "prune-interval-seconds =") } // TestWriteModeValues verifies WriteMode enum values match template output diff --git a/sei-db/ledger_db/parquet/store_config_test.go b/sei-db/ledger_db/parquet/store_config_test.go index e3528f1c10..b1e3458eab 100644 --- a/sei-db/ledger_db/parquet/store_config_test.go +++ b/sei-db/ledger_db/parquet/store_config_test.go @@ -64,6 +64,19 @@ func TestNewStoreUsesDefaultIntervalsWhenUnset(t *testing.T) { require.Equal(t, defaultMaxBlocksPerFile, store.CacheRotateInterval()) } +func TestNewStorePreservesKeepRecentAndPruneIntervalSettings(t *testing.T) { + store, err := NewStore(dbLogger.NewNopLogger(), StoreConfig{ + DBDirectory: t.TempDir(), + KeepRecent: 123, + PruneIntervalSeconds: 9, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + + require.Equal(t, int64(123), store.config.KeepRecent) + require.Equal(t, int64(9), store.config.PruneIntervalSeconds) +} + func TestPruneOldFilesKeepsTrackingOnDeleteFailure(t *testing.T) { store, err := NewStore(StoreConfig{ DBDirectory: t.TempDir(), diff --git a/sei-db/ledger_db/receipt/parquet_store_test.go b/sei-db/ledger_db/receipt/parquet_store_test.go index 4a3972d1bd..146a4fbe59 100644 --- a/sei-db/ledger_db/receipt/parquet_store_test.go +++ b/sei-db/ledger_db/receipt/parquet_store_test.go @@ -183,6 +183,31 @@ func TestParquetReceiptStoreWALReplay(t *testing.T) { require.Equal(t, receipt.TxHashHex, got.TxHashHex) } +func TestParquetReceiptStoreUsesConfiguredDirectory(t *testing.T) { + ctx, storeKey := newTestContext() + cfg := dbconfig.DefaultReceiptStoreConfig() + cfg.Backend = "parquet" + cfg.DBDirectory = t.TempDir() + + store, err := NewReceiptStore(dbLogger.NewNopLogger(), cfg, storeKey) + require.NoError(t, err) + + txHash := common.HexToHash("0x31") + receipt := makeTestReceipt(txHash, 11, 0, common.HexToAddress("0x401"), nil) + require.NoError(t, store.SetReceipts(ctx.WithBlockHeight(11), []ReceiptRecord{ + {TxHash: txHash, Receipt: receipt}, + })) + require.NoError(t, store.Close()) + + receiptFiles, err := filepath.Glob(filepath.Join(cfg.DBDirectory, "receipts_*.parquet")) + require.NoError(t, err) + require.NotEmpty(t, receiptFiles, "receipt parquet files should be written under the configured db directory") + + logFiles, err := filepath.Glob(filepath.Join(cfg.DBDirectory, "logs_*.parquet")) + require.NoError(t, err) + require.NotEmpty(t, logFiles, "log parquet files should be written under the configured db directory") +} + func TestParquetFilePruning(t *testing.T) { ctx, storeKey := newTestContext() cfg := dbconfig.DefaultReceiptStoreConfig() diff --git a/sei-db/ledger_db/receipt/receipt_store.go b/sei-db/ledger_db/receipt/receipt_store.go index 9d08f2ec8c..37276b0113 100644 --- a/sei-db/ledger_db/receipt/receipt_store.go +++ b/sei-db/ledger_db/receipt/receipt_store.go @@ -85,6 +85,25 @@ func NewReceiptStore(config dbconfig.ReceiptStoreConfig, storeKey sdk.StoreKey) return newCachedReceiptStore(backend), nil } +// BackendTypeName returns the backend implementation name ("parquet" or "pebble") for testing. +// Returns "" if store is nil or the backend type is unknown. +func BackendTypeName(store ReceiptStore) string { + if store == nil { + return "" + } + if c, ok := store.(*cachedReceiptStore); ok { + store = c.backend + } + switch store.(type) { + case *parquetReceiptStore: + return receiptBackendParquet + case *receiptStore: + return receiptBackendPebble + default: + return "unknown" + } +} + func newReceiptBackend(config dbconfig.ReceiptStoreConfig, storeKey sdk.StoreKey) (ReceiptStore, error) { if config.DBDirectory == "" { return nil, errors.New("receipt store db directory not configured") @@ -97,6 +116,7 @@ func newReceiptBackend(config dbconfig.ReceiptStoreConfig, storeKey sdk.StoreKey case receiptBackendPebble: ssConfig := dbconfig.DefaultStateStoreConfig() ssConfig.DBDirectory = config.DBDirectory + ssConfig.AsyncWriteBuffer = config.AsyncWriteBuffer ssConfig.KeepRecent = config.KeepRecent if config.PruneIntervalSeconds > 0 { ssConfig.PruneIntervalSeconds = config.PruneIntervalSeconds From 75bd555bf9ad9cb33e67c1917cf9f525dc1af476 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 13 Mar 2026 10:02:59 -0400 Subject: [PATCH 4/6] Restore config to enable lattice hash (#3043) This commit was inadvertently reverted by the squash-merge of #3039. Restores LatticeHashEnabled config option in StateCommitConfig, composite store lattice hash support, and related tests. Conflict resolved: store_test.go needed all three imports (metrics from HEAD, evm+logger from #3043). Made-with: Cursor --- app/seidb.go | 3 + app/seidb_test.go | 2 + sei-cosmos/server/config/config.go | 9 +- sei-db/config/sc_config.go | 22 ++-- sei-db/config/toml.go | 4 + sei-db/state_db/sc/composite/store.go | 36 ++++- sei-db/state_db/sc/composite/store_test.go | 145 +++++++++++++++++++++ sei-db/state_db/sc/flatkv/api.go | 3 + sei-db/state_db/sc/flatkv/store.go | 6 + 9 files changed, 213 insertions(+), 17 deletions(-) diff --git a/app/seidb.go b/app/seidb.go index 0c51f36980..ce6c725515 100644 --- a/app/seidb.go +++ b/app/seidb.go @@ -29,6 +29,7 @@ const ( FlagSCHistoricalProofBurst = "state-commit.sc-historical-proof-burst" FlagSCWriteMode = "state-commit.sc-write-mode" FlagSCReadMode = "state-commit.sc-read-mode" + FlagSCEnableLatticeHash = "state-commit.sc-enable-lattice-hash" // SS Store configs FlagSSEnable = "state-store.ss-enable" @@ -118,6 +119,8 @@ func parseSCConfigs(appOpts servertypes.AppOptions) config.StateCommitConfig { scConfig.ReadMode = parsedRM } + scConfig.EnableLatticeHash = cast.ToBool(appOpts.Get(FlagSCEnableLatticeHash)) + if v := appOpts.Get(FlagSCHistoricalProofMaxInFlight); v != nil { scConfig.HistoricalProofMaxInFlight = cast.ToInt(v) } diff --git a/app/seidb_test.go b/app/seidb_test.go index fa91e1436e..8504f32848 100644 --- a/app/seidb_test.go +++ b/app/seidb_test.go @@ -37,6 +37,8 @@ func (t TestSeiDBAppOpts) Get(s string) interface{} { return defaultSCConfig.MemIAVLConfig.SnapshotPrefetchThreshold case FlagSCSnapshotWriteRateMBps: return defaultSCConfig.MemIAVLConfig.SnapshotWriteRateMBps + case FlagSCEnableLatticeHash: + return defaultSCConfig.EnableLatticeHash case FlagSSEnable: return defaultSSConfig.Enable case FlagSSBackend: diff --git a/sei-cosmos/server/config/config.go b/sei-cosmos/server/config/config.go index 7e9b2e383a..b3ac1b746c 100644 --- a/sei-cosmos/server/config/config.go +++ b/sei-cosmos/server/config/config.go @@ -406,10 +406,11 @@ func GetConfig(v *viper.Viper) (Config, error) { SnapshotDirectory: v.GetString("state-sync.snapshot-directory"), }, StateCommit: config.StateCommitConfig{ - Enable: v.GetBool("state-commit.sc-enable"), - Directory: v.GetString("state-commit.sc-directory"), - WriteMode: config.WriteMode(v.GetString("state-commit.sc-write-mode")), - ReadMode: config.ReadMode(v.GetString("state-commit.sc-read-mode")), + Enable: v.GetBool("state-commit.sc-enable"), + Directory: v.GetString("state-commit.sc-directory"), + WriteMode: config.WriteMode(v.GetString("state-commit.sc-write-mode")), + ReadMode: config.ReadMode(v.GetString("state-commit.sc-read-mode")), + EnableLatticeHash: v.GetBool("state-commit.sc-enable-lattice-hash"), MemIAVLConfig: memiavl.Config{ AsyncCommitBuffer: v.GetInt("state-commit.sc-async-commit-buffer"), SnapshotKeepRecent: v.GetUint32("state-commit.sc-keep-recent"), diff --git a/sei-db/config/sc_config.go b/sei-db/config/sc_config.go index b32995a77d..830cc7ab48 100644 --- a/sei-db/config/sc_config.go +++ b/sei-db/config/sc_config.go @@ -32,12 +32,15 @@ type StateCommitConfig struct { // WriteMode defines the write routing mode for EVM data // Valid values: cosmos_only, dual_write, split_write, evm_only // defaults to cosmos_only - WriteMode WriteMode `mapstructure:"write_mode"` + WriteMode WriteMode `mapstructure:"write-mode"` // ReadMode defines the read routing mode for EVM data // Valid values: cosmos_only, evm_first, split_read // defaults to cosmos_only - ReadMode ReadMode `mapstructure:"read_mode"` + ReadMode ReadMode `mapstructure:"read-mode"` + + // EnableLatticeHash controls whether lattice hash will be participating in final app hash or not + EnableLatticeHash bool `mapstructure:"enable-lattice-hash"` // MemIAVLConfig is the configuration for the MemIAVL (Cosmos) backend MemIAVLConfig memiavl.Config @@ -59,11 +62,12 @@ type StateCommitConfig struct { // DefaultStateCommitConfig returns the default StateCommitConfig func DefaultStateCommitConfig() StateCommitConfig { return StateCommitConfig{ - Enable: true, - WriteMode: CosmosOnlyWrite, - ReadMode: CosmosOnlyRead, - MemIAVLConfig: memiavl.DefaultConfig(), - FlatKVConfig: flatkv.DefaultConfig(), + Enable: true, + WriteMode: CosmosOnlyWrite, + ReadMode: CosmosOnlyRead, + EnableLatticeHash: false, + MemIAVLConfig: memiavl.DefaultConfig(), + FlatKVConfig: flatkv.DefaultConfig(), HistoricalProofMaxInFlight: DefaultSCHistoricalProofMaxInFlight, HistoricalProofRateLimit: DefaultSCHistoricalProofRateLimit, @@ -74,10 +78,10 @@ func DefaultStateCommitConfig() StateCommitConfig { // Validate checks if the StateCommitConfig is valid func (c StateCommitConfig) Validate() error { if !c.WriteMode.IsValid() { - return fmt.Errorf("invalid write_mode: %s", c.WriteMode) + return fmt.Errorf("invalid write-mode: %s", c.WriteMode) } if !c.ReadMode.IsValid() { - return fmt.Errorf("invalid read_mode: %s", c.ReadMode) + return fmt.Errorf("invalid read-mode: %s", c.ReadMode) } return nil } diff --git a/sei-db/config/toml.go b/sei-db/config/toml.go index 7588e7b434..723832d0d6 100644 --- a/sei-db/config/toml.go +++ b/sei-db/config/toml.go @@ -23,6 +23,10 @@ sc-write-mode = "{{ .StateCommit.WriteMode }}" # defaults to cosmos_only sc-read-mode = "{{ .StateCommit.ReadMode }}" +# EnableLatticeHash controls whether the FlatKV lattice hash participates +# in the final app hash. Default: false. +sc-enable-lattice-hash = {{ .StateCommit.EnableLatticeHash }} + # Max concurrent historical proof queries (RPC /store path) sc-historical-proof-max-inflight = {{ .StateCommit.HistoricalProofMaxInFlight }} diff --git a/sei-db/state_db/sc/composite/store.go b/sei-db/state_db/sc/composite/store.go index a2dca7a431..2e9f730496 100644 --- a/sei-db/state_db/sc/composite/store.go +++ b/sei-db/state_db/sc/composite/store.go @@ -232,16 +232,44 @@ func (cs *CompositeCommitStore) GetEarliestVersion() (int64, error) { return cs.cosmosCommitter.GetEarliestVersion() } +// appendEvmLatticeHash returns a new CommitInfo with the EVM lattice hash +// appended, without mutating the original. Returns the original unchanged +// when lattice hashing is disabled. +func (cs *CompositeCommitStore) appendEvmLatticeHash(ci *proto.CommitInfo, evmHash []byte) *proto.CommitInfo { + if !cs.config.EnableLatticeHash { + return ci + } + combined := make([]proto.StoreInfo, len(ci.StoreInfos)+1) + copy(combined, ci.StoreInfos) + combined[len(combined)-1] = proto.StoreInfo{ + Name: "evm_lattice", + CommitId: proto.CommitID{ + Version: ci.Version, + Hash: evmHash, + }, + } + return &proto.CommitInfo{ + Version: ci.Version, + StoreInfos: combined, + } +} + // WorkingCommitInfo returns the working commit info func (cs *CompositeCommitStore) WorkingCommitInfo() *proto.CommitInfo { - // TODO: Need to combine hash for cosmos and evm - return cs.cosmosCommitter.WorkingCommitInfo() + ci := cs.cosmosCommitter.WorkingCommitInfo() + if cs.evmCommitter != nil { + return cs.appendEvmLatticeHash(ci, cs.evmCommitter.RootHash()) + } + return ci } // LastCommitInfo returns the last commit info func (cs *CompositeCommitStore) LastCommitInfo() *proto.CommitInfo { - // TODO: Need to combine hash for cosmos and evm - return cs.cosmosCommitter.LastCommitInfo() + ci := cs.cosmosCommitter.LastCommitInfo() + if cs.evmCommitter != nil { + return cs.appendEvmLatticeHash(ci, cs.evmCommitter.CommittedRootHash()) + } + return ci } // GetChildStoreByName returns the underlying child store by module name. diff --git a/sei-db/state_db/sc/composite/store_test.go b/sei-db/state_db/sc/composite/store_test.go index 03a83d3211..94c6b82102 100644 --- a/sei-db/state_db/sc/composite/store_test.go +++ b/sei-db/state_db/sc/composite/store_test.go @@ -6,6 +6,8 @@ import ( "github.com/stretchr/testify/require" + "github.com/sei-protocol/sei-chain/sei-db/common/evm" + "github.com/sei-protocol/sei-chain/sei-db/common/logger" "github.com/sei-protocol/sei-chain/sei-db/common/metrics" "github.com/sei-protocol/sei-chain/sei-db/config" "github.com/sei-protocol/sei-chain/sei-db/proto" @@ -179,6 +181,149 @@ func TestWorkingAndLastCommitInfo(t *testing.T) { require.Equal(t, int64(1), lastInfo.Version) } +func TestLatticeHashCommitInfo(t *testing.T) { + addr := [20]byte{0xAA} + slot := [32]byte{0xBB} + evmStorageKey := evm.BuildMemIAVLEVMKey(evm.EVMKeyStorage, append(addr[:], slot[:]...)) + + makeChangesets := func(round byte) []*proto.NamedChangeSet { + return []*proto.NamedChangeSet{ + { + Name: "test", + Changeset: iavl.ChangeSet{ + Pairs: []*iavl.KVPair{ + {Key: []byte("key"), Value: []byte{round}}, + }, + }, + }, + { + Name: EVMStoreName, + Changeset: iavl.ChangeSet{ + Pairs: []*iavl.KVPair{ + {Key: evmStorageKey, Value: []byte{round}}, + }, + }, + }, + } + } + + tests := []struct { + name string + writeMode config.WriteMode + enableLattice bool + expectLattice bool + }{ + {"CosmosOnly/lattice_off", config.CosmosOnlyWrite, false, false}, + {"CosmosOnly/lattice_on", config.CosmosOnlyWrite, true, false}, + {"DualWrite/lattice_off", config.DualWrite, false, false}, + {"DualWrite/lattice_on", config.DualWrite, true, true}, + {"SplitWrite/lattice_off", config.SplitWrite, false, false}, + {"SplitWrite/lattice_on", config.SplitWrite, true, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + cfg := config.DefaultStateCommitConfig() + cfg.WriteMode = tt.writeMode + cfg.EnableLatticeHash = tt.enableLattice + + cs := NewCompositeCommitStore(t.Context(), dir, logger.NewNopLogger(), cfg) + cs.Initialize([]string{"test", EVMStoreName}) + _, err := cs.LoadVersion(0, false) + require.NoError(t, err) + defer cs.Close() + + var prevLastHash []byte + + for round := byte(1); round <= 3; round++ { + require.NoError(t, cs.ApplyChangeSets(makeChangesets(round))) + + // --- Working commit info --- + expectedCosmos := cs.cosmosCommitter.WorkingCommitInfo() + var expectedEvmHash []byte + if tt.expectLattice { + expectedEvmHash = cs.evmCommitter.RootHash() + } + + workingInfo := cs.WorkingCommitInfo() + cosmosCount := len(expectedCosmos.StoreInfos) + if tt.expectLattice { + require.Equal(t, cosmosCount+1, len(workingInfo.StoreInfos)) + } else { + require.Equal(t, cosmosCount, len(workingInfo.StoreInfos)) + } + for i, si := range expectedCosmos.StoreInfos { + require.Equal(t, si.Name, workingInfo.StoreInfos[i].Name) + require.Equal(t, si.CommitId.Hash, workingInfo.StoreInfos[i].CommitId.Hash) + } + if tt.expectLattice { + entry := workingInfo.StoreInfos[len(workingInfo.StoreInfos)-1] + require.Equal(t, "evm_lattice", entry.Name) + require.Equal(t, expectedEvmHash, entry.CommitId.Hash) + require.Equal(t, workingInfo.Version, entry.CommitId.Version) + + // Verify no duplicate names — important for app hash merkle tree + names := make(map[string]int) + for _, si := range workingInfo.StoreInfos { + names[si.Name]++ + } + for name, count := range names { + require.Equal(t, 1, count, "duplicate store name %q in WorkingCommitInfo", name) + } + } + + // --- Commit --- + _, err = cs.Commit() + require.NoError(t, err) + + // --- Last commit info --- + expectedCosmosLast := cs.cosmosCommitter.LastCommitInfo() + var expectedEvmCommitted []byte + if tt.expectLattice { + expectedEvmCommitted = cs.evmCommitter.CommittedRootHash() + require.Equal(t, expectedEvmHash, expectedEvmCommitted) + } + + lastInfo := cs.LastCommitInfo() + require.Equal(t, int64(round), lastInfo.Version) + cosmosLastCount := len(expectedCosmosLast.StoreInfos) + if tt.expectLattice { + require.Equal(t, cosmosLastCount+1, len(lastInfo.StoreInfos)) + } else { + require.Equal(t, cosmosLastCount, len(lastInfo.StoreInfos)) + } + for i, si := range expectedCosmosLast.StoreInfos { + require.Equal(t, si.Name, lastInfo.StoreInfos[i].Name) + require.Equal(t, si.CommitId.Hash, lastInfo.StoreInfos[i].CommitId.Hash) + } + if tt.expectLattice { + entry := lastInfo.StoreInfos[len(lastInfo.StoreInfos)-1] + require.Equal(t, "evm_lattice", entry.Name) + require.Equal(t, expectedEvmCommitted, entry.CommitId.Hash) + require.Equal(t, lastInfo.Version, entry.CommitId.Version) + + // Verify no duplicate names — important for app hash merkle tree + names := make(map[string]int) + for _, si := range lastInfo.StoreInfos { + names[si.Name]++ + } + for name, count := range names { + require.Equal(t, 1, count, "duplicate store name %q in LastCommitInfo", name) + } + + // Hash must change between rounds since data differs + if prevLastHash != nil { + require.NotEqual(t, prevLastHash, entry.CommitId.Hash, + "lattice hash should change across commits") + } + prevLastHash = entry.CommitId.Hash + } + } + }) + } +} + func TestRollback(t *testing.T) { dir := t.TempDir() cfg := config.DefaultStateCommitConfig() diff --git a/sei-db/state_db/sc/flatkv/api.go b/sei-db/state_db/sc/flatkv/api.go index f764812667..ad21e60b43 100644 --- a/sei-db/state_db/sc/flatkv/api.go +++ b/sei-db/state_db/sc/flatkv/api.go @@ -58,6 +58,9 @@ type Store interface { // raw LtHash vector. RootHash() []byte + // CommittedRootHash returns the 32-byte checksum of the last committed LtHash. + CommittedRootHash() []byte + // Version returns the latest committed version. Version() int64 diff --git a/sei-db/state_db/sc/flatkv/store.go b/sei-db/state_db/sc/flatkv/store.go index 1c1c5ccfb1..e37cbc8437 100644 --- a/sei-db/state_db/sc/flatkv/store.go +++ b/sei-db/state_db/sc/flatkv/store.go @@ -533,6 +533,12 @@ func (s *CommitStore) RootHash() []byte { return checksum[:] } +// CommittedRootHash returns the Blake3-256 digest of the last committed LtHash. +func (s *CommitStore) CommittedRootHash() []byte { + checksum := s.committedLtHash.Checksum() + return checksum[:] +} + func (s *CommitStore) Importer(version int64) (types.Importer, error) { if s.readOnly { return nil, errReadOnly From 2d3a7ab912c12d75d8925156f12ba0a76c646840 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 13 Mar 2026 10:04:31 -0400 Subject: [PATCH 5/6] Restore memiavl config fix for benchmark (#3046) This commit was inadvertently reverted by the squash-merge of #3039. Restores the benchmark config changes (BlocksPerCommit=1, SnapshotInterval=1000, SnapshotMinTimeInterval=60). The consoleLogger addition from #3046 is dropped because #3050 replaced the logger package with slog, making it unnecessary. Made-with: Cursor --- app/test_helpers.go | 1 - sei-db/state_db/bench/cryptosim/cryptosim_config.go | 2 +- sei-db/state_db/bench/wrappers/db_implementations.go | 3 ++- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/test_helpers.go b/app/test_helpers.go index f5cef8f22c..2750f20c34 100644 --- a/app/test_helpers.go +++ b/app/test_helpers.go @@ -481,7 +481,6 @@ func SetupWithScReceiptFromOpts(t *testing.T, isCheckTx bool, enableEVMCustomPre cdc := encodingConfig.Marshaler res = New( - log.NewNopLogger(), db, nil, true, diff --git a/sei-db/state_db/bench/cryptosim/cryptosim_config.go b/sei-db/state_db/bench/cryptosim/cryptosim_config.go index a6434c269e..a4735a775b 100644 --- a/sei-db/state_db/bench/cryptosim/cryptosim_config.go +++ b/sei-db/state_db/bench/cryptosim/cryptosim_config.go @@ -158,7 +158,7 @@ func DefaultCryptoSimConfig() *CryptoSimConfig { Erc20StorageSlotSize: 32, Erc20InteractionsPerAccount: 10, TransactionsPerBlock: 1024, - BlocksPerCommit: 32, + BlocksPerCommit: 1, Seed: 1337, CannedRandomSize: 1024 * 1024 * 1024, // 1GB Backend: wrappers.FlatKV, diff --git a/sei-db/state_db/bench/wrappers/db_implementations.go b/sei-db/state_db/bench/wrappers/db_implementations.go index 2712c1ae35..f1523f8d8a 100644 --- a/sei-db/state_db/bench/wrappers/db_implementations.go +++ b/sei-db/state_db/bench/wrappers/db_implementations.go @@ -30,7 +30,8 @@ const ( func newMemIAVLCommitStore(dbDir string) (DBWrapper, error) { cfg := memiavl.DefaultConfig() cfg.AsyncCommitBuffer = 10 - cfg.SnapshotInterval = 100 + cfg.SnapshotInterval = 1000 + cfg.SnapshotMinTimeInterval = 60 fmt.Printf("Opening memIAVL from directory %s\n", dbDir) cs := memiavl.NewCommitStore(dbDir, cfg) cs.Initialize([]string{EVMStoreName}) From 9bec9d6447245f0a02c0645f8530c34969e780af Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 13 Mar 2026 10:34:19 -0400 Subject: [PATCH 6/6] fix: remove stale logger references from restored test files The cherry-picked tests from #3035 and #3043 still referenced the old logger package (removed by #3050 slog migration). Fix: - composite/store_test.go: remove logger import/arg, add CommittedRootHash to mock - parquet/store_config_test.go: remove dbLogger arg from NewStore - receipt/parquet_store_test.go: remove dbLogger arg from NewReceiptStore Made-with: Cursor --- sei-db/ledger_db/parquet/store_config_test.go | 2 +- sei-db/ledger_db/receipt/parquet_store_test.go | 2 +- sei-db/state_db/sc/composite/store_test.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sei-db/ledger_db/parquet/store_config_test.go b/sei-db/ledger_db/parquet/store_config_test.go index b1e3458eab..4b891a2ab6 100644 --- a/sei-db/ledger_db/parquet/store_config_test.go +++ b/sei-db/ledger_db/parquet/store_config_test.go @@ -65,7 +65,7 @@ func TestNewStoreUsesDefaultIntervalsWhenUnset(t *testing.T) { } func TestNewStorePreservesKeepRecentAndPruneIntervalSettings(t *testing.T) { - store, err := NewStore(dbLogger.NewNopLogger(), StoreConfig{ + store, err := NewStore(StoreConfig{ DBDirectory: t.TempDir(), KeepRecent: 123, PruneIntervalSeconds: 9, diff --git a/sei-db/ledger_db/receipt/parquet_store_test.go b/sei-db/ledger_db/receipt/parquet_store_test.go index 146a4fbe59..5d241937ab 100644 --- a/sei-db/ledger_db/receipt/parquet_store_test.go +++ b/sei-db/ledger_db/receipt/parquet_store_test.go @@ -189,7 +189,7 @@ func TestParquetReceiptStoreUsesConfiguredDirectory(t *testing.T) { cfg.Backend = "parquet" cfg.DBDirectory = t.TempDir() - store, err := NewReceiptStore(dbLogger.NewNopLogger(), cfg, storeKey) + store, err := NewReceiptStore(cfg, storeKey) require.NoError(t, err) txHash := common.HexToHash("0x31") diff --git a/sei-db/state_db/sc/composite/store_test.go b/sei-db/state_db/sc/composite/store_test.go index 94c6b82102..a5f1f34f7a 100644 --- a/sei-db/state_db/sc/composite/store_test.go +++ b/sei-db/state_db/sc/composite/store_test.go @@ -7,7 +7,6 @@ import ( "github.com/stretchr/testify/require" "github.com/sei-protocol/sei-chain/sei-db/common/evm" - "github.com/sei-protocol/sei-chain/sei-db/common/logger" "github.com/sei-protocol/sei-chain/sei-db/common/metrics" "github.com/sei-protocol/sei-chain/sei-db/config" "github.com/sei-protocol/sei-chain/sei-db/proto" @@ -37,6 +36,7 @@ func (f *failingEVMStore) Rollback(int64) error { retur func (f *failingEVMStore) Exporter(int64) (types.Exporter, error) { return nil, nil } func (f *failingEVMStore) Importer(int64) (types.Importer, error) { return nil, nil } func (f *failingEVMStore) GetPhaseTimer() *metrics.PhaseTimer { return nil } +func (f *failingEVMStore) CommittedRootHash() []byte { return nil } func (f *failingEVMStore) Close() error { return nil } func TestCompositeStoreBasicOperations(t *testing.T) { @@ -228,7 +228,7 @@ func TestLatticeHashCommitInfo(t *testing.T) { cfg.WriteMode = tt.writeMode cfg.EnableLatticeHash = tt.enableLattice - cs := NewCompositeCommitStore(t.Context(), dir, logger.NewNopLogger(), cfg) + cs := NewCompositeCommitStore(t.Context(), dir, cfg) cs.Initialize([]string{"test", EVMStoreName}) _, err := cs.LoadVersion(0, false) require.NoError(t, err)