Move clob hydration to preblocker

protocol/app/app.go

@@ -89,6 +89,7 @@ import (
 	"github.com/cosmos/ibc-go/modules/capability"
 	capabilitykeeper "github.com/cosmos/ibc-go/modules/capability/keeper"
 	capabilitytypes "github.com/cosmos/ibc-go/modules/capability/types"
+	antetypes "github.com/dydxprotocol/v4-chain/protocol/app/ante/types"
 	"github.com/gorilla/mux"
 	"github.com/rakyll/statik/fs"
 	"github.com/spf13/cast"
@@ -1164,7 +1165,8 @@ func New(
 	)
 
 	app.ModuleManager.SetOrderPreBlockers(
-		upgradetypes.ModuleName,
+		upgradetypes.ModuleName, // Must be first since upgrades may be state schema breaking.
+		clobmoduletypes.ModuleName,
 		pricesmoduletypes.ModuleName,
 	)
 
@@ -1400,16 +1402,6 @@ func New(
 		if err := app.LoadLatestVersion(); err != nil {
 			tmos.Exit(err.Error())
 		}
-
-		// Hydrate memStores used for caching state.
-		app.hydrateMemStores()
-
-		// Hydrate the `memclob` with all ordersbooks from state,
-		// and hydrate the next `checkState` as well as the `memclob` with stateful orders.
-		app.hydrateMemclobWithOrderbooksAndStatefulOrders()
-
-		// Hydrate the keeper in-memory data structures.
-		app.hydrateKeeperInMemoryDataStructures()
 	}
 	app.initializeRateLimiters()
 
@@ -1614,15 +1606,6 @@ func (app *App) DisableHealthMonitorForTesting() {
 	app.DaemonHealthMonitor.DisableForTesting()
 }
 
-// hydrateMemStores hydrates the memStores used for caching state.
-func (app *App) hydrateMemStores() {
-	// Create an `uncachedCtx` where the underlying MultiStore is the `rootMultiStore`.
-	// We use this to hydrate the `memStore` state with values from the underlying `rootMultiStore`.
-	uncachedCtx := app.BaseApp.NewUncachedContext(true, tmproto.Header{})
-	// Initialize memstore in clobKeeper with order fill amounts and stateful orders.
-	app.ClobKeeper.InitMemStore(uncachedCtx)
-}
-
 // initializeRateLimiters initializes the rate limiters from state if the application is
 // not started from genesis.
 func (app *App) initializeRateLimiters() {
@@ -1632,42 +1615,20 @@ func (app *App) initializeRateLimiters() {
 	app.ClobKeeper.InitalizeBlockRateLimitFromStateIfExists(uncachedCtx)
 }
 
-// hydrateMemclobWithOrderbooksAndStatefulOrders hydrates the memclob with orderbooks and stateful orders
-// from state.
-func (app *App) hydrateMemclobWithOrderbooksAndStatefulOrders() {
-	// Create a `checkStateCtx` where the underlying MultiStore is the `CacheMultiStore` for
-	// the `checkState`. We do this to avoid performing any state writes to the `rootMultiStore`
-	// directly.
-	checkStateCtx := app.BaseApp.NewContext(true)
-
-	// Initialize memclob in clobKeeper with orderbooks using `ClobPairs` in state.
-	app.ClobKeeper.InitMemClobOrderbooks(checkStateCtx)
-	// Initialize memclob with all existing stateful orders.
-	// TODO(DEC-1348): Emit indexer messages to indicate that application restarted.
-	app.ClobKeeper.InitStatefulOrders(checkStateCtx)
-}
-
-// hydrateKeeperInMemoryDataStructures hydrates the keeper with ClobPairId and PerpetualId mapping
-// and untriggered conditional orders from state.
-func (app *App) hydrateKeeperInMemoryDataStructures() {
-	// Create a `checkStateCtx` where the underlying MultiStore is the `CacheMultiStore` for
-	// the `checkState`. We do this to avoid performing any state writes to the `rootMultiStore`
-	// directly.
-	checkStateCtx := app.BaseApp.NewContext(true)
-
-	// Initialize the untriggered conditional orders data structure with untriggered
-	// conditional orders in state.
-	app.ClobKeeper.HydrateClobPairAndPerpetualMapping(checkStateCtx)
-	// Initialize the untriggered conditional orders data structure with untriggered
-	// conditional orders in state.
-	app.ClobKeeper.HydrateUntriggeredConditionalOrders(checkStateCtx)
-}
-
 // GetBaseApp returns the base app of the application
 func (app *App) GetBaseApp() *baseapp.BaseApp { return app.BaseApp }
 
 // PreBlocker application updates before each begin block.
 func (app *App) PreBlocker(ctx sdk.Context, _ *abci.RequestFinalizeBlock) (*sdk.ResponsePreBlock, error) {
+	// Set gas meter to the free gas meter.
+	// This is because there is currently non-deterministic gas usage in the
+	// pre-blocker, e.g. due to hydration of in-memory data structures.
+	currentGasMeter := ctx.GasMeter()
+	ctx = ctx.WithGasMeter(antetypes.NewFreeInfiniteGasMeter())
+	defer func() {
+		ctx.WithGasMeter(currentGasMeter)
+	}()
+
 	return app.ModuleManager.PreBlock(ctx)
 }
 

protocol/app/app_test.go

@@ -9,7 +9,6 @@ import (
 	evidencemodule "cosmossdk.io/x/evidence"
 	feegrantmodule "cosmossdk.io/x/feegrant/module"
 	"cosmossdk.io/x/upgrade"
-	tmproto "github.com/cometbft/cometbft/proto/tendermint/types"
 	"github.com/cosmos/cosmos-sdk/types/module"
 	"github.com/cosmos/cosmos-sdk/x/auth"
 	authzmodule "github.com/cosmos/cosmos-sdk/x/authz/module"
@@ -137,15 +136,6 @@ func TestAppPanicsWithGrpcDisabled(t *testing.T) {
 	require.Panics(t, func() { testapp.DefaultTestApp(customFlags) })
 }
 
-func TestClobKeeperMemStoreHasBeenInitialized(t *testing.T) {
-	dydxApp := testapp.DefaultTestApp(nil)
-	ctx := dydxApp.NewUncachedContext(true, tmproto.Header{})
-
-	// The memstore panics if initialized twice so initializing again outside of application
-	// start-up should cause a panic.
-	require.Panics(t, func() { dydxApp.ClobKeeper.InitMemStore(ctx) })
-}
-
 func TestBaseApp(t *testing.T) {
 	dydxApp := testapp.DefaultTestApp(nil)
 	require.NotNil(t, dydxApp.GetBaseApp(), "Expected non-nil BaseApp")

protocol/x/clob/abci.go

@@ -16,6 +16,14 @@ import (
 	"github.com/dydxprotocol/v4-chain/protocol/x/clob/types"
 )
 
+// PreBlocker executes all ABCI PreBlock logic respective to the clob module.
+func PreBlocker(
+	ctx sdk.Context,
+	keeper types.ClobKeeper,
+) {
+	keeper.Initialize(ctx)
+}
+
 // BeginBlocker executes all ABCI BeginBlock logic respective to the clob module.
 func BeginBlocker(
 	ctx sdk.Context,

protocol/x/clob/ante/clob.go

@@ -60,6 +60,14 @@ func (cd ClobDecorator) AnteHandle(
 		return next(ctx, tx, simulate)
 	}
 
+	// Disable order placement and cancelation processing if the clob keeper is not initialized.
+	if !cd.clobKeeper.IsInitialized() {
+		return ctx, errorsmod.Wrap(
+			types.ErrClobNotInitialized,
+			"clob keeper is not initialized. Please wait for the next block.",
+		)
+	}
+
 	msgs := tx.GetMsgs()
 	var msg = msgs[0]
 

protocol/x/clob/keeper/clob_pair.go

@@ -155,6 +155,12 @@ func (k Keeper) validateClobPair(ctx sdk.Context, clobPair *types.ClobPair) erro
 	return nil
 }
 
+// maybeCreateOrderbook creates a new orderbook in the memclob.
+func (k Keeper) maybeCreateOrderbook(ctx sdk.Context, clobPair types.ClobPair) {
+	// Create the corresponding orderbook in the memclob.
+	k.MemClob.MaybeCreateOrderbook(ctx, clobPair)
+}
+
 // createOrderbook creates a new orderbook in the memclob.
 func (k Keeper) createOrderbook(ctx sdk.Context, clobPair types.ClobPair) {
 	// Create the corresponding orderbook in the memclob.
@@ -192,12 +198,11 @@ func (k Keeper) setClobPair(ctx sdk.Context, clobPair types.ClobPair) {
 }
 
 // InitMemClobOrderbooks initializes the memclob with `ClobPair`s from state.
-// This is called during app initialization in `app.go`, before any ABCI calls are received.
 func (k Keeper) InitMemClobOrderbooks(ctx sdk.Context) {
 	clobPairs := k.GetAllClobPairs(ctx)
 	for _, clobPair := range clobPairs {
 		// Create the corresponding orderbook in the memclob.
-		k.createOrderbook(
+		k.maybeCreateOrderbook(
 			ctx,
 			clobPair,
 		)

protocol/x/clob/keeper/keeper.go

@@ -45,6 +45,7 @@ type (
 		indexerEventManager indexer_manager.IndexerEventManager
 		streamingManager    streamingtypes.GrpcStreamingManager
 
+		initialized         *atomic.Bool
 		memStoreInitialized *atomic.Bool
 
 		Flags flags.ClobFlags
@@ -110,7 +111,8 @@ func NewKeeper(
 		rewardsKeeper:                rewardsKeeper,
 		indexerEventManager:          indexerEventManager,
 		streamingManager:             grpcStreamingManager,
-		memStoreInitialized:          &atomic.Bool{},
+		memStoreInitialized:          &atomic.Bool{}, // False by default.
+		initialized:                  &atomic.Bool{}, // False by default.
 		txDecoder:                    txDecoder,
 		mevTelemetryConfig: MevTelemetryConfig{
 			Enabled:    clobFlags.MevTelemetryEnabled,
@@ -154,6 +156,43 @@ func (k Keeper) Logger(ctx sdk.Context) log.Logger {
 func (k Keeper) InitializeForGenesis(ctx sdk.Context) {
 }
 
+// IsInitialized returns whether the clob keeper has been hydrated.
+func (k Keeper) IsInitialized() bool {
+	return k.initialized.Load()
+}
+
+// Initialize hydrates the clob keeper with the necessary in memory data structures.
+func (k Keeper) Initialize(ctx sdk.Context) {
+	alreadyInitialized := k.initialized.Swap(true)
+	if alreadyInitialized {
+		return
+	}
+
+	// Initialize memstore in clobKeeper with order fill amounts and stateful orders.
+	k.InitMemStore(ctx)
+
+	// Branch the context for hydration.
+	// This means that new order matches from hydration will get added to the operations
+	// queue but the corresponding state changes will be discarded.
+	// This is needed because we are hydrating in memory structures in PreBlock
+	// which operates on deliver state. Writing optimistic matches breaks consensus.
+	checkCtx, _ := ctx.CacheContext()
+	checkCtx = checkCtx.WithIsCheckTx(true)
+
+	// Initialize memclob in clobKeeper with orderbooks using `ClobPairs` in state.
+	k.InitMemClobOrderbooks(checkCtx)
+	// Initialize memclob with all existing stateful orders.
+	// TODO(DEC-1348): Emit indexer messages to indicate that application restarted.
+	k.InitStatefulOrders(checkCtx)
+
+	// Initialize the untriggered conditional orders data structure with untriggered
+	// conditional orders in state.
+	k.HydrateClobPairAndPerpetualMapping(checkCtx)
+	// Initialize the untriggered conditional orders data structure with untriggered
+	// conditional orders in state.
+	k.HydrateUntriggeredConditionalOrders(checkCtx)
+}
+
 // InitMemStore initializes the memstore of the `clob` keeper.
 // This is called during app initialization in `app.go`, before any ABCI calls are received.
 func (k Keeper) InitMemStore(ctx sdk.Context) {

protocol/x/clob/memclob/memclob.go

@@ -148,6 +148,17 @@ func (m *MemClobPriceTimePriority) CancelOrder(
 	return offchainUpdates, nil
 }
 
+// MaybeCreateOrderbook is used for updating memclob internal data structures to mark an orderbook as created.
+func (m *MemClobPriceTimePriority) MaybeCreateOrderbook(
+	ctx sdk.Context,
+	clobPair types.ClobPair,
+) {
+	if _, exists := m.openOrders.orderbooksMap[clobPair.GetClobPairId()]; exists {
+		return
+	}
+	m.CreateOrderbook(ctx, clobPair)
+}
+
 // CreateOrderbook is used for updating memclob internal data structures to mark an orderbook as created.
 // This function will panic if `clobPairId` already exists in any of the memclob's internal data structures.
 func (m *MemClobPriceTimePriority) CreateOrderbook(

protocol/x/clob/module.go

@@ -20,6 +20,7 @@ import (
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/cosmos/cosmos-sdk/types/module"
 	"github.com/dydxprotocol/v4-chain/protocol/lib"
+	"github.com/dydxprotocol/v4-chain/protocol/lib/metrics"
 	"github.com/dydxprotocol/v4-chain/protocol/x/clob/client/cli"
 	"github.com/dydxprotocol/v4-chain/protocol/x/clob/keeper"
 	"github.com/dydxprotocol/v4-chain/protocol/x/clob/types"
@@ -164,6 +165,18 @@ func (am AppModule) ExportGenesis(ctx sdk.Context, cdc codec.JSONCodec) json.Raw
 // ConsensusVersion implements ConsensusVersion.
 func (AppModule) ConsensusVersion() uint64 { return 1 }
 
+// PreBlock executes all ABCI PreBlock logic respective to the clob module.
+func (am AppModule) PreBlock(ctx context.Context) (appmodule.ResponsePreBlock, error) {
+	defer telemetry.ModuleMeasureSince(am.Name(), time.Now(), metrics.PreBlocker)
+	PreBlocker(
+		lib.UnwrapSDKContext(ctx, types.ModuleName),
+		am.keeper,
+	)
+	return sdk.ResponsePreBlock{
+		ConsensusParamsChanged: false,
+	}, nil
+}
+
 // BeginBlock executes all ABCI BeginBlock logic respective to the clob module.
 func (am AppModule) BeginBlock(ctx context.Context) error {
 	defer telemetry.ModuleMeasureSince(am.Name(), time.Now(), telemetry.MetricKeyBeginBlocker)

protocol/x/clob/types/clob_keeper.go

@@ -14,6 +14,9 @@ type ClobKeeper interface {
 	LiquidationsKeeper
 	LiquidationsConfigKeeper
 
+	IsInitialized() bool
+	Initialize(ctx sdk.Context)
+
 	AddOrderToOrderbookSubaccountUpdatesCheck(
 		ctx sdk.Context,
 		clobPairId ClobPairId,

protocol/x/clob/types/errors.go

@@ -216,6 +216,11 @@ var (
 		46,
 		"Batch cancel has failed",
 	)
+	ErrClobNotInitialized = errorsmod.Register(
+		ModuleName,
+		47,
+		"CLOB has not been initialized",
+	)
 
 	// Liquidations errors.
 	ErrInvalidLiquidationsConfig = errorsmod.Register(

protocol/x/clob/types/memclob.go

@@ -22,6 +22,10 @@ type MemClob interface {
 		ctx sdk.Context,
 		clobPair ClobPair,
 	)
+	MaybeCreateOrderbook(
+		ctx sdk.Context,
+		clobPair ClobPair,
+	)
 	CountSubaccountShortTermOrders(
 		ctx sdk.Context,
 		subaccountId satypes.SubaccountId,