Move clob hydration to preblocker

protocol/app/app.go

@@ -1164,7 +1164,8 @@ func New(
 	)
 
 	app.ModuleManager.SetOrderPreBlockers(
-		upgradetypes.ModuleName,
+		upgradetypes.ModuleName, // Must be first since upgrades may be state schema breaking.
+		clobmoduletypes.ModuleName,
 	)
 
 	// During begin block slashing happens after distr.BeginBlocker so that
@@ -1400,16 +1401,6 @@ func New(
 			tmos.Exit(err.Error())
 		}
 
-		// Hydrate memStores used for caching state.
-		app.hydrateMemStores()
-
-		// Hydrate the `memclob` with all ordersbooks from state,
-		// and hydrate the next `checkState` as well as the `memclob` with stateful orders.
-		app.hydrateMemclobWithOrderbooksAndStatefulOrders()
-
-		// Hydrate the keeper in-memory data structures.
-		app.hydrateKeeperInMemoryDataStructures()
-
 		// load the x/prices keeper currency-pair ID cache
 		app.loadCurrencyPairIDsForMarkets()
 	}
@@ -1627,15 +1618,6 @@ func (app *App) loadCurrencyPairIDsForMarkets() {
 	app.PricesKeeper.LoadCurrencyPairIDCache(uncachedCtx)
 }
 
-// hydrateMemStores hydrates the memStores used for caching state.
-func (app *App) hydrateMemStores() {
-	// Create an `uncachedCtx` where the underlying MultiStore is the `rootMultiStore`.
-	// We use this to hydrate the `memStore` state with values from the underlying `rootMultiStore`.
-	uncachedCtx := app.BaseApp.NewUncachedContext(true, tmproto.Header{})
-	// Initialize memstore in clobKeeper with order fill amounts and stateful orders.
-	app.ClobKeeper.InitMemStore(uncachedCtx)
-}
-
 // initializeRateLimiters initializes the rate limiters from state if the application is
 // not started from genesis.
 func (app *App) initializeRateLimiters() {
@@ -1645,37 +1627,6 @@ func (app *App) initializeRateLimiters() {
 	app.ClobKeeper.InitalizeBlockRateLimitFromStateIfExists(uncachedCtx)
 }
 
-// hydrateMemclobWithOrderbooksAndStatefulOrders hydrates the memclob with orderbooks and stateful orders
-// from state.
-func (app *App) hydrateMemclobWithOrderbooksAndStatefulOrders() {
-	// Create a `checkStateCtx` where the underlying MultiStore is the `CacheMultiStore` for
-	// the `checkState`. We do this to avoid performing any state writes to the `rootMultiStore`
-	// directly.
-	checkStateCtx := app.BaseApp.NewContext(true)
-
-	// Initialize memclob in clobKeeper with orderbooks using `ClobPairs` in state.
-	app.ClobKeeper.InitMemClobOrderbooks(checkStateCtx)
-	// Initialize memclob with all existing stateful orders.
-	// TODO(DEC-1348): Emit indexer messages to indicate that application restarted.
-	app.ClobKeeper.InitStatefulOrders(checkStateCtx)
-}
-
-// hydrateKeeperInMemoryDataStructures hydrates the keeper with ClobPairId and PerpetualId mapping
-// and untriggered conditional orders from state.
-func (app *App) hydrateKeeperInMemoryDataStructures() {
-	// Create a `checkStateCtx` where the underlying MultiStore is the `CacheMultiStore` for
-	// the `checkState`. We do this to avoid performing any state writes to the `rootMultiStore`
-	// directly.
-	checkStateCtx := app.BaseApp.NewContext(true)
-
-	// Initialize the untriggered conditional orders data structure with untriggered
-	// conditional orders in state.
-	app.ClobKeeper.HydrateClobPairAndPerpetualMapping(checkStateCtx)
-	// Initialize the untriggered conditional orders data structure with untriggered
-	// conditional orders in state.
-	app.ClobKeeper.HydrateUntriggeredConditionalOrders(checkStateCtx)
-}
-
 // GetBaseApp returns the base app of the application
 func (app *App) GetBaseApp() *baseapp.BaseApp { return app.BaseApp }
 

protocol/lib/metrics/constants.go

@@ -56,6 +56,7 @@ const (
 	OriginalNumTxs       = "original_num_txs"
 	OtherTxs             = "other_txs"
 	RemoveDisallowMsgs   = "remove_disallow_msgs"
+	PreBlocker           = "pre_blocker"
 	PrepareProposalTxs   = "prepare_proposal_txs"
 	PrepareCheckState    = "prepare_check_state"
 	PricesTx             = "prices_tx"

protocol/x/clob/abci.go

@@ -16,6 +16,14 @@ import (
 	"github.com/dydxprotocol/v4-chain/protocol/x/clob/types"
 )
 
+// PreBlocker executes all ABCI PreBlock logic respective to the clob module.
+func PreBlocker(
+	ctx sdk.Context,
+	keeper types.ClobKeeper,
+) {
+	keeper.Initialize(ctx)
+}
+
 // BeginBlocker executes all ABCI BeginBlock logic respective to the clob module.
 func BeginBlocker(
 	ctx sdk.Context,

protocol/x/clob/ante/clob.go

@@ -60,6 +60,14 @@ func (cd ClobDecorator) AnteHandle(
 		return next(ctx, tx, simulate)
 	}
 
+	// Disable order placement and cancelation processing if the clob keeper is not hydrated.
+	if !cd.clobKeeper.IsInitialized() {
+		return ctx, errorsmod.Wrap(
+			types.ErrClobNotInitialized,
+			"clob keeper is not initialized. Please wait for the next block.",
+		)
+	}
+
 	msgs := tx.GetMsgs()
 	var msg = msgs[0]
 

protocol/x/clob/keeper/clob_pair.go

@@ -155,6 +155,12 @@ func (k Keeper) validateClobPair(ctx sdk.Context, clobPair *types.ClobPair) erro
 	return nil
 }
 
+// maybeCreateOrderbook creates a new orderbook in the memclob.
+func (k Keeper) maybeCreateOrderbook(ctx sdk.Context, clobPair types.ClobPair) {
+	// Create the corresponding orderbook in the memclob.
+	k.MemClob.MaybeCreateOrderbook(ctx, clobPair)
+}
+
 // createOrderbook creates a new orderbook in the memclob.
 func (k Keeper) createOrderbook(ctx sdk.Context, clobPair types.ClobPair) {
 	// Create the corresponding orderbook in the memclob.
@@ -192,12 +198,11 @@ func (k Keeper) setClobPair(ctx sdk.Context, clobPair types.ClobPair) {
 }
 
 // InitMemClobOrderbooks initializes the memclob with `ClobPair`s from state.
-// This is called during app initialization in `app.go`, before any ABCI calls are received.
 func (k Keeper) InitMemClobOrderbooks(ctx sdk.Context) {
 	clobPairs := k.GetAllClobPairs(ctx)
 	for _, clobPair := range clobPairs {
 		// Create the corresponding orderbook in the memclob.
-		k.createOrderbook(
+		k.maybeCreateOrderbook(
 			ctx,
 			clobPair,
 		)

protocol/x/clob/keeper/keeper.go

@@ -45,6 +45,7 @@ type (
 		indexerEventManager indexer_manager.IndexerEventManager
 		streamingManager    streamingtypes.GrpcStreamingManager
 
+		initialized         *atomic.Bool
 		memStoreInitialized *atomic.Bool
 
 		Flags flags.ClobFlags
@@ -111,6 +112,7 @@ func NewKeeper(
 		indexerEventManager:          indexerEventManager,
 		streamingManager:             grpcStreamingManager,
 		memStoreInitialized:          &atomic.Bool{},
+		initialized:                  &atomic.Bool{},
 		txDecoder:                    txDecoder,
 		mevTelemetryConfig: MevTelemetryConfig{
 			Enabled:    clobFlags.MevTelemetryEnabled,
@@ -154,6 +156,43 @@ func (k Keeper) Logger(ctx sdk.Context) log.Logger {
 func (k Keeper) InitializeForGenesis(ctx sdk.Context) {
 }
 
+// IsInitialized returns whether the clob keeper has been hydrated.
+func (k Keeper) IsInitialized() bool {
+	return k.initialized.Load()
+}
+
+// Initialize hydrates the clob keeper with the necessary in memory data structures.
+func (k Keeper) Initialize(ctx sdk.Context) {
+	alreadyInitialized := k.initialized.Swap(true)
+	if alreadyInitialized {
+		return
+	}
+
+	// Initialize memstore in clobKeeper with order fill amounts and stateful orders.
+	k.InitMemStore(ctx)
+
+	// Branch the context for hydration.
+	// This means that new order matches from hydration will get added to the operations
+	// queue but the corresponding state changes will be discarded.
+	// This is because we are currently in the deliver state so writing optimistic matches
+	// breaks consensus.
+	checkCtx, _ := ctx.CacheContext()
+	checkCtx = checkCtx.WithIsCheckTx(true)
+
+	// Initialize memclob in clobKeeper with orderbooks using `ClobPairs` in state.
+	k.InitMemClobOrderbooks(checkCtx)
+	// Initialize memclob with all existing stateful orders.
+	// TODO(DEC-1348): Emit indexer messages to indicate that application restarted.
+	k.InitStatefulOrders(checkCtx)
+
+	// Initialize the untriggered conditional orders data structure with untriggered
+	// conditional orders in state.
+	k.HydrateClobPairAndPerpetualMapping(checkCtx)
+	// Initialize the untriggered conditional orders data structure with untriggered
+	// conditional orders in state.
+	k.HydrateUntriggeredConditionalOrders(checkCtx)
+}
+
 // InitMemStore initializes the memstore of the `clob` keeper.
 // This is called during app initialization in `app.go`, before any ABCI calls are received.
 func (k Keeper) InitMemStore(ctx sdk.Context) {

protocol/x/clob/memclob/memclob.go

@@ -148,6 +148,17 @@ func (m *MemClobPriceTimePriority) CancelOrder(
 	return offchainUpdates, nil
 }
 
+// MaybeCreateOrderbook is used for updating memclob internal data structures to mark an orderbook as created.
+func (m *MemClobPriceTimePriority) MaybeCreateOrderbook(
+	ctx sdk.Context,
+	clobPair types.ClobPair,
+) {
+	if _, exists := m.openOrders.orderbooksMap[clobPair.GetClobPairId()]; exists {
+		return
+	}
+	m.CreateOrderbook(ctx, clobPair)
+}
+
 // CreateOrderbook is used for updating memclob internal data structures to mark an orderbook as created.
 // This function will panic if `clobPairId` already exists in any of the memclob's internal data structures.
 func (m *MemClobPriceTimePriority) CreateOrderbook(

protocol/x/clob/module.go

@@ -20,6 +20,7 @@ import (
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/cosmos/cosmos-sdk/types/module"
 	"github.com/dydxprotocol/v4-chain/protocol/lib"
+	"github.com/dydxprotocol/v4-chain/protocol/lib/metrics"
 	"github.com/dydxprotocol/v4-chain/protocol/x/clob/client/cli"
 	"github.com/dydxprotocol/v4-chain/protocol/x/clob/keeper"
 	"github.com/dydxprotocol/v4-chain/protocol/x/clob/types"
@@ -164,6 +165,18 @@ func (am AppModule) ExportGenesis(ctx sdk.Context, cdc codec.JSONCodec) json.Raw
 // ConsensusVersion implements ConsensusVersion.
 func (AppModule) ConsensusVersion() uint64 { return 1 }
 
+// PreBlock executes all ABCI PreBlock logic respective to the clob module.
+func (am AppModule) PreBlock(ctx context.Context) (appmodule.ResponsePreBlock, error) {
+	defer telemetry.ModuleMeasureSince(am.Name(), time.Now(), metrics.PreBlocker)
+	PreBlocker(
+		lib.UnwrapSDKContext(ctx, types.ModuleName),
+		am.keeper,
+	)
+	return sdk.ResponsePreBlock{
+		ConsensusParamsChanged: false,
+	}, nil
+}
+
 // BeginBlock executes all ABCI BeginBlock logic respective to the clob module.
 func (am AppModule) BeginBlock(ctx context.Context) error {
 	defer telemetry.ModuleMeasureSince(am.Name(), time.Now(), telemetry.MetricKeyBeginBlocker)

protocol/x/clob/types/clob_keeper.go

@@ -14,6 +14,9 @@ type ClobKeeper interface {
 	LiquidationsKeeper
 	LiquidationsConfigKeeper
 
+	IsInitialized() bool
+	Initialize(ctx sdk.Context)
+
 	AddOrderToOrderbookSubaccountUpdatesCheck(
 		ctx sdk.Context,
 		clobPairId ClobPairId,

protocol/x/clob/types/errors.go

@@ -216,6 +216,11 @@ var (
 		46,
 		"Batch cancel has failed",
 	)
+	ErrClobNotInitialized = errorsmod.Register(
+		ModuleName,
+		47,
+		"CLOB has not been initialized",
+	)
 
 	// Liquidations errors.
 	ErrInvalidLiquidationsConfig = errorsmod.Register(

protocol/x/clob/types/memclob.go

@@ -22,6 +22,10 @@ type MemClob interface {
 		ctx sdk.Context,
 		clobPair ClobPair,
 	)
+	MaybeCreateOrderbook(
+		ctx sdk.Context,
+		clobPair ClobPair,
+	)
 	CountSubaccountShortTermOrders(
 		ctx sdk.Context,
 		subaccountId satypes.SubaccountId,