Move clob hydration to preblocker (#1412)

* Move clob hydration to preblocker

* updates

* comments

* nit

* reset gas meter; comments

* remove defer

* fix test

* fix test

* e2e test for hydration in preblocker (#1437)

* e2e test for hydration in preblocker

* fix lint

protocol/app/app.go

@@ -89,6 +89,7 @@ import (
 	"github.com/cosmos/ibc-go/modules/capability"
 	capabilitykeeper "github.com/cosmos/ibc-go/modules/capability/keeper"
 	capabilitytypes "github.com/cosmos/ibc-go/modules/capability/types"
+	antetypes "github.com/dydxprotocol/v4-chain/protocol/app/ante/types"
 	"github.com/gorilla/mux"
 	"github.com/rakyll/statik/fs"
 	"github.com/spf13/cast"
@@ -1164,7 +1165,8 @@ func New(
 	)
 
 	app.ModuleManager.SetOrderPreBlockers(
-		upgradetypes.ModuleName,
+		upgradetypes.ModuleName, // Must be first since upgrades may be state schema breaking.
+		clobmoduletypes.ModuleName,
 		pricesmoduletypes.ModuleName,
 	)
 
@@ -1400,16 +1402,6 @@ func New(
 		if err := app.LoadLatestVersion(); err != nil {
 			tmos.Exit(err.Error())
 		}
-
-		// Hydrate memStores used for caching state.
-		app.hydrateMemStores()
-
-		// Hydrate the `memclob` with all ordersbooks from state,
-		// and hydrate the next `checkState` as well as the `memclob` with stateful orders.
-		app.hydrateMemclobWithOrderbooksAndStatefulOrders()
-
-		// Hydrate the keeper in-memory data structures.
-		app.hydrateKeeperInMemoryDataStructures()
 	}
 	app.initializeRateLimiters()
 
@@ -1614,15 +1606,6 @@ func (app *App) DisableHealthMonitorForTesting() {
 	app.DaemonHealthMonitor.DisableForTesting()
 }
 
-// hydrateMemStores hydrates the memStores used for caching state.
-func (app *App) hydrateMemStores() {
-	// Create an `uncachedCtx` where the underlying MultiStore is the `rootMultiStore`.
-	// We use this to hydrate the `memStore` state with values from the underlying `rootMultiStore`.
-	uncachedCtx := app.BaseApp.NewUncachedContext(true, tmproto.Header{})
-	// Initialize memstore in clobKeeper with order fill amounts and stateful orders.
-	app.ClobKeeper.InitMemStore(uncachedCtx)
-}
-
 // initializeRateLimiters initializes the rate limiters from state if the application is
 // not started from genesis.
 func (app *App) initializeRateLimiters() {
@@ -1632,42 +1615,18 @@ func (app *App) initializeRateLimiters() {
 	app.ClobKeeper.InitalizeBlockRateLimitFromStateIfExists(uncachedCtx)
 }
 
-// hydrateMemclobWithOrderbooksAndStatefulOrders hydrates the memclob with orderbooks and stateful orders
-// from state.
-func (app *App) hydrateMemclobWithOrderbooksAndStatefulOrders() {
-	// Create a `checkStateCtx` where the underlying MultiStore is the `CacheMultiStore` for
-	// the `checkState`. We do this to avoid performing any state writes to the `rootMultiStore`
-	// directly.
-	checkStateCtx := app.BaseApp.NewContext(true)
-
-	// Initialize memclob in clobKeeper with orderbooks using `ClobPairs` in state.
-	app.ClobKeeper.InitMemClobOrderbooks(checkStateCtx)
-	// Initialize memclob with all existing stateful orders.
-	// TODO(DEC-1348): Emit indexer messages to indicate that application restarted.
-	app.ClobKeeper.InitStatefulOrders(checkStateCtx)
-}
-
-// hydrateKeeperInMemoryDataStructures hydrates the keeper with ClobPairId and PerpetualId mapping
-// and untriggered conditional orders from state.
-func (app *App) hydrateKeeperInMemoryDataStructures() {
-	// Create a `checkStateCtx` where the underlying MultiStore is the `CacheMultiStore` for
-	// the `checkState`. We do this to avoid performing any state writes to the `rootMultiStore`
-	// directly.
-	checkStateCtx := app.BaseApp.NewContext(true)
-
-	// Initialize the untriggered conditional orders data structure with untriggered
-	// conditional orders in state.
-	app.ClobKeeper.HydrateClobPairAndPerpetualMapping(checkStateCtx)
-	// Initialize the untriggered conditional orders data structure with untriggered
-	// conditional orders in state.
-	app.ClobKeeper.HydrateUntriggeredConditionalOrders(checkStateCtx)
-}
-
 // GetBaseApp returns the base app of the application
 func (app *App) GetBaseApp() *baseapp.BaseApp { return app.BaseApp }
 
 // PreBlocker application updates before each begin block.
 func (app *App) PreBlocker(ctx sdk.Context, _ *abci.RequestFinalizeBlock) (*sdk.ResponsePreBlock, error) {
+	// Set gas meter to the free gas meter.
+	// This is because there is currently non-deterministic gas usage in the
+	// pre-blocker, e.g. due to hydration of in-memory data structures.
+	//
+	// Note that we don't need to reset the gas meter after the pre-blocker
+	// because Go is pass by value.
+	ctx = ctx.WithGasMeter(antetypes.NewFreeInfiniteGasMeter())
 	return app.ModuleManager.PreBlock(ctx)
 }
 

protocol/app/app_test.go

@@ -9,7 +9,6 @@ import (
 	evidencemodule "cosmossdk.io/x/evidence"
 	feegrantmodule "cosmossdk.io/x/feegrant/module"
 	"cosmossdk.io/x/upgrade"
-	tmproto "github.com/cometbft/cometbft/proto/tendermint/types"
 	"github.com/cosmos/cosmos-sdk/types/module"
 	"github.com/cosmos/cosmos-sdk/x/auth"
 	authzmodule "github.com/cosmos/cosmos-sdk/x/authz/module"
@@ -137,15 +136,6 @@ func TestAppPanicsWithGrpcDisabled(t *testing.T) {
 	require.Panics(t, func() { testapp.DefaultTestApp(customFlags) })
 }
 
-func TestClobKeeperMemStoreHasBeenInitialized(t *testing.T) {
-	dydxApp := testapp.DefaultTestApp(nil)
-	ctx := dydxApp.NewUncachedContext(true, tmproto.Header{})
-
-	// The memstore panics if initialized twice so initializing again outside of application
-	// start-up should cause a panic.
-	require.Panics(t, func() { dydxApp.ClobKeeper.InitMemStore(ctx) })
-}
-
 func TestBaseApp(t *testing.T) {
 	dydxApp := testapp.DefaultTestApp(nil)
 	require.NotNil(t, dydxApp.GetBaseApp(), "Expected non-nil BaseApp")

protocol/x/clob/abci.go

@@ -16,6 +16,14 @@ import (
 	"github.com/dydxprotocol/v4-chain/protocol/x/clob/types"
 )
 
+// PreBlocker executes all ABCI PreBlock logic respective to the clob module.
+func PreBlocker(
+	ctx sdk.Context,
+	keeper types.ClobKeeper,
+) {
+	keeper.Initialize(ctx)
+}
+
 // BeginBlocker executes all ABCI BeginBlock logic respective to the clob module.
 func BeginBlocker(
 	ctx sdk.Context,

protocol/x/clob/ante/clob.go

@@ -60,6 +60,14 @@ func (cd ClobDecorator) AnteHandle(
 		return next(ctx, tx, simulate)
 	}
 
+	// Disable order placement and cancelation processing if the clob keeper is not initialized.
+	if !cd.clobKeeper.IsInitialized() {
+		return ctx, errorsmod.Wrap(
+			types.ErrClobNotInitialized,
+			"clob keeper is not initialized. Please wait for the next block.",
+		)
+	}
+
 	msgs := tx.GetMsgs()
 	var msg = msgs[0]
 

protocol/x/clob/ante/clob_test.go

@@ -50,6 +50,7 @@ func runTestCase(t *testing.T, tc TestCase) {
 	// Setup AnteHandler.
 	mockClobKeeper := &mocks.ClobKeeper{}
 	mockClobKeeper.On("Logger", mock.Anything).Return(log.NewNopLogger()).Maybe()
+	mockClobKeeper.On("IsInitialized").Return(true).Maybe()
 	cd := ante.NewClobDecorator(mockClobKeeper)
 	antehandler := sdk.ChainAnteDecorators(cd)
 	if tc.setupMocks != nil {