Merge branch 'main' into son/timeout_height_time_based

.github/ISSUE_TEMPLATE/module-readiness-checklist.md

@@ -16,7 +16,7 @@ The module **should not** be included in any Release Candidate tag until it has
 
 - [ ] API audit (at least 1 person) (@assignee)
   - [ ] Are Msg and Query methods and types well-named and organized?
-  - [ ] Is everything well documented (inline godoc as well as the spec [README.md](https://github.com/cosmos/cosmos-sdk/blob/main/docs/spec/SPEC-SPEC.md) in module directory)
+  - [ ] Is everything well documented (inline godoc as well as the spec [README.md](https://github.com/cosmos/cosmos-sdk/blob/main/docs/spec/README.md) in module directory)
 - [ ] State machine audit (at least 2 people) (@assignee1, @assignee2)
   - [ ] Read through MsgServer code and verify correctness upon visual inspection
   - [ ] Ensure all state machine code which could be confusing is properly commented

baseapp/streaming.go

@@ -1,12 +1,18 @@
 package baseapp
 
 import (
+	"context"
 	"fmt"
 	"sort"
 	"strings"
 
+	abci "github.com/cometbft/cometbft/api/cometbft/abci/v1"
 	"github.com/spf13/cast"
 
+	"cosmossdk.io/schema"
+	"cosmossdk.io/schema/appdata"
+	"cosmossdk.io/schema/decoding"
+	"cosmossdk.io/schema/indexer"
 	"cosmossdk.io/store/streaming"
 	storetypes "cosmossdk.io/store/types"
 
@@ -22,6 +28,31 @@ const (
 	StreamingABCIStopNodeOnErrTomlKey = "stop-node-on-err"
 )
 
+// EnableIndexer enables the built-in indexer with the provided options (usually from the app.toml indexer key),
+// kv-store keys, and app modules. Using the built-in indexer framework is mutually exclusive from using other
+// types of streaming listeners.
+func (app *BaseApp) EnableIndexer(indexerOpts interface{}, keys map[string]*storetypes.KVStoreKey, appModules map[string]any) error {
+	listener, err := indexer.StartManager(indexer.ManagerOptions{
+		Config:     indexerOpts,
+		Resolver:   decoding.ModuleSetDecoderResolver(appModules),
+		SyncSource: nil,
+		Logger:     app.logger.With("module", "indexer"),
+	})
+	if err != nil {
+		return err
+	}
+
+	exposedKeys := exposeStoreKeysSorted([]string{"*"}, keys)
+	app.cms.AddListeners(exposedKeys)
+
+	app.streamingManager = storetypes.StreamingManager{
+		ABCIListeners: []storetypes.ABCIListener{listenerWrapper{listener}},
+		StopNodeOnErr: true,
+	}
+
+	return nil
+}
+
 // RegisterStreamingServices registers streaming services with the BaseApp.
 func (app *BaseApp) RegisterStreamingServices(appOpts servertypes.AppOptions, keys map[string]*storetypes.KVStoreKey) error {
 	// register streaming services
@@ -110,3 +141,51 @@ func exposeStoreKeysSorted(keysStr []string, keys map[string]*storetypes.KVStore
 
 	return exposeStoreKeys
 }
+
+type listenerWrapper struct {
+	listener appdata.Listener
+}
+
+func (p listenerWrapper) ListenFinalizeBlock(_ context.Context, req abci.FinalizeBlockRequest, res abci.FinalizeBlockResponse) error {
+	if p.listener.StartBlock != nil {
+		err := p.listener.StartBlock(appdata.StartBlockData{
+			Height: uint64(req.Height),
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	//// TODO txs, events
+
+	return nil
+}
+
+func (p listenerWrapper) ListenCommit(ctx context.Context, res abci.CommitResponse, changeSet []*storetypes.StoreKVPair) error {
+	if cb := p.listener.OnKVPair; cb != nil {
+		updates := make([]appdata.ModuleKVPairUpdate, len(changeSet))
+		for i, pair := range changeSet {
+			updates[i] = appdata.ModuleKVPairUpdate{
+				ModuleName: pair.StoreKey,
+				Update: schema.KVPairUpdate{
+					Key:    pair.Key,
+					Value:  pair.Value,
+					Delete: pair.Delete,
+				},
+			}
+		}
+		err := cb(appdata.KVPairData{Updates: updates})
+		if err != nil {
+			return err
+		}
+	}
+
+	if p.listener.Commit != nil {
+		err := p.listener.Commit(appdata.CommitData{})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

client/v2/go.mod

@@ -170,7 +170,10 @@ require (
 	pgregory.net/rapid v1.1.0 // indirect
 )
 
-require github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+require (
+	cosmossdk.io/schema v0.1.1 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+)
 
 replace github.com/cosmos/cosmos-sdk => ./../../
 

client/v2/go.sum

@@ -10,6 +10,8 @@ cosmossdk.io/errors v1.0.1 h1:bzu+Kcr0kS/1DuPBtUFdWjzLqyUuCiyHjyJB6srBV/0=
 cosmossdk.io/errors v1.0.1/go.mod h1:MeelVSZThMi4bEakzhhhE/CKqVv3nOJDA25bIqRDu/U=
 cosmossdk.io/math v1.3.0 h1:RC+jryuKeytIiictDslBP9i1fhkVm6ZDmZEoNP316zE=
 cosmossdk.io/math v1.3.0/go.mod h1:vnRTxewy+M7BtXBNFybkuhSH4WfedVAAnERHgVFhp3k=
+cosmossdk.io/schema v0.1.1 h1:I0M6pgI7R10nq+/HCQfbO6BsGBZA8sQy+duR1Y3aKcA=
+cosmossdk.io/schema v0.1.1/go.mod h1:RDAhxIeNB4bYqAlF4NBJwRrgtnciMcyyg0DOKnhNZQQ=
 filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs=

docs/architecture/adr-014-proportional-slashing.md

@@ -65,7 +65,7 @@ Whenever a new slash occurs, a `SlashEvent` struct is created with the faulting
 
 We then will iterate over all the SlashEvents in the queue, adding their `ValidatorVotingPercent` to calculate the new percent to slash all the validators in the queue at, using the "Square of Sum of Roots" formula introduced above.
 
-Once we have the `NewSlashPercent`, we then iterate over all the `SlashEvent`s in the queue once again, and if `NewSlashPercent > SlashedSoFar` for that SlashEvent, we call the `staking.Slash(slashEvent.Address, slashEvent.Power, Math.Min(Math.Max(minSlashPercent, NewSlashPercent - SlashedSoFar), maxSlashPercent)` (we pass in the power of the validator before any slashes occurred, so that we slash the right amount of tokens).  We then set `SlashEvent.SlashedSoFar` amount to `NewSlashPercent`.
+Once we have the `NewSlashPercent`, we then iterate over all the `SlashEvent`s in the queue once again, and if `NewSlashPercent > SlashedSoFar` for that SlashEvent, we call the `staking.Slash(slashEvent.Address, slashEvent.Power, Math.Min(Math.Max(minSlashPercent, NewSlashPercent - SlashedSoFar), maxSlashPercent))` (we pass in the power of the validator before any slashes occurred, so that we slash the right amount of tokens).  We then set `SlashEvent.SlashedSoFar` amount to `NewSlashPercent`.
 
 ## Status
 

docs/architecture/adr-039-epoched-staking.md

@@ -21,7 +21,7 @@ This ADR updates the proof of stake module to buffer the staking weight updates
 
 The current proof of stake module takes the design decision to apply staking weight changes to the consensus engine immediately. This means that delegations and unbonds get applied immediately to the validator set. This decision was primarily done as it was implementationally simplest, and because we at the time believed that this would lead to better UX for clients.
 
-An alternative design choice is to allow buffering staking updates (delegations, unbonds, validators joining) for a number of blocks. This 'epoch'd proof of stake consensus provides the guarantee that the consensus weights for validators will not change mid-epoch, except in the event of a slash condition.
+An alternative design choice is to allow buffering staking updates (delegations, unbonds, validators joining) for a number of blocks. This 'epoch'ed proof of stake consensus provides the guarantee that the consensus weights for validators will not change mid-epoch, except in the event of a slash condition.
 
 Additionally, the UX hurdle may not be as significant as was previously thought. This is because it is possible to provide users immediate acknowledgement that their bond was recorded and will be executed.
 

docs/architecture/adr-049-state-sync-hooks.md

@@ -139,7 +139,7 @@ type ExtensionSnapshotter interface {
 
 ## Consequences
 
-As a result of this implementation, we are able to create snapshots of binary chunk stream for the state that we maintain outside of the IAVL Tree, CosmWasm blobs for example. And new clients are able to fetch sanpshots of state for all modules that have implemented the corresponding interface from peer nodes. 
+As a result of this implementation, we are able to create snapshots of binary chunk stream for the state that we maintain outside of the IAVL Tree, CosmWasm blobs for example. And new clients are able to fetch snapshots of state for all modules that have implemented the corresponding interface from peer nodes.
 
 
 ### Backwards Compatibility

docs/build/building-modules/11-structure.md

@@ -50,41 +50,39 @@ x/{module_name}
 │   ├── keys.go
 │   ├── msg_server.go
 │   └── querier.go
-├── module
-│   └── module.go
-│   └── abci.go
-│   └── autocli.go
-│   └── depinject.go
 ├── simulation
 │   ├── decoder.go
 │   ├── genesis.go
 │   ├── operations.go
 │   └── params.go
-├── {module_name}.pb.go
-├── codec.go
-├── errors.go
-├── events.go
-├── events.pb.go
-├── expected_keepers.go
-├── genesis.go
-├── genesis.pb.go
-├── keys.go
-├── msgs.go
-├── params.go
-├── query.pb.go
-├── tx.pb.go
+├── types
+│   ├── {module_name}.pb.go
+│   ├── codec.go
+│   ├── errors.go
+│   ├── events.go
+│   ├── events.pb.go
+│   ├── expected_keepers.go
+│   ├── genesis.go
+│   ├── genesis.pb.go
+│   ├── keys.go
+│   ├── msgs.go
+│   ├── params.go
+│   ├── query.pb.go
+│   └── tx.pb.go
+├── module.go
+├── abci.go
+├── autocli.go
+├── depinject.go
 └── README.md
 ```
 
 * `client/`: The module's CLI client functionality implementation and the module's CLI testing suite.
 * `exported/`: The module's exported types - typically interface types. If a module relies on keepers from another module, it is expected to receive the keepers as interface contracts through the `expected_keepers.go` file (see below) in order to avoid a direct dependency on the module implementing the keepers. However, these interface contracts can define methods that operate on and/or return types that are specific to the module that is implementing the keepers and this is where `exported/` comes into play. The interface types that are defined in `exported/` use canonical types, allowing for the module to receive the keepers as interface contracts through the `expected_keepers.go` file. This pattern allows for code to remain DRY and also alleviates import cycle chaos.
 * `keeper/`: The module's `Keeper` and `MsgServer` implementation.
     * `abci.go`: The module's `BeginBlocker` and `EndBlocker` implementations (this file is only required if `BeginBlocker` and/or `EndBlocker` need to be defined).
-* `module/`: The module's `AppModule` implementation.
-    * `autocli.go`: The module [autocli](https://docs.cosmos.network/main/core/autocli) options.
 * `simulation/`: The module's [simulation](./14-simulator.md) package defines functions used by the blockchain simulator application (`simapp`).
 * `README.md`: The module's specification documents outlining important concepts, state storage structure, and message and event type definitions. Learn more how to write module specs in the [spec guidelines](../spec/SPEC_MODULE.md).
-* The root directory includes type definitions for messages, events, and genesis state, including the type definitions generated by Protocol Buffers.
+* `types/`: includes type definitions for messages, events, and genesis state, including the type definitions generated by Protocol Buffers.
     * `codec.go`: The module's registry methods for interface types.
     * `errors.go`: The module's sentinel errors.
     * `events.go`: The module's event types and constructors.
@@ -94,3 +92,8 @@ x/{module_name}
     * `msgs.go`: The module's message type definitions and associated methods.
     * `params.go`: The module's parameter type definitions and associated methods.
     * `*.pb.go`: The module's type definitions generated by Protocol Buffers (as defined in the respective `*.proto` files above).
+* The root directory includes the module's `AppModule` implementation.
+    * `autocli.go`: The module [autocli](https://docs.cosmos.network/main/core/autocli) options.
+    * `depinject.go`: The module [depinject](./15-depinject.md#type-definition) options.
+
+> Note: although the above pattern is followed by most of the Cosmos SDK modules, there are some modules that don't follow this pattern. E.g `x/group` and `x/nft` dont have a `types` folder, instead all of the type definitions for messages, events, and genesis state are live in the root directory and the module's `AppModule` implementation lives in the `module` folder.

go.mod

@@ -11,6 +11,7 @@ require (
 	cosmossdk.io/errors v1.0.1
 	cosmossdk.io/log v1.3.1
 	cosmossdk.io/math v1.3.0
+	cosmossdk.io/schema v0.1.1
 	cosmossdk.io/store v1.1.1-0.20240418092142-896cdf1971bc
 	cosmossdk.io/x/auth v0.0.0-00010101000000-000000000000
 	cosmossdk.io/x/bank v0.0.0-20240226161501-23359a0b6d91

go.sum

@@ -8,6 +8,8 @@ cosmossdk.io/errors v1.0.1 h1:bzu+Kcr0kS/1DuPBtUFdWjzLqyUuCiyHjyJB6srBV/0=
 cosmossdk.io/errors v1.0.1/go.mod h1:MeelVSZThMi4bEakzhhhE/CKqVv3nOJDA25bIqRDu/U=
 cosmossdk.io/math v1.3.0 h1:RC+jryuKeytIiictDslBP9i1fhkVm6ZDmZEoNP316zE=
 cosmossdk.io/math v1.3.0/go.mod h1:vnRTxewy+M7BtXBNFybkuhSH4WfedVAAnERHgVFhp3k=
+cosmossdk.io/schema v0.1.1 h1:I0M6pgI7R10nq+/HCQfbO6BsGBZA8sQy+duR1Y3aKcA=
+cosmossdk.io/schema v0.1.1/go.mod h1:RDAhxIeNB4bYqAlF4NBJwRrgtnciMcyyg0DOKnhNZQQ=
 filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs=

schema/decoding/resolver.go

@@ -0,0 +1,66 @@
+package decoding
+
+import (
+	"sort"
+
+	"cosmossdk.io/schema"
+)
+
+// DecoderResolver is an interface that allows indexers to discover and use module decoders.
+type DecoderResolver interface {
+	// IterateAll iterates over all available module decoders.
+	IterateAll(func(moduleName string, cdc schema.ModuleCodec) error) error
+
+	// LookupDecoder looks up a specific module decoder.
+	LookupDecoder(moduleName string) (decoder schema.ModuleCodec, found bool, err error)
+}
+
+// ModuleSetDecoderResolver returns DecoderResolver that will discover modules implementing
+// DecodeableModule in the provided module set.
+func ModuleSetDecoderResolver(moduleSet map[string]interface{}) DecoderResolver {
+	return &moduleSetDecoderResolver{
+		moduleSet: moduleSet,
+	}
+}
+
+type moduleSetDecoderResolver struct {
+	moduleSet map[string]interface{}
+}
+
+func (a moduleSetDecoderResolver) IterateAll(f func(string, schema.ModuleCodec) error) error {
+	keys := make([]string, 0, len(a.moduleSet))
+	for k := range a.moduleSet {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		module := a.moduleSet[k]
+		dm, ok := module.(schema.HasModuleCodec)
+		if ok {
+			decoder, err := dm.ModuleCodec()
+			if err != nil {
+				return err
+			}
+			err = f(k, decoder)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (a moduleSetDecoderResolver) LookupDecoder(moduleName string) (schema.ModuleCodec, bool, error) {
+	mod, ok := a.moduleSet[moduleName]
+	if !ok {
+		return schema.ModuleCodec{}, false, nil
+	}
+
+	dm, ok := mod.(schema.HasModuleCodec)
+	if !ok {
+		return schema.ModuleCodec{}, false, nil
+	}
+
+	decoder, err := dm.ModuleCodec()
+	return decoder, true, err
+}