feat!: add PSS reward distribution spike

app/provider/app.go

@@ -102,6 +102,7 @@ import (
 
 	appencoding "github.com/cosmos/interchain-security/v4/app/encoding"
 	testutil "github.com/cosmos/interchain-security/v4/testutil/integration"
+	"github.com/cosmos/interchain-security/v4/x/ccv/provider"
 	ibcprovider "github.com/cosmos/interchain-security/v4/x/ccv/provider"
 	ibcproviderclient "github.com/cosmos/interchain-security/v4/x/ccv/provider/client"
 	ibcproviderkeeper "github.com/cosmos/interchain-security/v4/x/ccv/provider/keeper"
@@ -470,12 +471,15 @@ func New(
 		app.BankKeeper,
 		scopedTransferKeeper,
 	)
-	transferModule := transfer.NewAppModule(app.TransferKeeper)
-	ibcmodule := transfer.NewIBCModule(app.TransferKeeper)
+
+	// Add an IBC middleware callback to track the consumer rewards
+	var transferStack porttypes.IBCModule
+	transferStack = transfer.NewIBCModule(app.TransferKeeper)
+	transferStack = provider.NewIBCMiddleware(transferStack, app.ProviderKeeper)
 
 	// create static IBC router, add transfer route, then set and seal it
 	ibcRouter := porttypes.NewRouter()
-	ibcRouter.AddRoute(ibctransfertypes.ModuleName, ibcmodule)
+	ibcRouter.AddRoute(ibctransfertypes.ModuleName, transferStack)
 	ibcRouter.AddRoute(providertypes.ModuleName, providerModule)
 	app.IBCKeeper.SetRouter(ibcRouter)
 
@@ -514,7 +518,7 @@ func New(
 		evidence.NewAppModule(app.EvidenceKeeper),
 		ibc.NewAppModule(app.IBCKeeper),
 		params.NewAppModule(app.ParamsKeeper),
-		transferModule,
+		transfer.NewAppModule(app.TransferKeeper),
 		providerModule,
 	)
 
@@ -610,7 +614,7 @@ func New(
 		params.NewAppModule(app.ParamsKeeper),
 		evidence.NewAppModule(app.EvidenceKeeper),
 		ibc.NewAppModule(app.IBCKeeper),
-		transferModule,
+		transfer.NewAppModule(app.TransferKeeper),
 	)
 
 	app.sm.RegisterStoreDecoders()

proto/interchain_security/ccv/provider/v1/provider.proto

@@ -13,6 +13,8 @@ import "ibc/lightclients/tendermint/v1/tendermint.proto";
 import "tendermint/crypto/keys.proto";
 import "cosmos/evidence/v1beta1/evidence.proto";
 import "cosmos/base/v1beta1/coin.proto";
+import "amino/amino.proto";
+
 
 //
 // Note any type defined in this file is ONLY used internally to the provider CCV module.
@@ -300,3 +302,14 @@ message ConsumerAddrsToPrune {
   uint64 vsc_id = 2;
   AddressList consumer_addrs = 3;
 }
+
+// ConsumerRewardsAllocation stores the rewards allocated by a consumer chain
+// to the consumer rewards pool. It is used to allocate the tokens to the consumer 
+// opted-in validators and the community pool during BeginBlock.
+message ConsumerRewardsAllocation {
+  repeated cosmos.base.v1beta1.DecCoin rewards = 1 [
+    (gogoproto.nullable)     = false,
+    (amino.dont_omitempty)   = true,
+    (gogoproto.castrepeated) = "github.com/cosmos/cosmos-sdk/types.DecCoins"
+  ];
+}

tests/integration/distribution.go

@@ -3,14 +3,21 @@
 import (
 	"strings"
 
+	"cosmossdk.io/math"
+	abci "github.com/cometbft/cometbft/abci/types"
+	"github.com/cosmos/ibc-go/v7/modules/apps/transfer/types"
 	transfertypes "github.com/cosmos/ibc-go/v7/modules/apps/transfer/types"
+	clienttypes "github.com/cosmos/ibc-go/v7/modules/core/02-client/types"
+	channeltypes "github.com/cosmos/ibc-go/v7/modules/core/04-channel/types"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	authtypes "github.com/cosmos/cosmos-sdk/x/auth/types"
 
+	ibctransfertypes "github.com/cosmos/ibc-go/v7/modules/apps/transfer/types"
 	icstestingutils "github.com/cosmos/interchain-security/v4/testutil/integration"
 	consumerkeeper "github.com/cosmos/interchain-security/v4/x/ccv/consumer/keeper"
 	consumertypes "github.com/cosmos/interchain-security/v4/x/ccv/consumer/types"
+	providerkeeper "github.com/cosmos/interchain-security/v4/x/ccv/provider/keeper"
 	providertypes "github.com/cosmos/interchain-security/v4/x/ccv/provider/types"
 	ccv "github.com/cosmos/interchain-security/v4/x/ccv/types"
 )
@@ -110,6 +117,9 @@
 	s.Require().Equal(0, len(rewardCoins))
 
 	// check that the fee pool has the expected amount of coins
+	// Note that all rewards are allocated to the community pool since
+	// BeginBlock is called without the validators' votes in ibctesting.
+	// See NextBlock() in https://github.com/cosmos/ibc-go/blob/release/v7.3.x/testing/chain.go#L281
 	communityCoins := s.providerApp.GetTestDistributionKeeper().GetFeePoolCommunityCoins(s.providerCtx())
 	s.Require().True(communityCoins[ibcCoinIndex].Amount.Equal(sdk.NewDecCoinFromCoin(providerExpectedRewards[0]).Amount))
 }
@@ -454,6 +464,233 @@
 	}
 }
 
+// TestIBCTransferMiddleware tests the logic of the IBC transfer OnRecvPacket callback
+func (s *CCVTestSuite) TestIBCTransferMiddleware() {
+
+	var (
+		data   ibctransfertypes.FungibleTokenPacketData
+		packet channeltypes.Packet
+	)
+
+	testCases := []struct {
+		name             string
+		setup            func(sdk.Context, *providerkeeper.Keeper, icstestingutils.TestBankKeeper)
+		expError         bool
+		rewardsAllocated int
+		tokenTransfers   int
+	}{
+		{
+			"invalid IBC packet",
+			func(sdk.Context, *providerkeeper.Keeper, icstestingutils.TestBankKeeper) {
+				packet = channeltypes.Packet{}
+			},
+			true,
+			0,
+			0,
+		},
+		{
+			"invalid fungible token packet data",
+			func(ctx sdk.Context, keeper *providerkeeper.Keeper, bankKeeper icstestingutils.TestBankKeeper) {
+				packet.Data = nil
+			},
+			true,
+			0,
+			0,
+		},
+		{
+			"successful token transfer to empty pool",
+			func(ctx sdk.Context, keeper *providerkeeper.Keeper, bankKeeper icstestingutils.TestBankKeeper) {
+				bankKeeper.SendCoinsFromAccountToModule(
+					ctx,
+					s.providerChain.SenderAccount.GetAddress(),
+					providertypes.ConsumerRewardsPool,
+					sdk.NewCoins(sdk.NewCoin(sdk.DefaultBondDenom, math.NewInt(100_000))),
+				)
+
+				keeper.SetConsumerRewardsAllocation(
+					ctx,
+					s.consumerChain.ChainID,
+					providertypes.ConsumerRewardsAllocation{
+						Rewards: sdk.NewDecCoins(sdk.NewDecCoin(sdk.DefaultBondDenom, math.NewInt(100_000))),
+					},
+				)
+			},
+			false,
+			1,
+			1,
+		},
+		{
+			"successful token transfer to filled pool",
+			func(ctx sdk.Context, keeper *providerkeeper.Keeper, bankKeeper icstestingutils.TestBankKeeper) {
+				// fill consumer reward pool
+				bankKeeper.SendCoinsFromAccountToModule(
+					ctx,
+					s.providerChain.SenderAccount.GetAddress(),
+					providertypes.ConsumerRewardsPool,
+					sdk.NewCoins(sdk.NewCoin(sdk.DefaultBondDenom, math.NewInt(100_000))),
+				)
+				// update consumer allocation
+				keeper.SetConsumerRewardsAllocation(
+					ctx,
+					s.consumerChain.ChainID,
+					providertypes.ConsumerRewardsAllocation{
+						Rewards: sdk.NewDecCoins(sdk.NewDecCoin(sdk.DefaultBondDenom, math.NewInt(100_000))),
+					},
+				)
+			},
+			false,
+			1,
+			1,
+		},
+	}
+
+	for _, tc := range testCases {
+		s.Run(tc.name, func() {
+			s.SetupTest()
+			s.SetupCCVChannel(s.path)
+			s.SetupTransferChannel()
+
+			providerkeeper := s.providerApp.GetProviderKeeper()
+			amount := sdk.NewInt(100)
+
+			data = types.NewFungibleTokenPacketData( // can be explicitly changed in setup
+				sdk.DefaultBondDenom,
+				amount.String(),
+				authtypes.NewModuleAddress(consumertypes.ConsumerToSendToProviderName).String(),
+				providerkeeper.GetConsumerRewardsPoolAddressStr(s.providerCtx()),
+				"",
+			)
+
+			packet = channeltypes.NewPacket( // can be explicitly changed in setup
+				data.GetBytes(),
+				uint64(1),
+				s.transferPath.EndpointA.ChannelConfig.PortID,
+				s.transferPath.EndpointA.ChannelID,
+				s.transferPath.EndpointB.ChannelConfig.PortID,
+				s.transferPath.EndpointB.ChannelID,
+				clienttypes.NewHeight(1, 100),
+				0,
+			)
+
+			tc.setup(s.providerCtx(), &providerkeeper, s.providerApp.GetTestBankKeeper())
+
+			cbs, ok := s.providerChain.App.GetIBCKeeper().Router.GetRoute(ibctransfertypes.ModuleName)
+			s.Require().True(ok)
+
+			prevConsumerPool := providerkeeper.GetConsumerRewardsPool(s.providerCtx())
+			prevConsumerAlloc := providerkeeper.GetConsumerRewardsAllocation(s.providerCtx(), s.consumerChain.ChainID)
+
+			ack := cbs.OnRecvPacket(s.providerCtx(), packet, sdk.AccAddress{})
+			if !tc.expError {
+				s.Require().True(ack.Success())
+
+				// verify consumer rewards pool and allocation are updated
+				pool := providerkeeper.GetConsumerRewardsPool(s.providerCtx()).Sub(prevConsumerPool...)
+				s.Require().True(amount.Equal(pool[0].Amount))
+
+				alloc := providerkeeper.GetConsumerRewardsAllocation(s.providerCtx(), s.consumerChain.ChainID).Rewards.Sub(prevConsumerAlloc.Rewards)
+				s.Require().True(sdk.NewDecCoinsFromCoins(pool...).IsEqual(alloc))
+			} else {
+				s.Require().False(ack.Success())
+				receivedCoins := providerkeeper.GetConsumerRewardsPool(s.providerCtx()).Sub(prevConsumerPool...)
+				s.Require().True(receivedCoins.Empty())
+
+				alloc := providerkeeper.GetConsumerRewardsAllocation(s.providerCtx(), s.consumerChain.ChainID).Rewards.Sub(prevConsumerAlloc.Rewards)
+				s.Require().True(alloc.Empty())
+			}
+		})
+	}
+}
+
+// TestAllocateTokens is a happy-path test of the consumer rewards pool allocation
+// to opted-in validators and the community pool
+func (s *CCVTestSuite) TestAllocateTokens() {
+	// set up channel and delegate some tokens in order for validator set update to be sent to the consumer chain
+	s.SetupAllCCVChannels()
+	providerKeeper := s.providerApp.GetProviderKeeper()
+	bankKeeper := s.providerApp.GetTestBankKeeper()
+	distributionKeeper := s.providerApp.GetTestDistributionKeeper()
+
+	totalRewards := sdk.Coins{sdk.NewCoin(sdk.DefaultBondDenom, sdk.NewInt(100))}
+	providerKeeper.SetConsumerRewardDenom(s.providerCtx(), sdk.DefaultBondDenom) // register a consumer reward denom
+
+	// fund consumer rewards pool
+	bankKeeper.SendCoinsFromAccountToModule(
+		s.providerCtx(),
+		s.providerChain.SenderAccount.GetAddress(),
+		providertypes.ConsumerRewardsPool,
+		totalRewards,
+	)
+
+	// Allocate rewards evenly between consumers
+	rewardsPerConsumer := totalRewards.QuoInt(math.NewInt(int64(len(s.consumerBundles))))
+	for chainID := range s.consumerBundles {
+		// update consumer allocation
+		providerKeeper.SetConsumerRewardsAllocation(
+			s.providerCtx(),
+			chainID,
+			providertypes.ConsumerRewardsAllocation{
+				Rewards: sdk.NewDecCoinsFromCoins(rewardsPerConsumer...),
+			},
+		)
+	}
+
+	// Iterate over the validators and
+	// store their current voting power and outstanding rewards
+	lastValOutRewards := map[string]sdk.DecCoins{}
+	votes := []abci.VoteInfo{}
+	for _, val := range s.providerChain.Vals.Validators {
+		votes = append(votes,
+			abci.VoteInfo{
+				Validator:       abci.Validator{Address: val.Address, Power: val.VotingPower},
+				SignedLastBlock: true,
+			},
+		)
+
+		valRewards := distributionKeeper.GetValidatorOutstandingRewards(s.providerCtx(), sdk.ValAddress(val.Address))
+		lastValOutRewards[sdk.ValAddress(val.Address).String()] = valRewards.Rewards
+	}
+
+	// store community pool balance
+	lastCommPool := distributionKeeper.GetFeePoolCommunityCoins(s.providerCtx())
+
+	// execute BeginBlock to trigger the token allocation
+	providerKeeper.BeginBlockRD(
+		s.providerCtx(),
+		abci.RequestBeginBlock{
+			LastCommitInfo: abci.CommitInfo{
+				Votes: votes,
+			},
+		},
+	)
+
+	valNum := len(s.providerChain.Vals.Validators)
+	consuNum := len(s.consumerBundles)
+
+	// compute the expected validators token allocation by subtracting the community tax
+	rewardsPerConsumerDec := sdk.NewDecCoinsFromCoins(rewardsPerConsumer...)
+	communityTax := distributionKeeper.GetCommunityTax(s.providerCtx())
+	validatorsExpRewards := rewardsPerConsumerDec.
+		MulDecTruncate(math.LegacyOneDec().Sub(communityTax)).
+		// multiply by the number of consumers since all the validators opted in
+		MulDec(sdk.NewDec(int64(consuNum)))
+	perValExpReward := validatorsExpRewards.QuoDec(sdk.NewDec(int64(valNum)))
+
+	// verify the validator tokens allocation
+	// note all validators have the same voting power to keep things simple
+	for _, val := range s.providerChain.Vals.Validators {
+		valReward := distributionKeeper.GetValidatorOutstandingRewards(s.providerCtx(), sdk.ValAddress(val.Address))
+		s.Require().True(valReward.Rewards.IsEqual(
+			lastValOutRewards[sdk.ValAddress(val.Address).String()].Add(perValExpReward...),
+		))
+	}
+
+	commPoolExpRewards := sdk.NewDecCoinsFromCoins(totalRewards...).Sub(validatorsExpRewards)
+	currCommPool := distributionKeeper.GetFeePoolCommunityCoins(s.providerCtx())
+
+	s.Require().True(currCommPool.IsEqual(lastCommPool.Add(commPoolExpRewards...)))
+}
+
 // getEscrowBalance gets the current balances in the escrow account holding the transferred tokens to the provider
 func (s *CCVTestSuite) getEscrowBalance() sdk.Coins {
 	consumerBankKeeper := s.consumerApp.GetTestBankKeeper()

testutil/integration/debug_test.go

@@ -283,3 +283,11 @@ func TestHandleConsumerDoubleVotingSlashesUndelegationsAndRelegations(t *testing
 func TestSlashRetries(t *testing.T) {
 	runCCVTestByName(t, "TestSlashRetries")
 }
+
+func TestIBCTransferMiddleware(t *testing.T) {
+	runCCVTestByName(t, "TestIBCTransferMiddleware")
+}
+
+func TestAllocateTokens(t *testing.T) {
+	runCCVTestByName(t, "TestAllocateTokens")
+}