feat(cosmos): robustly handle kvstore rollback

golang/cosmos/app/app.go

@@ -1,7 +1,6 @@
 package gaia
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
 	stdlog "log"
@@ -411,10 +410,6 @@ func NewAgoricApp(
 
 	// This function is tricky to get right, so we build it ourselves.
 	callToController := func(ctx sdk.Context, str string) (string, error) {
-		if vm.IsSimulation(ctx) {
-			// Just return empty, since the message is being simulated.
-			return "", nil
-		}
 		// We use SwingSet-level metering to charge the user for the call.
 		app.MustInitController(ctx)
 		defer vm.SetControllerContext(ctx)()
@@ -435,7 +430,7 @@ func NewAgoricApp(
 		app.IBCKeeper.ChannelKeeper, &app.IBCKeeper.PortKeeper,
 		app.BankKeeper,
 		scopedVibcKeeper,
-		callToController,
+		app.SwingSetKeeper.PushAction,
 	)
 
 	vibcModule := vibc.NewAppModule(app.VibcKeeper)
@@ -444,13 +439,17 @@ func NewAgoricApp(
 	app.VbankKeeper = vbank.NewKeeper(
 		appCodec, keys[vbank.StoreKey], app.GetSubspace(vbank.ModuleName),
 		app.BankKeeper, authtypes.FeeCollectorName,
-		callToController,
+		app.SwingSetKeeper.PushAction,
 	)
 	vbankModule := vbank.NewAppModule(app.VbankKeeper)
 	app.vbankPort = vm.RegisterPortHandler("bank", vbank.NewPortHandler(vbankModule, app.VbankKeeper))
 
 	// Lien keeper, and circular reference back to wrappedAccountKeeper
-	app.LienKeeper = lien.NewKeeper(keys[lien.StoreKey], appCodec, wrappedAccountKeeper, app.BankKeeper, app.StakingKeeper, callToController)
+	app.LienKeeper = lien.NewKeeper(
+		appCodec, keys[lien.StoreKey],
+		wrappedAccountKeeper, app.BankKeeper, app.StakingKeeper,
+		app.SwingSetKeeper.PushAction,
+	)
 	wrappedAccountKeeper.SetWrapper(app.LienKeeper.GetAccountWrapper())
 	lienModule := lien.NewAppModule(app.LienKeeper)
 	app.lienPort = vm.RegisterPortHandler("lien", lien.NewPortHandler(app.LienKeeper))
@@ -705,10 +704,7 @@ func (app *GaiaApp) MustInitController(ctx sdk.Context) {
 		VbankPort:   app.vbankPort,
 		LienPort:    app.lienPort,
 	}
-	bz, err := json.Marshal(action)
-	if err == nil {
-		_, err = app.SwingSetKeeper.CallToController(ctx, string(bz))
-	}
+	_, err := app.SwingSetKeeper.BlockingSend(ctx, action)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "Cannot initialize Controller", err)
 		os.Exit(1)

golang/cosmos/x/lien/keeper/keeper.go

@@ -33,20 +33,20 @@ type keeperImpl struct {
 	bankKeeper    types.BankKeeper
 	stakingKeeper types.StakingKeeper
 
-	callToController func(ctx sdk.Context, str string) (string, error)
+	pushAction func(ctx sdk.Context, action interface{}) error
 }
 
 // NewKeeper returns a new Keeper.
 // The ak must be the same accout keeper that the bk and sk use.
-func NewKeeper(key sdk.StoreKey, cdc codec.Codec, ak *types.WrappedAccountKeeper, bk types.BankKeeper, sk types.StakingKeeper,
-	callToController func(ctx sdk.Context, str string) (string, error)) Keeper {
+func NewKeeper(cdc codec.Codec, key sdk.StoreKey, ak *types.WrappedAccountKeeper, bk types.BankKeeper, sk types.StakingKeeper,
+	pushAction func(ctx sdk.Context, action interface{}) error) Keeper {
 	return keeperImpl{
-		key:              key,
-		cdc:              cdc,
-		accountKeeper:    ak,
-		bankKeeper:       bk,
-		stakingKeeper:    sk,
-		callToController: callToController,
+		key:           key,
+		cdc:           cdc,
+		accountKeeper: ak,
+		bankKeeper:    bk,
+		stakingKeeper: sk,
+		pushAction:    pushAction,
 	}
 }
 

golang/cosmos/x/swingset/abci.go

@@ -1,12 +1,10 @@
 package swingset
 
 import (
-	"encoding/json"
 	"time"
 
 	"github.com/cosmos/cosmos-sdk/telemetry"
 	sdk "github.com/cosmos/cosmos-sdk/types"
-	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
 	abci "github.com/tendermint/tendermint/abci/types"
 
 	"github.com/Agoric/agoric-sdk/golang/cosmos/vm"
@@ -23,11 +21,10 @@ type beginBlockAction struct {
 }
 
 type endBlockAction struct {
-	Type        string       `json:"type"`
-	StoragePort int          `json:"storagePort"`
-	BlockHeight int64        `json:"blockHeight"`
-	BlockTime   int64        `json:"blockTime"`
-	Params      types.Params `json:"params"`
+	Type        string `json:"type"`
+	StoragePort int    `json:"storagePort"`
+	BlockHeight int64  `json:"blockHeight"`
+	BlockTime   int64  `json:"blockTime"`
 }
 
 type commitBlockAction struct {
@@ -47,13 +44,11 @@ func BeginBlock(ctx sdk.Context, req abci.RequestBeginBlock, keeper Keeper) erro
 		ChainID:     ctx.ChainID(),
 		Params:      keeper.GetParams(ctx),
 	}
-	b, err := json.Marshal(action)
-	if err != nil {
-		return sdkerrors.Wrap(sdkerrors.ErrJSONMarshal, err.Error())
+	_, err := keeper.BlockingSend(ctx, action)
+	if err == nil {
+		err = keeper.PushAction(ctx, action)
 	}
 
-	_, err = keeper.CallToController(ctx, string(b))
-
 	// fmt.Fprintln(os.Stderr, "Returned from SwingSet", out, err)
 	return err
 }
@@ -68,14 +63,12 @@ func EndBlock(ctx sdk.Context, req abci.RequestEndBlock, keeper Keeper) ([]abci.
 		BlockHeight: ctx.BlockHeight(),
 		BlockTime:   ctx.BlockTime().Unix(),
 		StoragePort: vm.GetPort("storage"),
-		Params:      keeper.GetParams(ctx),
-	}
-	b, err := json.Marshal(action)
-	if err != nil {
-		return nil, sdkerrors.Wrap(sdkerrors.ErrJSONMarshal, err.Error())
 	}
 
-	_, err = keeper.CallToController(ctx, string(b))
+	err := keeper.PushAction(ctx, action)
+	if err == nil {
+		_, err = keeper.BlockingSend(ctx, action)
+	}
 
 	// fmt.Fprintln(os.Stderr, "Returned from SwingSet", out, err)
 	if err != nil {
@@ -99,14 +92,7 @@ func CommitBlock(keeper Keeper) error {
 		BlockHeight: endBlockHeight,
 		BlockTime:   endBlockTime,
 	}
-	vm.SetCommittedHeight(endBlockHeight)
-
-	b, err := json.Marshal(action)
-	if err != nil {
-		return sdkerrors.Wrap(sdkerrors.ErrJSONMarshal, err.Error())
-	}
-
-	_, err = keeper.CallToController(sdk.Context{}, string(b))
+	_, err := keeper.BlockingSend(sdk.Context{}, action)
 
 	// fmt.Fprintln(os.Stderr, "Returned from SwingSet", out, err)
 	if err != nil {

golang/cosmos/x/swingset/genesis.go

@@ -2,7 +2,7 @@ package swingset
 
 import (
 	// "fmt"
-	"encoding/json"
+
 	"fmt"
 	stdlog "log"
 
@@ -56,12 +56,8 @@ func InitGenesis(ctx sdk.Context, keeper Keeper, data *types.GenesisState) []abc
 		BlockTime:   ctx.BlockTime().Unix(),
 		StoragePort: vm.GetPort("storage"),
 	}
-	b, err := json.Marshal(action)
-	if err != nil {
-		panic(err)
-	}
 
-	_, err = keeper.CallToController(ctx, string(b))
+	_, err := keeper.BlockingSend(ctx, action)
 
 	if err != nil {
 		// NOTE: A failed BOOTSTRAP_BLOCK means that the SwingSet state is inconsistent.

golang/cosmos/x/swingset/keeper/keeper.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"sort"
+	"strconv"
 	"strings"
 
 	"github.com/tendermint/tendermint/libs/log"
@@ -28,7 +29,7 @@ type Keeper struct {
 	feeCollectorName string
 
 	// CallToController dispatches a message to the controlling process
-	CallToController func(ctx sdk.Context, str string) (string, error)
+	callToController func(ctx sdk.Context, str string) (string, error)
 }
 
 var _ types.SwingSetKeeper = &Keeper{}
@@ -68,10 +69,49 @@ func NewKeeper(
 		accountKeeper:    accountKeeper,
 		bankKeeper:       bankKeeper,
 		feeCollectorName: feeCollectorName,
-		CallToController: callToController,
+		callToController: callToController,
 	}
 }
 
+// PushAction appends an action to the controller's action queue.  This queue is
+// kept in the kvstore so that changes to it are properly reverted if the
+// kvstore as rolled back.  By the time the block manager runs, it can commit
+// its SwingSet transactions without fear of side-effecting the world with
+// intermediate transaction state.
+func (k Keeper) PushAction(ctx sdk.Context, obj interface{}) error {
+	bz, err := json.Marshal(obj)
+	if err != nil {
+		return err
+	}
+	tailStr := k.GetStorage(ctx, "actionQueue.tail")
+	var tail uint64
+	if len(tailStr) > 0 {
+		tail, err = strconv.ParseUint(tailStr, 10, 64)
+		if err != nil {
+			return err
+		}
+	}
+	k.SetStorage(ctx, fmt.Sprintf("actionQueue.%d", tail), string(bz))
+	bz, err = json.Marshal(tail + 1)
+	if err != nil {
+		return err
+	}
+	k.SetStorage(ctx, "actionQueue.tail", string(bz))
+	return nil
+}
+
+// BlockingSend sends a message to the controller and blocks the Golang process
+// until the response.  It is orthogonal to PushAction, and should only be used
+// by SwingSet to perform block lifecycle events (BEGIN_BLOCK, END_BLOCK,
+// COMMIT_BLOCK).
+func (k Keeper) BlockingSend(ctx sdk.Context, obj interface{}) (string, error) {
+	bz, err := json.Marshal(obj)
+	if err != nil {
+		return "", err
+	}
+	return k.callToController(ctx, string(bz))
+}
+
 func (k Keeper) GetParams(ctx sdk.Context) (params types.Params) {
 	k.paramSpace.GetParamSet(ctx, &params)
 	return params

golang/cosmos/x/swingset/keeper/msg_server.go

@@ -2,7 +2,6 @@ package keeper
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 
 	"github.com/Agoric/agoric-sdk/golang/cosmos/vm"
@@ -31,7 +30,6 @@ type deliverInboundAction struct {
 	StoragePort int             `json:"storagePort"`
 	BlockHeight int64           `json:"blockHeight"`
 	BlockTime   int64           `json:"blockTime"`
-	Params      types.Params    `json:"params"`
 }
 
 func (keeper msgServer) DeliverInbound(goCtx context.Context, msg *types.MsgDeliverInbound) (*types.MsgDeliverInboundResponse, error) {
@@ -52,15 +50,9 @@ func (keeper msgServer) DeliverInbound(goCtx context.Context, msg *types.MsgDeli
 		StoragePort: vm.GetPort("storage"),
 		BlockHeight: ctx.BlockHeight(),
 		BlockTime:   ctx.BlockTime().Unix(),
-		Params:      keeper.GetParams(ctx),
-	}
-	// fmt.Fprintf(os.Stderr, "Context is %+v\n", ctx)
-	b, err := json.Marshal(action)
-	if err != nil {
-		return nil, sdkerrors.Wrap(sdkerrors.ErrJSONMarshal, err.Error())
 	}
 
-	_, err = keeper.CallToController(ctx, string(b))
+	err := keeper.PushAction(ctx, action)
 	// fmt.Fprintln(os.Stderr, "Returned from SwingSet", out, err)
 	if err != nil {
 		return nil, err
@@ -87,12 +79,8 @@ func (keeper msgServer) WalletAction(goCtx context.Context, msg *types.MsgWallet
 		BlockTime:   ctx.BlockTime().Unix(),
 	}
 	// fmt.Fprintf(os.Stderr, "Context is %+v\n", ctx)
-	b, err := json.Marshal(action)
-	if err != nil {
-		return nil, sdkerrors.Wrap(sdkerrors.ErrJSONMarshal, err.Error())
-	}
 
-	_, err = keeper.CallToController(ctx, string(b))
+	err := keeper.PushAction(ctx, action)
 	// fmt.Fprintln(os.Stderr, "Returned from SwingSet", out, err)
 	if err != nil {
 		return nil, err
@@ -119,13 +107,7 @@ func (keeper msgServer) WalletSpendAction(goCtx context.Context, msg *types.MsgW
 		BlockTime:   ctx.BlockTime().Unix(),
 	}
 	// fmt.Fprintf(os.Stderr, "Context is %+v\n", ctx)
-	b, err := json.Marshal(action)
-	if err != nil {
-		return nil, sdkerrors.Wrap(sdkerrors.ErrJSONMarshal, err.Error())
-	}
-
-	_, err = keeper.CallToController(ctx, string(b))
-	// fmt.Fprintln(os.Stderr, "Returned from SwingSet", out, err)
+	err := keeper.PushAction(ctx, action)
 	if err != nil {
 		return nil, err
 	}
@@ -157,20 +139,15 @@ func (keeper msgServer) Provision(goCtx context.Context, msg *types.MsgProvision
 		BlockHeight:  ctx.BlockHeight(),
 		BlockTime:    ctx.BlockTime().Unix(),
 	}
-	// fmt.Fprintf(os.Stderr, "Context is %+v\n", ctx)
-	b, err := json.Marshal(action)
-	if err != nil {
-		return nil, sdkerrors.Wrap(sdkerrors.ErrJSONMarshal, err.Error())
-	}
 
 	// Create the account, if it doesn't already exist.
 	egress := types.NewEgress(msg.Nickname, msg.Address, msg.PowerFlags)
-	err = keeper.SetEgress(ctx, egress)
+	err := keeper.SetEgress(ctx, egress)
 	if err != nil {
 		return nil, err
 	}
 
-	_, err = keeper.CallToController(ctx, string(b))
+	err = keeper.PushAction(ctx, action)
 	// fmt.Fprintln(os.Stderr, "Returned from SwingSet", out, err)
 	if err != nil {
 		return nil, err

golang/cosmos/x/swingset/keeper/proposal.go

@@ -1,10 +1,7 @@
 package keeper
 
 import (
-	"encoding/json"
-
 	sdk "github.com/cosmos/cosmos-sdk/types"
-	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
 
 	"github.com/Agoric/agoric-sdk/golang/cosmos/x/swingset/types"
 )
@@ -25,15 +22,5 @@ func (k Keeper) CoreEvalProposal(ctx sdk.Context, p *types.CoreEvalProposal) err
 		BlockTime:   ctx.BlockTime().Unix(),
 	}
 
-	bz, err := json.Marshal(action)
-	if err != nil {
-		return sdkerrors.Wrap(sdkerrors.ErrJSONMarshal, err.Error())
-	}
-
-	_, err = k.CallToController(ctx, string(bz))
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return k.PushAction(ctx, action)
 }