From d791ee4d097296e3a46a421b0548f99d921cb8ff Mon Sep 17 00:00:00 2001 From: Hleb Albau Date: Thu, 27 Dec 2018 19:27:00 +0300 Subject: [PATCH] #3194 Configurable pruning. Fix docs and create strategies const. --- PENDING.md | 1 + baseapp/baseapp_test.go | 8 +++-- baseapp/options.go | 17 ++--------- cmd/gaia/cmd/gaiad/main.go | 3 +- cmd/gaia/cmd/gaiadebug/hack.go | 3 +- cmd/gaia/cmd/gaiareplay/main.go | 3 +- docs/examples/basecoin/cmd/basecoind/main.go | 3 +- server/mock/store.go | 2 +- store/codec.go | 2 +- store/dbstoreadapter.go | 2 +- store/iavlstore.go | 17 +++-------- store/multistoreproof_test.go | 2 +- store/pruning.go | 29 ++++++++++++++++++ store/rootmultistore.go | 10 +++--- store/rootmultistore_test.go | 1 + store/transientstore.go | 2 +- types/store.go | 32 +++++++++++++------- 17 files changed, 81 insertions(+), 56 deletions(-) create mode 100644 store/pruning.go diff --git a/PENDING.md b/PENDING.md index 8be9469675f0..610a9432d269 100644 --- a/PENDING.md +++ b/PENDING.md @@ -24,6 +24,7 @@ BREAKING CHANGES * [stake] \#2513 Validator power type from Dec -> Int * [stake] \#3233 key and value now contain duplicate fields to simplify code * [\#3064](https://github.com/cosmos/cosmos-sdk/issues/3064) Sanitize `sdk.Coin` denom. Coins denoms are now case insensitive, i.e. 100fooToken equals to 100FOOTOKEN. + * [\#3195](https://github.com/cosmos/cosmos-sdk/issues/3195) Allows custom configuration for syncable strategy * Tendermint diff --git a/baseapp/baseapp_test.go b/baseapp/baseapp_test.go index 8bde78585d10..d088b841fc45 100644 --- a/baseapp/baseapp_test.go +++ b/baseapp/baseapp_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "fmt" + "github.com/cosmos/cosmos-sdk/store" "os" "testing" @@ -85,9 +86,10 @@ func TestMountStores(t *testing.T) { // Test that LoadLatestVersion actually does. func TestLoadVersion(t *testing.T) { logger := defaultLogger() + pruningOpt := SetPruning(store.PruneSyncable) db := dbm.NewMemDB() name := t.Name() - app := NewBaseApp(name, logger, db, nil) + app := NewBaseApp(name, logger, db, nil, pruningOpt) // make a cap key and mount the store capKey := sdk.NewKVStoreKey(MainStoreKey) @@ -116,7 +118,7 @@ func TestLoadVersion(t *testing.T) { commitID2 := sdk.CommitID{2, res.Data} // reload with LoadLatestVersion - app = NewBaseApp(name, logger, db, nil) + app = NewBaseApp(name, logger, db, nil, pruningOpt) app.MountStores(capKey) err = app.LoadLatestVersion(capKey) require.Nil(t, err) @@ -124,7 +126,7 @@ func TestLoadVersion(t *testing.T) { // reload with LoadVersion, see if you can commit the same block and get // the same result - app = NewBaseApp(name, logger, db, nil) + app = NewBaseApp(name, logger, db, nil, pruningOpt) app.MountStores(capKey) err = app.LoadVersion(1, capKey) require.Nil(t, err) diff --git a/baseapp/options.go b/baseapp/options.go index 9ec9db023897..8729e5ac1287 100644 --- a/baseapp/options.go +++ b/baseapp/options.go @@ -14,21 +14,8 @@ import ( // for options that need access to non-exported fields of the BaseApp // SetPruning sets a pruning option on the multistore associated with the app -func SetPruning(pruning string) func(*BaseApp) { - var pruningEnum sdk.PruningStrategy - switch pruning { - case "nothing": - pruningEnum = sdk.PruneNothing - case "everything": - pruningEnum = sdk.PruneEverything - case "syncable": - pruningEnum = sdk.PruneSyncable - default: - panic(fmt.Sprintf("invalid pruning strategy: %s", pruning)) - } - return func(bap *BaseApp) { - bap.cms.SetPruning(pruningEnum) - } +func SetPruning(opts sdk.PruningOptions) func(*BaseApp) { + return func(bap *BaseApp) { bap.cms.SetPruning(opts) } } // SetMinimumFees returns an option that sets the minimum fees on the app. diff --git a/cmd/gaia/cmd/gaiad/main.go b/cmd/gaia/cmd/gaiad/main.go index 2791415c9350..6d597eed7089 100644 --- a/cmd/gaia/cmd/gaiad/main.go +++ b/cmd/gaia/cmd/gaiad/main.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "github.com/cosmos/cosmos-sdk/store" "io" "github.com/cosmos/cosmos-sdk/baseapp" @@ -56,7 +57,7 @@ func main() { func newApp(logger log.Logger, db dbm.DB, traceStore io.Writer) abci.Application { return app.NewGaiaApp(logger, db, traceStore, true, - baseapp.SetPruning(viper.GetString("pruning")), + baseapp.SetPruning(store.NewPruningOptions(viper.GetString("pruning"))), baseapp.SetMinimumFees(viper.GetString("minimum_fees")), ) } diff --git a/cmd/gaia/cmd/gaiadebug/hack.go b/cmd/gaia/cmd/gaiadebug/hack.go index 41b9e41fafb3..5a4a847390e9 100644 --- a/cmd/gaia/cmd/gaiadebug/hack.go +++ b/cmd/gaia/cmd/gaiadebug/hack.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "encoding/hex" "fmt" + "github.com/cosmos/cosmos-sdk/store" "os" "path" @@ -48,7 +49,7 @@ func runHackCmd(cmd *cobra.Command, args []string) error { fmt.Println(err) os.Exit(1) } - app := NewGaiaApp(logger, db, baseapp.SetPruning(viper.GetString("pruning"))) + app := NewGaiaApp(logger, db, baseapp.SetPruning(store.NewPruningOptions(viper.GetString("pruning")))) // print some info id := app.LastCommitID() diff --git a/cmd/gaia/cmd/gaiareplay/main.go b/cmd/gaia/cmd/gaiareplay/main.go index 7e6392bf1230..cf946edce3c1 100644 --- a/cmd/gaia/cmd/gaiareplay/main.go +++ b/cmd/gaia/cmd/gaiareplay/main.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "github.com/cosmos/cosmos-sdk/store" "io" "os" "path/filepath" @@ -106,7 +107,7 @@ func run(rootDir string) { fmt.Println("Creating application") myapp := app.NewGaiaApp( ctx.Logger, appDB, traceStoreWriter, true, - baseapp.SetPruning("everything"), // nothing + baseapp.SetPruning(store.PruneEverything), // nothing ) // Genesis diff --git a/docs/examples/basecoin/cmd/basecoind/main.go b/docs/examples/basecoin/cmd/basecoind/main.go index 9cb246671858..8b5351da7bc0 100644 --- a/docs/examples/basecoin/cmd/basecoind/main.go +++ b/docs/examples/basecoin/cmd/basecoind/main.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "fmt" + "github.com/cosmos/cosmos-sdk/store" "io" "os" @@ -122,7 +123,7 @@ func InitCmd(ctx *server.Context, cdc *codec.Codec) *cobra.Command { } func newApp(logger log.Logger, db dbm.DB, storeTracer io.Writer) abci.Application { - return app.NewBasecoinApp(logger, db, baseapp.SetPruning(viper.GetString("pruning"))) + return app.NewBasecoinApp(logger, db, baseapp.SetPruning(store.NewPruningOptions(viper.GetString("pruning")))) } func exportAppStateAndTMValidators(logger log.Logger, db dbm.DB, storeTracer io.Writer, _ int64, _ bool) ( diff --git a/server/mock/store.go b/server/mock/store.go index ec963a1bc22c..3aecc1734407 100644 --- a/server/mock/store.go +++ b/server/mock/store.go @@ -50,7 +50,7 @@ func (ms multiStore) LastCommitID() sdk.CommitID { panic("not implemented") } -func (ms multiStore) SetPruning(s sdk.PruningStrategy) { +func (ms multiStore) SetPruning(opts sdk.PruningOptions) { panic("not implemented") } diff --git a/store/codec.go b/store/codec.go index 353cd2e3cd77..181f12e511b7 100644 --- a/store/codec.go +++ b/store/codec.go @@ -7,7 +7,7 @@ import ( // Import cosmos-sdk/types/store.go for convenience. // nolint type ( - PruningStrategy = types.PruningStrategy + PruningOptions = types.PruningOptions Store = types.Store Committer = types.Committer CommitStore = types.CommitStore diff --git a/store/dbstoreadapter.go b/store/dbstoreadapter.go index 76e673de5a67..b662bcf45ddd 100644 --- a/store/dbstoreadapter.go +++ b/store/dbstoreadapter.go @@ -64,4 +64,4 @@ func (cdsa commitDBStoreAdapter) LastCommitID() CommitID { } } -func (cdsa commitDBStoreAdapter) SetPruning(_ PruningStrategy) {} +func (cdsa commitDBStoreAdapter) SetPruning(_ PruningOptions) {} diff --git a/store/iavlstore.go b/store/iavlstore.go index fccde38e27e0..26c739da3bf3 100644 --- a/store/iavlstore.go +++ b/store/iavlstore.go @@ -19,7 +19,7 @@ const ( ) // load the iavl store -func LoadIAVLStore(db dbm.DB, id CommitID, pruning sdk.PruningStrategy) (CommitStore, error) { +func LoadIAVLStore(db dbm.DB, id CommitID, pruning sdk.PruningOptions) (CommitStore, error) { tree := iavl.NewMutableTree(db, defaultIAVLCacheSize) _, err := tree.LoadVersion(id.Version) if err != nil { @@ -38,7 +38,6 @@ var _ Queryable = (*iavlStore)(nil) // iavlStore Implements KVStore and CommitStore. type iavlStore struct { - // The underlying tree. tree *iavl.MutableTree @@ -102,17 +101,9 @@ func (st *iavlStore) LastCommitID() CommitID { } // Implements Committer. -func (st *iavlStore) SetPruning(pruning sdk.PruningStrategy) { - switch pruning { - case sdk.PruneEverything: - st.numRecent = 0 - st.storeEvery = 0 - case sdk.PruneNothing: - st.storeEvery = 1 - case sdk.PruneSyncable: - st.numRecent = 100 - st.storeEvery = 10000 - } +func (st *iavlStore) SetPruning(opt sdk.PruningOptions) { + st.numRecent = opt.KeepRecent() + st.storeEvery = opt.KeepEvery() } // VersionExists returns whether or not a given version is stored. diff --git a/store/multistoreproof_test.go b/store/multistoreproof_test.go index 0f80657b84b6..3d70451e7273 100644 --- a/store/multistoreproof_test.go +++ b/store/multistoreproof_test.go @@ -13,7 +13,7 @@ import ( func TestVerifyIAVLStoreQueryProof(t *testing.T) { // Create main tree for testing. db := dbm.NewMemDB() - iStore, err := LoadIAVLStore(db, CommitID{}, sdk.PruneNothing) + iStore, err := LoadIAVLStore(db, CommitID{}, PruneNothing) store := iStore.(*iavlStore) require.Nil(t, err) store.Set([]byte("MYKEY"), []byte("MYVALUE")) diff --git a/store/pruning.go b/store/pruning.go new file mode 100644 index 000000000000..9a7aeb4d29e6 --- /dev/null +++ b/store/pruning.go @@ -0,0 +1,29 @@ +package store + +import ( + sdk "github.com/cosmos/cosmos-sdk/types" +) + +// default pruning strategies +var ( + // PruneEverything means all saved states will be deleted, storing only the current state + PruneEverything = sdk.NewPruningOptions(0, 0) + // PruneNothing means all historic states will be saved, nothing will be deleted + PruneNothing = sdk.NewPruningOptions(0, 1) + // PruneSyncable means only those states not needed for state syncing will be deleted (keeps last 100 + every 10000th) + PruneSyncable = sdk.NewPruningOptions(100, 10000) +) + +func NewPruningOptions(strategy string) (opt PruningOptions) { + switch strategy { + case "nothing": + opt = PruneNothing + case "everything": + opt = PruneEverything + case "syncable": + opt = PruneSyncable + default: + opt = PruneSyncable + } + return +} diff --git a/store/rootmultistore.go b/store/rootmultistore.go index c309f9e9b28a..fa576d3dcc6a 100644 --- a/store/rootmultistore.go +++ b/store/rootmultistore.go @@ -24,7 +24,7 @@ const ( type rootMultiStore struct { db dbm.DB lastCommitID CommitID - pruning sdk.PruningStrategy + pruningOpts sdk.PruningOptions storesParams map[StoreKey]storeParams stores map[StoreKey]CommitStore keysByName map[string]StoreKey @@ -47,10 +47,10 @@ func NewCommitMultiStore(db dbm.DB) *rootMultiStore { } // Implements CommitMultiStore -func (rs *rootMultiStore) SetPruning(pruning sdk.PruningStrategy) { - rs.pruning = pruning +func (rs *rootMultiStore) SetPruning(pruningOpts sdk.PruningOptions) { + rs.pruningOpts = pruningOpts for _, substore := range rs.stores { - substore.SetPruning(pruning) + substore.SetPruning(pruningOpts) } } @@ -355,7 +355,7 @@ func (rs *rootMultiStore) loadCommitStoreFromParams(key sdk.StoreKey, id CommitI // TODO: id? // return NewCommitMultiStore(db, id) case sdk.StoreTypeIAVL: - store, err = LoadIAVLStore(db, id, rs.pruning) + store, err = LoadIAVLStore(db, id, rs.pruningOpts) return case sdk.StoreTypeDB: store = commitDBStoreAdapter{dbStoreAdapter{db}} diff --git a/store/rootmultistore_test.go b/store/rootmultistore_test.go index cd555d6f2d5a..10f0956562ba 100644 --- a/store/rootmultistore_test.go +++ b/store/rootmultistore_test.go @@ -195,6 +195,7 @@ func TestMultiStoreQuery(t *testing.T) { func newMultiStoreWithMounts(db dbm.DB) *rootMultiStore { store := NewCommitMultiStore(db) + store.pruningOpts = PruneSyncable store.MountStoreWithDB( sdk.NewKVStoreKey("store1"), sdk.StoreTypeIAVL, nil) store.MountStoreWithDB( diff --git a/store/transientstore.go b/store/transientstore.go index 63b154c017f7..2de1197b977d 100644 --- a/store/transientstore.go +++ b/store/transientstore.go @@ -26,7 +26,7 @@ func (ts *transientStore) Commit() (id CommitID) { } // Implements CommitStore -func (ts *transientStore) SetPruning(pruning PruningStrategy) { +func (ts *transientStore) SetPruning(opts PruningOptions) { } // Implements CommitStore diff --git a/types/store.go b/types/store.go index 671a881c28be..3118fba8f166 100644 --- a/types/store.go +++ b/types/store.go @@ -12,19 +12,29 @@ import ( // NOTE: These are implemented in cosmos-sdk/store. -// PruningStrategy specfies how old states will be deleted over time -type PruningStrategy uint8 +// PruningStrategy specifies how old states will be deleted over time where +// keepRecent can be used with keepEvery to create a pruning "strategy". +type PruningOptions struct { + keepRecent int64 + keepEvery int64 +} -const ( - // PruneSyncable means only those states not needed for state syncing will be deleted (keeps last 100 + every 10000th) - PruneSyncable PruningStrategy = iota +func NewPruningOptions(keepRecent, keepEvery int64) PruningOptions { + return PruningOptions{ + keepRecent: keepRecent, + keepEvery: keepEvery, + } +} - // PruneEverything means all saved states will be deleted, storing only the current state - PruneEverything PruningStrategy = iota +// How much recent state will be kept. Older state will be deleted. +func (po PruningOptions) KeepRecent() int64 { + return po.keepRecent +} - // PruneNothing means all historic states will be saved, nothing will be deleted - PruneNothing PruningStrategy = iota -) +// Keeps every N stated, deleting others. +func (po PruningOptions) KeepEvery() int64 { + return po.keepEvery +} type Store interface { //nolint GetStoreType() StoreType @@ -35,7 +45,7 @@ type Store interface { //nolint type Committer interface { Commit() CommitID LastCommitID() CommitID - SetPruning(PruningStrategy) + SetPruning(PruningOptions) } // Stores of MultiStore must implement CommitStore.