From 6a420a8d371d98a6b6e8400cf25118443f958166 Mon Sep 17 00:00:00 2001 From: Christopher Goes Date: Tue, 29 Jan 2019 18:22:53 +0100 Subject: [PATCH 1/6] Update PENDING.md --- PENDING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/PENDING.md b/PENDING.md index ad915ada610e..6b8f4b51cac8 100644 --- a/PENDING.md +++ b/PENDING.md @@ -22,6 +22,7 @@ FEATURES * Gaia - [\#3397](https://github.com/cosmos/cosmos-sdk/pull/3397) Implement genesis file sanitization to avoid failures at chain init. + * \#3182 Run the simulation from a particular genesis state loaded from a file * SDK * \#3270 [x/staking] limit number of ongoing unbonding delegations /redelegations per pair/trio From 494d83ff892207d52300b6abbe2c8aaa468959a5 Mon Sep 17 00:00:00 2001 From: Christopher Goes Date: Tue, 29 Jan 2019 18:40:18 +0100 Subject: [PATCH 2/6] Work in progress --- Makefile | 11 +++++++++++ PENDING.md | 2 +- cmd/gaia/app/sim_test.go | 32 +++++++++++++++++++++++++------- scripts/multisim.sh | 4 +++- 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 96be180fd293..472efcc676f9 100644 --- a/Makefile +++ b/Makefile @@ -150,6 +150,11 @@ test_sim_gaia_nondeterminism: @echo "Running nondeterminism test..." @go test ./cmd/gaia/app -run TestAppStateDeterminism -SimulationEnabled=true -v -timeout 10m +test_sim_gaia_custom_genesis_fast: + @echo "Running custom genesis simulation..." + @echo "By default, ${HOME}/.gaiad/config/genesis.json will be used." + @go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationGenesis=${HOME}/.gaiad/config/genesis.json -SimulationEnabled=true -SimulationNumBlocks=1000 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=99 -v -timeout 24h + test_sim_gaia_fast: @echo "Running quick Gaia simulation. This may take several minutes..." @go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=1000 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=99 -v -timeout 24h @@ -162,6 +167,11 @@ test_sim_gaia_simulation_after_import: @echo "Running Gaia simulation-after-import. This may take several minutes..." @bash scripts/multisim.sh 50 5 TestGaiaSimulationAfterImport +test_sim_gaia_custom_genesis_multi_seed: + @echo "Running multi-seed custom genesis simulation..." + @echo "By default, ${HOME}/.gaiad/config/genesis.json will be used." + @bash scripts/multisim.sh 400 5 TestFullGaiaSimulation ${HOME}/.gaiad/config/genesis.json + test_sim_gaia_multi_seed: @echo "Running multi-seed Gaia simulation. This may take awhile!" @bash scripts/multisim.sh 400 5 TestFullGaiaSimulation @@ -255,5 +265,6 @@ check_tools check_dev_tools get_vendor_deps draw_deps test test_cli test_unit \ test_cover test_lint benchmark devdoc_init devdoc devdoc_save devdoc_update \ build-linux build-docker-gaiadnode localnet-start localnet-stop \ format check-ledger test_sim_gaia_nondeterminism test_sim_modules test_sim_gaia_fast \ +test_sim_gaia_custom_genesis_fast test_sim_gaia_custom_genesis_multi_seed \ test_sim_gaia_multi_seed test_sim_gaia_import_export update_tools update_dev_tools \ build-snap-edge diff --git a/PENDING.md b/PENDING.md index 6b8f4b51cac8..c55a33263a00 100644 --- a/PENDING.md +++ b/PENDING.md @@ -22,7 +22,7 @@ FEATURES * Gaia - [\#3397](https://github.com/cosmos/cosmos-sdk/pull/3397) Implement genesis file sanitization to avoid failures at chain init. - * \#3182 Run the simulation from a particular genesis state loaded from a file + * \#3428 Run the simulation from a particular genesis state loaded from a file * SDK * \#3270 [x/staking] limit number of ongoing unbonding delegations /redelegations per pair/trio diff --git a/cmd/gaia/app/sim_test.go b/cmd/gaia/app/sim_test.go index 44e53425c7be..f0b055062498 100644 --- a/cmd/gaia/app/sim_test.go +++ b/cmd/gaia/app/sim_test.go @@ -15,6 +15,7 @@ import ( abci "github.com/tendermint/tendermint/abci/types" dbm "github.com/tendermint/tendermint/libs/db" "github.com/tendermint/tendermint/libs/log" + tmtypes "github.com/tendermint/tendermint/types" "github.com/cosmos/cosmos-sdk/baseapp" sdk "github.com/cosmos/cosmos-sdk/types" @@ -36,16 +37,18 @@ import ( ) var ( - seed int64 - numBlocks int - blockSize int - enabled bool - verbose bool - commit bool - period int + genesisFile string + seed int64 + numBlocks int + blockSize int + enabled bool + verbose bool + commit bool + period int ) func init() { + flag.StringVar(&genesisFile, "SimulationGenesis", "", "Custom simulation genesis file") flag.Int64Var(&seed, "SimulationSeed", 42, "Simulation random seed") flag.IntVar(&numBlocks, "SimulationNumBlocks", 500, "Number of blocks") flag.IntVar(&blockSize, "SimulationBlockSize", 200, "Operations per block") @@ -56,6 +59,21 @@ func init() { } func appStateFn(r *rand.Rand, accs []simulation.Account, genesisTimestamp time.Time) json.RawMessage { + + if genesisFile != "" { + var genesis tmtypes.GenesisDoc + cdc := MakeCodec() + bytes, err := ioutil.ReadFile(genesisFile) + if err != nil { + panic(err) + } + if err = cdc.UnmarshalJSON(bytes, &genesis); err != nil { + panic(err) + } + fmt.Printf("app state: %s\n", json.RawMessage(genesis.AppState)) + return json.RawMessage(genesis.AppState) + } + var genesisAccounts []GenesisAccount amount := int64(r.Intn(1e6)) diff --git a/scripts/multisim.sh b/scripts/multisim.sh index e27035983dde..e8df463dcf8a 100755 --- a/scripts/multisim.sh +++ b/scripts/multisim.sh @@ -5,10 +5,12 @@ seeds=(1 2 4 7 9 20 32 123 124 582 1893 2989 3012 4728 37827 981928 87821 891823 blocks=$1 period=$2 testname=$3 +genesis=$4 echo "Running multi-seed simulation with seeds ${seeds[@]}" echo "Running $blocks blocks per seed" echo "Running test $testname" +echo "Using genesis file $genesis" echo "Edit scripts/multisim.sh to add new seeds. Keeping parameters in the file makes failures easy to reproduce." echo "This script will kill all sub-simulations on SIGINT/SIGTERM (i.e. Ctrl-C)." @@ -22,7 +24,7 @@ sim() { echo "Running Gaia simulation with seed $seed. This may take awhile!" file="$tmpdir/gaia-simulation-seed-$seed-date-$(date -u +"%Y-%m-%dT%H:%M:%S+00:00").stdout" echo "Writing stdout to $file..." - go test ./cmd/gaia/app -run $testname -SimulationEnabled=true -SimulationNumBlocks=$blocks \ + go test ./cmd/gaia/app -run $testname -SimulationEnabled=true -SimulationNumBlocks=$blocks -SimulationGenesis=$genesis \ -SimulationVerbose=true -SimulationCommit=true -SimulationSeed=$seed -SimulationPeriod=$period -v -timeout 24h > $file } From d80171f65a0a00d693e543e8ca4e668f6b8b29ff Mon Sep 17 00:00:00 2001 From: Christopher Goes Date: Tue, 29 Jan 2019 20:20:10 +0100 Subject: [PATCH 3/6] Functional, needs cleanup --- cmd/gaia/app/sim_test.go | 19 +++++++++++++++---- x/mock/simulation/simulate.go | 17 ++++++++++++----- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/cmd/gaia/app/sim_test.go b/cmd/gaia/app/sim_test.go index f0b055062498..cacc75f560f3 100644 --- a/cmd/gaia/app/sim_test.go +++ b/cmd/gaia/app/sim_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/crypto/secp256k1" dbm "github.com/tendermint/tendermint/libs/db" "github.com/tendermint/tendermint/libs/log" tmtypes "github.com/tendermint/tendermint/types" @@ -58,7 +59,7 @@ func init() { flag.IntVar(&period, "SimulationPeriod", 1, "Run slow invariants only once every period assertions") } -func appStateFn(r *rand.Rand, accs []simulation.Account, genesisTimestamp time.Time) json.RawMessage { +func appStateFn(r *rand.Rand, accs []simulation.Account, genesisTimestamp time.Time) (json.RawMessage, []simulation.Account, string) { if genesisFile != "" { var genesis tmtypes.GenesisDoc @@ -70,8 +71,18 @@ func appStateFn(r *rand.Rand, accs []simulation.Account, genesisTimestamp time.T if err = cdc.UnmarshalJSON(bytes, &genesis); err != nil { panic(err) } - fmt.Printf("app state: %s\n", json.RawMessage(genesis.AppState)) - return json.RawMessage(genesis.AppState) + var appState GenesisState + if err = cdc.UnmarshalJSON(genesis.AppState, &appState); err != nil { + panic(err) + } + var newAccs []simulation.Account + for _, acc := range appState.Accounts { + privkeySeed := make([]byte, 15) + r.Read(privkeySeed) + privKey := secp256k1.GenPrivKeySecp256k1(privkeySeed) + newAccs = append(newAccs, simulation.Account{privKey, privKey.PubKey(), acc.Address}) + } + return json.RawMessage(genesis.AppState), newAccs, genesis.ChainID } var genesisAccounts []GenesisAccount @@ -239,7 +250,7 @@ func appStateFn(r *rand.Rand, accs []simulation.Account, genesisTimestamp time.T panic(err) } - return appState + return appState, accs, "simulation" } func randIntBetween(r *rand.Rand, min, max int) int { diff --git a/x/mock/simulation/simulate.go b/x/mock/simulation/simulate.go index 81e360a1a827..b8dc6820c084 100644 --- a/x/mock/simulation/simulate.go +++ b/x/mock/simulation/simulate.go @@ -19,7 +19,7 @@ import ( ) // AppStateFn returns the app state json bytes -type AppStateFn func(r *rand.Rand, accs []Account, genesisTimestamp time.Time) json.RawMessage +type AppStateFn func(r *rand.Rand, accs []Account, genesisTimestamp time.Time) (json.RawMessage, []Account, string) // Simulate tests application by sending random messages. func Simulate(t *testing.T, app *baseapp.BaseApp, @@ -35,15 +35,18 @@ func Simulate(t *testing.T, app *baseapp.BaseApp, func initChain( r *rand.Rand, params Params, accounts []Account, app *baseapp.BaseApp, appStateFn AppStateFn, genesisTimestamp time.Time, -) mockValidators { +) (mockValidators, []Account) { + + appState, accounts, chainID := appStateFn(r, accounts, genesisTimestamp) req := abci.RequestInitChain{ - AppStateBytes: appStateFn(r, accounts, genesisTimestamp), + AppStateBytes: appState, + ChainId: chainID, } res := app.InitChain(req) validators := newMockValidators(r, res.Validators, params) - return validators + return validators, accounts } // SimulateFromSeed tests an application by running the provided @@ -73,7 +76,11 @@ func SimulateFromSeed(tb testing.TB, app *baseapp.BaseApp, // Second variable to keep pending validator set (delayed one block since // TM 0.24) Initially this is the same as the initial validator set - validators := initChain(r, params, accs, app, appStateFn, genesisTimestamp) + validators, accs := initChain(r, params, accs, app, appStateFn, genesisTimestamp) + if len(accs) == 0 { + return true, fmt.Errorf("must have greater than zero genesis accounts") + } + nextValidators := validators header := abci.Header{ From 65a759f141074041b0ca50d7659a04af2379ee3f Mon Sep 17 00:00:00 2001 From: Christopher Goes Date: Tue, 29 Jan 2019 20:29:25 +0100 Subject: [PATCH 4/6] Cleanup a bit --- Makefile | 4 +-- cmd/gaia/app/sim_test.go | 57 +++++++++++++++++++++++----------------- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index 472efcc676f9..54495112268b 100644 --- a/Makefile +++ b/Makefile @@ -153,11 +153,11 @@ test_sim_gaia_nondeterminism: test_sim_gaia_custom_genesis_fast: @echo "Running custom genesis simulation..." @echo "By default, ${HOME}/.gaiad/config/genesis.json will be used." - @go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationGenesis=${HOME}/.gaiad/config/genesis.json -SimulationEnabled=true -SimulationNumBlocks=1000 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=99 -v -timeout 24h + @go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationGenesis=${HOME}/.gaiad/config/genesis.json -SimulationEnabled=true -SimulationNumBlocks=100 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=99 -SimulationPeriod=5 -v -timeout 24h test_sim_gaia_fast: @echo "Running quick Gaia simulation. This may take several minutes..." - @go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=1000 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=99 -v -timeout 24h + @go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=100 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=99 -SimulationPeriod=5 -v -timeout 24h test_sim_gaia_import_export: @echo "Running Gaia import/export simulation. This may take several minutes..." diff --git a/cmd/gaia/app/sim_test.go b/cmd/gaia/app/sim_test.go index cacc75f560f3..f75f412cc5ed 100644 --- a/cmd/gaia/app/sim_test.go +++ b/cmd/gaia/app/sim_test.go @@ -59,31 +59,33 @@ func init() { flag.IntVar(&period, "SimulationPeriod", 1, "Run slow invariants only once every period assertions") } -func appStateFn(r *rand.Rand, accs []simulation.Account, genesisTimestamp time.Time) (json.RawMessage, []simulation.Account, string) { - - if genesisFile != "" { - var genesis tmtypes.GenesisDoc - cdc := MakeCodec() - bytes, err := ioutil.ReadFile(genesisFile) - if err != nil { - panic(err) - } - if err = cdc.UnmarshalJSON(bytes, &genesis); err != nil { - panic(err) - } - var appState GenesisState - if err = cdc.UnmarshalJSON(genesis.AppState, &appState); err != nil { - panic(err) - } - var newAccs []simulation.Account - for _, acc := range appState.Accounts { - privkeySeed := make([]byte, 15) - r.Read(privkeySeed) - privKey := secp256k1.GenPrivKeySecp256k1(privkeySeed) - newAccs = append(newAccs, simulation.Account{privKey, privKey.PubKey(), acc.Address}) - } - return json.RawMessage(genesis.AppState), newAccs, genesis.ChainID +func appStateFromGenesisFileFn(r *rand.Rand, accs []simulation.Account, genesisTimestamp time.Time) (json.RawMessage, []simulation.Account, string) { + var genesis tmtypes.GenesisDoc + cdc := MakeCodec() + bytes, err := ioutil.ReadFile(genesisFile) + if err != nil { + panic(err) + } + if err = cdc.UnmarshalJSON(bytes, &genesis); err != nil { + panic(err) } + var appState GenesisState + if err = cdc.UnmarshalJSON(genesis.AppState, &appState); err != nil { + panic(err) + } + var newAccs []simulation.Account + for _, acc := range appState.Accounts { + // Pick a random private key, since we don't know the actual key + // This should be fine as it's only used for mock Tendermint validators + privkeySeed := make([]byte, 15) + r.Read(privkeySeed) + privKey := secp256k1.GenPrivKeySecp256k1(privkeySeed) + newAccs = append(newAccs, simulation.Account{privKey, privKey.PubKey(), acc.Address}) + } + return json.RawMessage(genesis.AppState), newAccs, genesis.ChainID +} + +func appStateRandomizedFn(r *rand.Rand, accs []simulation.Account, genesisTimestamp time.Time) (json.RawMessage, []simulation.Account, string) { var genesisAccounts []GenesisAccount @@ -253,6 +255,13 @@ func appStateFn(r *rand.Rand, accs []simulation.Account, genesisTimestamp time.T return appState, accs, "simulation" } +func appStateFn(r *rand.Rand, accs []simulation.Account, genesisTimestamp time.Time) (json.RawMessage, []simulation.Account, string) { + if genesisFile != "" { + return appStateFromGenesisFileFn(r, accs, genesisTimestamp) + } + return appStateRandomizedFn(r, accs, genesisTimestamp) +} + func randIntBetween(r *rand.Rand, min, max int) int { return r.Intn(max-min) + min } From 6dae58c841cc064c45f79a7a0f2b159a853ee135 Mon Sep 17 00:00:00 2001 From: Christopher Goes Date: Tue, 29 Jan 2019 22:16:46 +0100 Subject: [PATCH 5/6] Break long lines in Makefile --- Makefile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 54495112268b..be1449ca0946 100644 --- a/Makefile +++ b/Makefile @@ -153,7 +153,8 @@ test_sim_gaia_nondeterminism: test_sim_gaia_custom_genesis_fast: @echo "Running custom genesis simulation..." @echo "By default, ${HOME}/.gaiad/config/genesis.json will be used." - @go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationGenesis=${HOME}/.gaiad/config/genesis.json -SimulationEnabled=true -SimulationNumBlocks=100 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=99 -SimulationPeriod=5 -v -timeout 24h + @go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationGenesis=${HOME}/.gaiad/config/genesis.json \ + -SimulationEnabled=true -SimulationNumBlocks=100 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=99 -SimulationPeriod=5 -v -timeout 24h test_sim_gaia_fast: @echo "Running quick Gaia simulation. This may take several minutes..." @@ -181,11 +182,13 @@ SIM_BLOCK_SIZE ?= 200 SIM_COMMIT ?= true test_sim_gaia_benchmark: @echo "Running Gaia benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!" - @go test -benchmem -run=^$$ github.com/cosmos/cosmos-sdk/cmd/gaia/app -bench ^BenchmarkFullGaiaSimulation$$ -SimulationEnabled=true -SimulationNumBlocks=$(SIM_NUM_BLOCKS) -SimulationBlockSize=$(SIM_BLOCK_SIZE) -SimulationCommit=$(SIM_COMMIT) -timeout 24h + @go test -benchmem -run=^$$ github.com/cosmos/cosmos-sdk/cmd/gaia/app -bench ^BenchmarkFullGaiaSimulation$$ \ + -SimulationEnabled=true -SimulationNumBlocks=$(SIM_NUM_BLOCKS) -SimulationBlockSize=$(SIM_BLOCK_SIZE) -SimulationCommit=$(SIM_COMMIT) -timeout 24h test_sim_gaia_profile: @echo "Running Gaia benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!" - @go test -benchmem -run=^$$ github.com/cosmos/cosmos-sdk/cmd/gaia/app -bench ^BenchmarkFullGaiaSimulation$$ -SimulationEnabled=true -SimulationNumBlocks=$(SIM_NUM_BLOCKS) -SimulationBlockSize=$(SIM_BLOCK_SIZE) -SimulationCommit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out + @go test -benchmem -run=^$$ github.com/cosmos/cosmos-sdk/cmd/gaia/app -bench ^BenchmarkFullGaiaSimulation$$ \ + -SimulationEnabled=true -SimulationNumBlocks=$(SIM_NUM_BLOCKS) -SimulationBlockSize=$(SIM_BLOCK_SIZE) -SimulationCommit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out test_cover: @export VERSION=$(VERSION); bash tests/test_cover.sh From 4448eb24009cfceb7818366fe8d739595795b6d5 Mon Sep 17 00:00:00 2001 From: Christopher Goes Date: Tue, 29 Jan 2019 23:42:12 +0100 Subject: [PATCH 6/6] Address @rigelrozanski comments --- cmd/gaia/app/sim_test.go | 9 +++------ x/mock/simulation/simulate.go | 4 ++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/cmd/gaia/app/sim_test.go b/cmd/gaia/app/sim_test.go index f75f412cc5ed..bbb5a2b4e6c4 100644 --- a/cmd/gaia/app/sim_test.go +++ b/cmd/gaia/app/sim_test.go @@ -66,17 +66,14 @@ func appStateFromGenesisFileFn(r *rand.Rand, accs []simulation.Account, genesisT if err != nil { panic(err) } - if err = cdc.UnmarshalJSON(bytes, &genesis); err != nil { - panic(err) - } + cdc.MustUnmarshalJSON(bytes, &genesis) var appState GenesisState - if err = cdc.UnmarshalJSON(genesis.AppState, &appState); err != nil { - panic(err) - } + cdc.MustUnmarshalJSON(genesis.AppState, &appState) var newAccs []simulation.Account for _, acc := range appState.Accounts { // Pick a random private key, since we don't know the actual key // This should be fine as it's only used for mock Tendermint validators + // and these keys are never actually used to sign by mock Tendermint. privkeySeed := make([]byte, 15) r.Read(privkeySeed) privKey := secp256k1.GenPrivKeySecp256k1(privkeySeed) diff --git a/x/mock/simulation/simulate.go b/x/mock/simulation/simulate.go index b8dc6820c084..723955f6b367 100644 --- a/x/mock/simulation/simulate.go +++ b/x/mock/simulation/simulate.go @@ -18,8 +18,8 @@ import ( sdk "github.com/cosmos/cosmos-sdk/types" ) -// AppStateFn returns the app state json bytes -type AppStateFn func(r *rand.Rand, accs []Account, genesisTimestamp time.Time) (json.RawMessage, []Account, string) +// AppStateFn returns the app state json bytes, the genesis accounts, and the chain identifier +type AppStateFn func(r *rand.Rand, accs []Account, genesisTimestamp time.Time) (appState json.RawMessage, accounts []Account, chainId string) // Simulate tests application by sending random messages. func Simulate(t *testing.T, app *baseapp.BaseApp,