From 3bcb1980899c581358952b980dd97ea255adcd20 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Tue, 27 Sep 2022 14:05:01 +0800 Subject: [PATCH 01/15] Problem: there's no compact historical state storage Closes: #704 Solution: - Integration version store and streaming service. multiple backends db size benchmark support mdbx backend through tm-db store latest version support GetLatestVersion query multistore test versiondb streamer fix lint fix state listener temp fix state listening optimize common case fix lint try to fix mdbx build update cosmos-sdk fix clone append add test case fix subkey problem in history index revert chunking, hard to work with variable length key support iterator check future height fix lint new state listener fix latest state in query fix integration test fix prune node test update dependency add utility to read from file streamer Update versiondb/multistore.go Signed-off-by: yihuang add unit test create common backend test cases update dependency update with new file streamer format Problem: python3.10 is not used in integration tests Solution: - start using python3.10, prepare to later PRs which need the new features - update nixpkgs nesserary for the nix stuff to work. python-roaring64 remove debug log add test cases, improve coverage --- Makefile | 4 + app/app.go | 45 ++- default.nix | 3 +- go.mod | 19 +- gomod2nix.toml | 51 ++- integration_tests/configs/default.jsonnet | 2 +- integration_tests/configs/pruned-node.jsonnet | 2 + .../configs/state_benchmark.jsonnet | 36 ++ integration_tests/conftest.py | 22 ++ .../contracts/contracts/BenchmarkStorage.sol | 15 + integration_tests/pyproject.toml | 5 +- integration_tests/test_benchmark_storage.py | 53 +++ integration_tests/test_streamer.py | 57 +-- integration_tests/utils.py | 1 + integration_tests/versiondb.py | 86 +++++ nix/testenv.nix | 10 +- scripts/cronos-devnet.yaml | 2 +- versiondb/backend_test_utils.go | 276 ++++++++++++++ versiondb/dbutils.go | 84 +++++ versiondb/multistore.go | 135 +++++++ versiondb/store.go | 94 +++++ versiondb/streaming_service.go | 84 +++++ versiondb/sync.go | 31 ++ versiondb/sync_test.go | 340 ++++++++++++++++++ versiondb/tmdb/history.go | 67 ++++ versiondb/tmdb/iterator.go | 183 ++++++++++ versiondb/tmdb/store.go | 266 ++++++++++++++ versiondb/tmdb/store_test.go | 14 + versiondb/types.go | 21 ++ 29 files changed, 1944 insertions(+), 64 deletions(-) create mode 100644 integration_tests/configs/state_benchmark.jsonnet create mode 100644 integration_tests/contracts/contracts/BenchmarkStorage.sol create mode 100644 integration_tests/test_benchmark_storage.py create mode 100644 integration_tests/versiondb.py create mode 100644 versiondb/backend_test_utils.go create mode 100644 versiondb/dbutils.go create mode 100644 versiondb/multistore.go create mode 100644 versiondb/store.go create mode 100644 versiondb/streaming_service.go create mode 100644 versiondb/sync.go create mode 100644 versiondb/sync_test.go create mode 100644 versiondb/tmdb/history.go create mode 100644 versiondb/tmdb/iterator.go create mode 100644 versiondb/tmdb/store.go create mode 100644 versiondb/tmdb/store_test.go create mode 100644 versiondb/types.go diff --git a/Makefile b/Makefile index 94ea62b5e1..e92f965247 100644 --- a/Makefile +++ b/Makefile @@ -74,6 +74,10 @@ ifeq (boltdb,$(findstring boltdb,$(COSMOS_BUILD_OPTIONS))) BUILD_TAGS += boltdb endif +ifeq (mdbx,$(findstring mdbx,$(COSMOS_BUILD_OPTIONS))) + BUILD_TAGS += mdbx +endif + ifeq (,$(findstring nostrip,$(COSMOS_BUILD_OPTIONS))) ldflags += -w -s endif diff --git a/app/app.go b/app/app.go index 7cd77b35e4..8d423a3a44 100644 --- a/app/app.go +++ b/app/app.go @@ -5,12 +5,15 @@ import ( "net/http" "os" "path/filepath" + "strings" "sync" + "github.com/crypto-org-chain/cronos/x/cronos" "github.com/crypto-org-chain/cronos/x/cronos/middleware" "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/codec/types" + "github.com/cosmos/cosmos-sdk/server" "github.com/gorilla/mux" "github.com/rakyll/statik/fs" "github.com/spf13/cast" @@ -122,7 +125,8 @@ import ( // this line is used by starport scaffolding # stargate/app/moduleImport cronosappclient "github.com/crypto-org-chain/cronos/client" - "github.com/crypto-org-chain/cronos/x/cronos" + "github.com/crypto-org-chain/cronos/versiondb" + "github.com/crypto-org-chain/cronos/versiondb/tmdb" cronosclient "github.com/crypto-org-chain/cronos/x/cronos/client" cronoskeeper "github.com/crypto-org-chain/cronos/x/cronos/keeper" evmhandlers "github.com/crypto-org-chain/cronos/x/cronos/keeper/evmhandlers" @@ -350,7 +354,8 @@ func New( // configure state listening capabilities using AppOptions // we are doing nothing with the returned streamingServices and waitGroup in this case // Only support file streamer right now. - if cast.ToString(appOpts.Get(cronosappclient.FlagStreamers)) == "file" { + streamers := cast.ToString(appOpts.Get(cronosappclient.FlagStreamers)) + if strings.Contains(streamers, "file") { streamingDir := filepath.Join(cast.ToString(appOpts.Get(flags.FlagHome)), "data", FileStreamerDirectory) if err := os.MkdirAll(streamingDir, os.ModePerm); err != nil { panic(err) @@ -361,7 +366,7 @@ func New( for _, storeKey := range keys { exposeStoreKeys = append(exposeStoreKeys, storeKey) } - service, err := file.NewStreamingService(streamingDir, "", exposeStoreKeys, appCodec) + service, err := file.NewStreamingService(streamingDir, "", exposeStoreKeys, appCodec, false) if err != nil { panic(err) } @@ -373,6 +378,40 @@ func New( } } + if strings.Contains(streamers, "versiondb") { + rootDir := cast.ToString(appOpts.Get(flags.FlagHome)) + dataDir := filepath.Join(rootDir, "data", "versiondb") + if err := os.MkdirAll(dataDir, os.ModePerm); err != nil { + panic(err) + } + backendType := server.GetAppDBBackend(appOpts) + plainDB, err := dbm.NewDB("plain", backendType, dataDir) + if err != nil { + panic(err) + } + historyDB, err := dbm.NewDB("history", backendType, dataDir) + if err != nil { + panic(err) + } + changesetDB, err := dbm.NewDB("changeset", backendType, dataDir) + if err != nil { + panic(err) + } + versionDB := tmdb.NewStore(plainDB, historyDB, changesetDB) + + // default to exposing all + exposeStoreKeys := make([]storetypes.StoreKey, 0, len(keys)) + for _, storeKey := range keys { + exposeStoreKeys = append(exposeStoreKeys, storeKey) + } + service := versiondb.NewStreamingService(versionDB, exposeStoreKeys) + bApp.SetStreamingService(service) + qms := versiondb.NewMultiStore(versionDB, exposeStoreKeys) + qms.MountTransientStores(tkeys) + qms.MountMemoryStores(memKeys) + bApp.SetQueryMultiStore(qms) + } + app := &App{ BaseApp: bApp, cdc: cdc, diff --git a/default.nix b/default.nix index fcbab2154a..846e83eea0 100644 --- a/default.nix +++ b/default.nix @@ -8,7 +8,7 @@ let version = "v0.9.0"; pname = "cronosd"; - tags = [ "ledger" "netgo" network ] + tags = [ "ledger" "netgo" network "mdbx" ] ++ lib.lists.optionals (rocksdb != null) [ "rocksdb" "rocksdb_build" ]; ldflags = lib.concatStringsSep "\n" ([ "-X github.com/cosmos/cosmos-sdk/version.Name=cronos" @@ -27,6 +27,7 @@ buildGoApplication rec { "!/app/" "!/cmd/" "!/client/" + "!/versiondb/" "!go.mod" "!go.sum" "!gomod2nix.toml" diff --git a/go.mod b/go.mod index 17476fe929..40fa2911d7 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.18 require ( cosmossdk.io/math v1.0.0-beta.3 + github.com/RoaringBitmap/roaring v1.2.1 github.com/armon/go-metrics v0.4.1 github.com/cosmos/cosmos-sdk v0.46.3 github.com/cosmos/ibc-go/v5 v5.0.0 @@ -45,10 +46,12 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/bgentry/speakeasy v0.1.0 // indirect + github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/btcsuite/btcd v0.22.1 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect + github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect @@ -69,9 +72,8 @@ require ( github.com/deckarep/golang-set v1.8.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect - github.com/dgraph-io/badger/v2 v2.2007.4 // indirect + github.com/dgraph-io/badger/v3 v3.2103.2 // indirect github.com/dgraph-io/ristretto v0.1.0 // indirect - github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect github.com/dop251/goja v0.0.0-20220405120441-9037c2b61cbf // indirect github.com/dustin/go-humanize v1.0.0 // indirect @@ -91,7 +93,8 @@ require ( github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/btree v1.0.1 // indirect + github.com/google/btree v1.1.2 // indirect + github.com/google/flatbuffers v2.0.0+incompatible // indirect github.com/google/go-cmp v0.5.8 // indirect github.com/google/orderedcode v0.0.1 // indirect github.com/google/uuid v1.3.0 // indirect @@ -135,7 +138,9 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mschoch/smat v0.2.0 // indirect github.com/mtibben/percent v0.2.1 // indirect + github.com/natefinch/atomic v1.0.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.5 // indirect @@ -165,6 +170,7 @@ require ( github.com/tendermint/go-amino v0.16.0 // indirect github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect + github.com/torquem-ch/mdbx-go v0.26.0 // indirect github.com/tyler-smith/go-bip39 v1.1.0 // indirect github.com/ulikunitz/xz v0.5.8 // indirect github.com/zondax/hid v0.9.1-0.20220302062450-5552068d2266 // indirect @@ -189,7 +195,7 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/cosmos/cosmos-sdk v0.46.2 + github.com/cosmos/cosmos-sdk => github.com/yihuang/cosmos-sdk v0.43.0-beta1.0.20221014023203-2c6b9d06b12d github.com/ethereum/go-ethereum => github.com/ethereum/go-ethereum v1.10.19 // Fix upstream GHSA-h395-qcrw-5vmq vulnerability. @@ -199,6 +205,11 @@ replace ( github.com/peggyjv/gravity-bridge/module/v2 => github.com/crypto-org-chain/gravity-bridge/module/v2 v2.0.1-0.20221027085649-2107c6bd6bc4 + // https://github.com/tendermint/tm-db/pull/297 + github.com/tendermint/tm-db => github.com/yihuang/tm-db v0.0.0-20221006023748-f6214ae9454d + + github.com/torquem-ch/mdbx-go => github.com/yihuang/mdbx-go v0.0.0-20221010042614-b72b4f091d88 + // TODO: remove after fixed https://github.com/cosmos/cosmos-sdk/issues/11364 github.com/zondax/hid => github.com/zondax/hid v0.9.0 ) diff --git a/gomod2nix.toml b/gomod2nix.toml index a05555ba67..b98b5ed14b 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -31,6 +31,9 @@ schema = 3 [mod."github.com/ChainSafe/go-schnorrkel"] version = "v0.0.0-20200405005733-88cbf1b4c40d" hash = "sha256-i8RXZemJGlSjBT35oPm0SawFiBoIU5Pkq5xp4n/rzCY=" + [mod."github.com/RoaringBitmap/roaring"] + version = "v1.2.1" + hash = "sha256-0/R956wrCW71eOE36CbxGJJRuQjKwvvIQ/D8QTn2A6w=" [mod."github.com/StackExchange/wmi"] version = "v1.2.1" hash = "sha256-1BoEeWAWyebH+1mMuyPhWZut8nWHb6r73MgcqlGuUEY=" @@ -58,6 +61,9 @@ schema = 3 [mod."github.com/bgentry/speakeasy"] version = "v0.1.0" hash = "sha256-Gt1vj6CFovLnO6wX5u2O4UfecY9V2J9WGw1ez4HMrgk=" + [mod."github.com/bits-and-blooms/bitset"] + version = "v1.2.0" + hash = "sha256-IxNmtELycM+XVzg4qBv04hAJUT3nSWuyP9R+8zc9LmU=" [mod."github.com/btcsuite/btcd"] version = "v0.22.1" hash = "sha256-hBU+roIELcmbW2Gz7eGZzL9qNA1bakq5wNxqCgs4TKc=" @@ -70,6 +76,9 @@ schema = 3 [mod."github.com/btcsuite/btcutil"] version = "v1.0.3-0.20201208143702-a53e38424cce" hash = "sha256-4kasJReFcj25JRHx9dJMct3yDkHqVoHGUx5cu45Msfo=" + [mod."github.com/c2h5oh/datasize"] + version = "v0.0.0-20220606134207-859f65c6625b" + hash = "sha256-1uH+D3w0Y/B3poXm545XGrT4S4c+msTbj7gKgu9pbPM=" [mod."github.com/cenkalti/backoff/v4"] version = "v4.1.3" hash = "sha256-u6MEDopHoTWAZoVvvXOKnAg++xre53YgQx0gmf6t2KU=" @@ -98,12 +107,15 @@ schema = 3 version = "v1.0.0-alpha7" hash = "sha256-2wCH+toTF2A6MfFjOa13muEH5oBCcxAhZEqirNOrBA0=" [mod."github.com/cosmos/cosmos-sdk"] - version = "v0.46.2" - hash = "sha256-Lgn4+Vd5PUUkfHc+lTdK2G6/nymZekFVTe1FxWRqh2w=" - replaced = "github.com/cosmos/cosmos-sdk" + version = "v0.43.0-beta1.0.20221014023203-2c6b9d06b12d" + hash = "sha256-t/QEOJgATr82KFXes+AzNlNL6w2tGHfiwZAE4PdO5GE=" + replaced = "github.com/yihuang/cosmos-sdk" [mod."github.com/cosmos/go-bip39"] version = "v1.0.0" hash = "sha256-Qm2aC2vaS8tjtMUbHmlBSagOSqbduEEDwc51qvQaBmA=" + [mod."github.com/cosmos/gogoproto"] + version = "v1.4.2" + hash = "sha256-hOY+mhPDYWcSYSdth2AW7IONdgicqQir0z/1XrXt9NY=" [mod."github.com/cosmos/gorocksdb"] version = "v1.2.0" hash = "sha256-209TcVuXc5s/TcOvNlaQ1HEJAUDTEK3nxPhs+d8TEcY=" @@ -137,15 +149,12 @@ schema = 3 [mod."github.com/desertbit/timer"] version = "v0.0.0-20180107155436-c41aec40b27f" hash = "sha256-abLOtEcomAqCWLphd2X6WkD/ED764w6sa6unox4BXss=" - [mod."github.com/dgraph-io/badger/v2"] - version = "v2.2007.4" - hash = "sha256-+KwqZJZpViv8S3TqUVvPXrFoMgWFyS3NoLsi4RR5fGk=" + [mod."github.com/dgraph-io/badger/v3"] + version = "v3.2103.2" + hash = "sha256-F6pvsaSKwXOl9RfnUQFqAl6xpCVu9+rthQgOxhKVk1g=" [mod."github.com/dgraph-io/ristretto"] version = "v0.1.0" hash = "sha256-01jneg1+1x8tTfUTBZ+6mHkQaqXVnPYxLJyJhJQcvt4=" - [mod."github.com/dgryski/go-farm"] - version = "v0.0.0-20200201041132-a6ae2369ad13" - hash = "sha256-aOMlPwFY36bLiiIx4HonbCYRAhagk5N6HAWN7Ygif+E=" [mod."github.com/dlclark/regexp2"] version = "v1.4.1-0.20201116162257-a2a8dda75c91" hash = "sha256-VNNMZIc7NkDg3DVLnqeJNM/KZqkkaZu2/HTLBL8X2xE=" @@ -164,6 +173,7 @@ schema = 3 [mod."github.com/ethereum/go-ethereum"] version = "v1.10.19" hash = "sha256-7FPnTGcCb8Xd1QVR+6PmGTaHdTY1mm/8osFTW1JLuG8=" + replaced = "github.com/ethereum/go-ethereum" [mod."github.com/evmos/ethermint"] version = "v0.6.1-0.20221003153722-491c3da7ebd7" hash = "sha256-vnfjk57gYa+F8nn0LByX/B1LV8PY2Jvm8vXV6be4ufc=" @@ -217,8 +227,11 @@ schema = 3 version = "v0.0.4" hash = "sha256-Umx+5xHAQCN/Gi4HbtMhnDCSPFAXSsjVbXd8n5LhjAA=" [mod."github.com/google/btree"] - version = "v1.0.1" - hash = "sha256-1PIeFGgUL4BK/StL/D12pg9bEQ5HfMT/fMLdus4pZTs=" + version = "v1.1.2" + hash = "sha256-K7V2obq3pLM71Mg0vhhHtZ+gtaubwXPQx3xcIyZDCjM=" + [mod."github.com/google/flatbuffers"] + version = "v2.0.0+incompatible" + hash = "sha256-4Db9FdOL60Da4H1+K4Qv02w4omxdsh3uzpmY1vtqHeA=" [mod."github.com/google/go-cmp"] version = "v0.5.8" hash = "sha256-8zkIo+Sr1NXMnj3PNmvjX2sZKnAKWXOFvmnX7D9bwxQ=" @@ -354,9 +367,15 @@ schema = 3 [mod."github.com/mitchellh/mapstructure"] version = "v1.5.0" hash = "sha256-ztVhGQXs67MF8UadVvG72G3ly0ypQW0IRDdOOkjYwoE=" + [mod."github.com/mschoch/smat"] + version = "v0.2.0" + hash = "sha256-DZvUJXjIcta3U+zxzgU3wpoGn/V4lpBY7Xme8aQUi+E=" [mod."github.com/mtibben/percent"] version = "v0.2.1" hash = "sha256-Zj1lpCP6mKQ0UUTMs2By4LC414ou+iJzKkK+eBHfEcc=" + [mod."github.com/natefinch/atomic"] + version = "v1.0.1" + hash = "sha256-fbOVHCwRNI8PFjC4o0YXpKZO0JU2aWTfH5c7WXXKMHg=" [mod."github.com/olekukonko/tablewriter"] version = "v0.0.5" hash = "sha256-/5i70IkH/qSW5KjGzv8aQNKh9tHoz98tqtL0K2DMFn4=" @@ -461,14 +480,19 @@ schema = 3 version = "v0.34.22" hash = "sha256-4p4cpyCWjBbNQUpYN2gDJvnyj+Pov9hw5uRjHrrO++Y=" [mod."github.com/tendermint/tm-db"] - version = "v0.6.7" - hash = "sha256-hl/3RrBrpkk2zA6dmrNlIYKs1/GfqegSscDSkA5Pjlo=" + version = "v0.0.0-20221006023748-f6214ae9454d" + hash = "sha256-mtTVR3f3A9CmcyJBXTeurQuHGiVA5hIzqlxskz1M1qk=" + replaced = "github.com/yihuang/tm-db" [mod."github.com/tklauser/go-sysconf"] version = "v0.3.10" hash = "sha256-Zf2NsgM9+HeM949vCce4HQtSbfUiFpeiQ716yKcFyx4=" [mod."github.com/tklauser/numcpus"] version = "v0.4.0" hash = "sha256-ndE82nOb3agubhEV7aRzEqqTlN4DPbKFHEm2+XZLn8k=" + [mod."github.com/torquem-ch/mdbx-go"] + version = "v0.0.0-20221010042614-b72b4f091d88" + hash = "sha256-HWsrhzSGoYgEUUziQPHEtQGwBQ/upg3/f1fY2NGiC0g=" + replaced = "github.com/yihuang/mdbx-go" [mod."github.com/tyler-smith/go-bip39"] version = "v1.1.0" hash = "sha256-3YhWBtSwRLGwm7vNwqumphZG3uLBW1vwT9QkQ8JuSjU=" @@ -478,6 +502,7 @@ schema = 3 [mod."github.com/zondax/hid"] version = "v0.9.0" hash = "sha256-PvXtxXo/3C+DS9ZeGBlr4zXbIpaYNtMqLzxYhusFXNY=" + replaced = "github.com/zondax/hid" [mod."go.etcd.io/bbolt"] version = "v1.3.6" hash = "sha256-DenVAmyN22xUiivk6fdJp4C9ZnUJXCMDUf8E0goRRV4=" diff --git a/integration_tests/configs/default.jsonnet b/integration_tests/configs/default.jsonnet index 12ca83f47d..23b183f7d5 100644 --- a/integration_tests/configs/default.jsonnet +++ b/integration_tests/configs/default.jsonnet @@ -2,7 +2,7 @@ dotenv: '../../scripts/.env', 'cronos_777-1': { cmd: 'cronosd', - 'start-flags': '--trace --streamers file', + 'start-flags': '--trace --streamers versiondb,file', config: { mempool: { version: 'v1', diff --git a/integration_tests/configs/pruned-node.jsonnet b/integration_tests/configs/pruned-node.jsonnet index 2a3edd147f..6e87b2288b 100644 --- a/integration_tests/configs/pruned-node.jsonnet +++ b/integration_tests/configs/pruned-node.jsonnet @@ -2,6 +2,8 @@ local config = import 'default.jsonnet'; config { 'cronos_777-1'+: { + // don't enable versiondb, since it don't do pruning right now + 'start-flags': '--trace --streamers file', 'app-config'+: { pruning: 'everything', 'state-sync'+: { diff --git a/integration_tests/configs/state_benchmark.jsonnet b/integration_tests/configs/state_benchmark.jsonnet new file mode 100644 index 0000000000..ba8dbf1741 --- /dev/null +++ b/integration_tests/configs/state_benchmark.jsonnet @@ -0,0 +1,36 @@ +local config = import 'default.jsonnet'; + +config { + 'cronos_777-1'+: { + 'start-flags': '--trace --streamers file,versiondb', + 'app-config'+: { + 'app-db-backend': 'rocksdb', + 'state-sync'+: { + 'snapshot-interval': 0, + }, + }, + validators: [ + super.validators[0], + super.validators[1] { + 'app-config'+: { + pruning: 'everything', + }, + }, + ] + super.validators[2:], + genesis+: { + consensus_params+: { + block+: { + max_gas: '163000000', + }, + }, + app_state+: { + feemarket+: { + params+: { + no_base_fee: true, + min_gas_multiplier: '0', + }, + }, + }, + }, + }, +} diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py index 980d4a9627..a496fe7a12 100644 --- a/integration_tests/conftest.py +++ b/integration_tests/conftest.py @@ -13,6 +13,28 @@ def pytest_configure(config): config.addinivalue_line("markers", "slow: marks tests as slow") config.addinivalue_line("markers", "gravity: gravity bridge test cases") + config.addinivalue_line( + "markers", "benchmark: benchmarks, only run if '--run-benchmark' is passed" + ) + + +def pytest_addoption(parser): + parser.addoption( + "--run-benchmark", + action="store_true", + default=False, + help="include benchmark cases", + ) + + +def pytest_collection_modifyitems(config, items): + if config.getoption("--run-benchmark"): + # run benchmarks + return + skip = pytest.mark.skip(reason="need --run-benchmark option to run") + for item in items: + if "benchmark" in item.keywords: + item.add_marker(skip) @pytest.fixture(scope="session") diff --git a/integration_tests/contracts/contracts/BenchmarkStorage.sol b/integration_tests/contracts/contracts/BenchmarkStorage.sol new file mode 100644 index 0000000000..69dcfde451 --- /dev/null +++ b/integration_tests/contracts/contracts/BenchmarkStorage.sol @@ -0,0 +1,15 @@ +pragma solidity 0.8.10; + +contract BenchmarkStorage { + uint seed; + mapping(uint => uint) state; + function random(uint i) private view returns (uint) { + return uint(keccak256(abi.encodePacked(i, seed))); + } + function batch_set(uint _seed, uint n, uint range) public { + seed = _seed; + for (uint i=0; i< n; i++) { + state[random(i) % range] = random(i+i); + } + } +} diff --git a/integration_tests/pyproject.toml b/integration_tests/pyproject.toml index 65ca14c207..54fb3b3f60 100644 --- a/integration_tests/pyproject.toml +++ b/integration_tests/pyproject.toml @@ -5,7 +5,7 @@ description = "" authors = ["chain-dev "] [tool.poetry.dependencies] -python = "^3.8" +python = "^3.10" pytest = "^7.0.1" pytest-github-actions-annotate-failures = "^0.1.1" flake8 = "^4.0.1" @@ -28,6 +28,9 @@ jsonnet = "^0.18.0" eth-account = "^0.7.0" cprotobuf = "^0.1.11" pathspec = "^0.10.1" +rocksdb = { git = "https://github.com/HathorNetwork/python-rocksdb.git", branch = "master" } +click = "^8.1.2" +roaring64 = { git = "https://github.com/yihuang/python-roaring64.git", branch = "main" } [tool.poetry.dev-dependencies] diff --git a/integration_tests/test_benchmark_storage.py b/integration_tests/test_benchmark_storage.py new file mode 100644 index 0000000000..41668a94ac --- /dev/null +++ b/integration_tests/test_benchmark_storage.py @@ -0,0 +1,53 @@ +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import pytest +from web3 import Web3 + +from .network import setup_custom_cronos +from .utils import ( + ACCOUNTS, + CONTRACTS, + deploy_contract, + send_transaction, + w3_wait_for_block, +) + + +@pytest.fixture(scope="module") +def custom_cronos(tmp_path_factory): + path = tmp_path_factory.mktemp("benchmark") + yield from setup_custom_cronos( + path, 26200, Path(__file__).parent / "configs/state_benchmark.jsonnet" + ) + + +@pytest.mark.benchmark +def test_benchmark_storage(custom_cronos): + w3: Web3 = custom_cronos.w3 + w3_wait_for_block(w3, 1) + contract = deploy_contract(w3, CONTRACTS["BenchmarkStorage"]) + + n = 3000 + gas = 81500000 + iterations = 200 + parity = 100 + + def task(acct, acct_i): + for i in range(iterations): + seed = i * 10 + acct_i + tx = contract.functions.batch_set(seed, n, n * parity).buildTransaction( + {"from": acct.address, "gas": gas} + ) + print(send_transaction(w3, tx, acct.key)) + + accounts = [ + ACCOUNTS["validator"], + ACCOUNTS["community"], + ACCOUNTS["signer1"], + ACCOUNTS["signer2"], + ] + with ThreadPoolExecutor(len(accounts)) as exec: + tasks = [exec.submit(task, acct, i) for i, acct in enumerate(accounts)] + for t in tasks: + t.result() diff --git a/integration_tests/test_streamer.py b/integration_tests/test_streamer.py index 249aa346f2..f0128d1e29 100644 --- a/integration_tests/test_streamer.py +++ b/integration_tests/test_streamer.py @@ -13,40 +13,20 @@ class StoreKVPairs(ProtoEntity): value = Field("bytes", 4) -def decode_stream_file(data, body_cls=StoreKVPairs, header_cls=None, footer_cls=None): +def decode_stream_file(data, entry_cls=StoreKVPairs): """ - header, body*, footer + StoreKVPairs, StoreKVPairs, ... """ - header = footer = None - body = [] + items = [] offset = 0 - size, n = decode_primitive(data, "uint64") - offset += n - - # header - if header_cls is not None: - header = header_cls() - header.ParseFromString(data[offset : offset + size]) - offset += size - - while True: + while offset < len(data): size, n = decode_primitive(data[offset:], "uint64") offset += n - if offset + size == len(data): - # footer - if footer_cls is not None: - footer = footer_cls() - footer.ParseFromString(data[offset : offset + size]) - offset += size - break - else: - # body - if body_cls is not None: - item = body_cls() - item.ParseFromString(data[offset : offset + size]) - body.append(item) - offset += size - return header, body, footer + item = entry_cls() + item.ParseFromString(data[offset : offset + size]) + items.append(item) + offset += size + return items def test_streamers(cronos): @@ -55,23 +35,22 @@ def test_streamers(cronos): - try to parse the state change sets """ # inspect the first state change of the first tx in genesis - path = cronos.node_home(0) / "data/file_streamer/block-0-tx-0" - _, body, _ = decode_stream_file(open(path, "rb").read()) + # the InitChainer is committed together with the first block. + path = cronos.node_home(0) / "data/file_streamer/block-1-data" + items = decode_stream_file(open(path, "rb").read()) # creation of the validator account - assert body[0].store_key == "acc" - # the order in gen_txs is undeterministic, could be either one. - assert body[0].key in ( - b"\x01" + HexBytes(ADDRS["validator"]), - b"\x01" + HexBytes(ADDRS["validator2"]), - ) + assert items[0].store_key == "acc" + # the writes are sorted by key, find the minimal address + min_addr = min(ADDRS.values()) + assert items[0].key == b"\x01" + HexBytes(min_addr) if __name__ == "__main__": import binascii import sys - _, body, _ = decode_stream_file(open(sys.argv[1], "rb").read()) - for item in body: + items = decode_stream_file(open(sys.argv[1], "rb").read()) + for item in items: print( item.store_key, item.delete, diff --git a/integration_tests/utils.py b/integration_tests/utils.py index 462d472bfa..a714ff3b89 100644 --- a/integration_tests/utils.py +++ b/integration_tests/utils.py @@ -47,6 +47,7 @@ "TestBlackListERC20": "TestBlackListERC20.sol", "CroBridge": "CroBridge.sol", "CronosGravityCancellation": "CronosGravityCancellation.sol", + "BenchmarkStorage": "BenchmarkStorage.sol", } diff --git a/integration_tests/versiondb.py b/integration_tests/versiondb.py new file mode 100644 index 0000000000..40c0c672e0 --- /dev/null +++ b/integration_tests/versiondb.py @@ -0,0 +1,86 @@ +""" +cli utilities for versiondb +""" +import binascii +from pathlib import Path + +import click +import rocksdb +from cprotobuf import decode_primitive +from roaring64 import BitMap64 + + +def rocksdb_stats(path): + db = rocksdb.DB(str(path), rocksdb.Options()) + for field in ["rocksdb.stats", "rocksdb.sstables"]: + print(f"############# {field}") + print(db.get_property(field.encode()).decode()) + + # space amplification + it = db.iteritems() + it.seek_to_first() + count = 0 + size = 0 + for k, v in it: + count += 1 + size += len(k) + len(v) + # directory size + fsize = sum(f.stat().st_size for f in path.glob("**/*") if f.is_file()) + print( + f"space_amplification: {fsize / size:.2f}, kv pairs: {count}, " + f"data size: {size}, file size: {fsize}" + ) + + +@click.group() +def cli(): + pass + + +@cli.command() +@click.option("--dbpath", help="path of plain db") +def latest_version(dbpath): + db = rocksdb.DB(dbpath, rocksdb.Options()) + bz = db.get(b"s/latest") + # gogoproto std int64, the first byte is field tag + print(decode_primitive(bz[1:], "int64")[0]) + + +@cli.command() +@click.option("--dbpath", help="path of version db") +@click.option("--version", help="version of the value, optional") +@click.argument("store-key") +@click.argument("hex-key") +def get(dbpath, version, store_key, hex_key): + """ + get a value at version + """ + key = f"s/k:{store_key}/".decode() + binascii.unhexlify(hex_key) + plain_db = rocksdb.DB(dbpath + "plain.db", rocksdb.Options()) + if version is None: + v = plain_db.get(key) + else: + version = int(version) + print(binascii.hexlify(v)) + + history_db = rocksdb.DB(dbpath + "history.db", rocksdb.Options()) + bz = history_db.get(key) + bm = BitMap64.deserialize(bz) + + # seek in bitmap + bm.Rank(version) + + +@cli.command() +def sync(path): + pass + + +@cli.command() +@click.option("--dbpath", help="path of rocksdb") +def rocksdbstats(dbpath): + rocksdb_stats(Path(dbpath)) + + +if __name__ == "__main__": + cli() diff --git a/nix/testenv.nix b/nix/testenv.nix index e951eccb7f..0f84ddc388 100644 --- a/nix/testenv.nix +++ b/nix/testenv.nix @@ -1,4 +1,4 @@ -{ poetry2nix, lib, python310 }: +{ poetry2nix, lib, python310, rocksdb }: poetry2nix.mkPoetryEnv { projectDir = ../integration_tests; python = python310; @@ -14,6 +14,9 @@ poetry2nix.mkPoetryEnv { pytest-github-actions-annotate-failures = [ "setuptools" ]; flake8-black = [ "setuptools" ]; multiaddr = [ "setuptools" ]; + rocksdb = [ "setuptools" "cython" "pkgconfig" ]; + pyroaring = [ "setuptools" ]; + roaring64 = [ "poetry" ]; }; in lib.mapAttrs @@ -34,6 +37,11 @@ poetry2nix.mkPoetryEnv { substituteInPlace setup.py --replace "setup()" "setup(version=\"1.3\")" ''; }; + rocksdb = super.rocksdb.overridePythonAttrs ( + old: { + buildInputs = (old.buildInputs or [ ]) ++ [ rocksdb ]; + } + ); }) ]); } diff --git a/scripts/cronos-devnet.yaml b/scripts/cronos-devnet.yaml index 0b9b865e1a..ec2f3c9fe0 100644 --- a/scripts/cronos-devnet.yaml +++ b/scripts/cronos-devnet.yaml @@ -1,7 +1,7 @@ dotenv: .env cronos_777-1: cmd: cronosd - start-flags: "--trace" + start-flags: "--trace --streamers versiondb,file" app-config: minimum-gas-prices: 0basetcro index-events: diff --git a/versiondb/backend_test_utils.go b/versiondb/backend_test_utils.go new file mode 100644 index 0000000000..a5cff0cb00 --- /dev/null +++ b/versiondb/backend_test_utils.go @@ -0,0 +1,276 @@ +package versiondb + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + dbm "github.com/tendermint/tm-db" + + "github.com/cosmos/cosmos-sdk/store/types" +) + +var ( + key1 = []byte("key1") + key2 = []byte("key2") + value1 = []byte("value1") + value2 = []byte("value2") + value3 = []byte("value3") + key1_subkey = []byte("key1/subkey") +) + +func SetupTestDB(t *testing.T, store VersionStore) { + changeSets := [][]types.StoreKVPair{ + { + {StoreKey: "evm", Key: []byte("delete-in-block2"), Value: []byte("1")}, + {StoreKey: "evm", Key: []byte("re-add-in-block3"), Value: []byte("1")}, + {StoreKey: "evm", Key: []byte("z-genesis-only"), Value: []byte("2")}, + {StoreKey: "evm", Key: []byte("modify-in-block2"), Value: []byte("1")}, + {StoreKey: "staking", Key: []byte("key1"), Value: []byte("value1")}, + {StoreKey: "staking", Key: []byte("key1/subkey"), Value: []byte("value1")}, + }, + { + {StoreKey: "evm", Key: []byte("re-add-in-block3"), Delete: true}, + {StoreKey: "evm", Key: []byte("add-in-block1"), Value: []byte("1")}, + {StoreKey: "staking", Key: []byte("key1"), Delete: true}, + }, + { + {StoreKey: "evm", Key: []byte("add-in-block2"), Value: []byte("1")}, + {StoreKey: "evm", Key: []byte("delete-in-block2"), Delete: true}, + {StoreKey: "evm", Key: []byte("modify-in-block2"), Value: []byte("2")}, + {StoreKey: "evm", Key: []byte("key2"), Delete: true}, + {StoreKey: "staking", Key: []byte("key1"), Value: []byte("value2")}, + }, + { + {StoreKey: "evm", Key: []byte("re-add-in-block3"), Value: []byte("2")}, + }, + { + {StoreKey: "evm", Key: []byte("re-add-in-block3"), Delete: true}, + }, + } + for i, changeSet := range changeSets { + require.NoError(t, store.PutAtVersion(int64(i), changeSet)) + } +} + +func Run(t *testing.T, storeCreator func() VersionStore) { + testBasics(t, storeCreator()) + testIterator(t, storeCreator()) + testHeightInFuture(t, storeCreator()) + + // test delete in genesis + store := storeCreator() + err := store.PutAtVersion(0, []types.StoreKVPair{ + {StoreKey: "evm", Key: []byte{1}, Delete: true}, + }) + require.Error(t, err) +} + +func testBasics(t *testing.T, store VersionStore) { + var v int64 + + SetupTestDB(t, store) + + value, err := store.GetAtVersion("evm", []byte("z-genesis-only"), nil) + require.NoError(t, err) + require.Equal(t, value, []byte("2")) + + v = 4 + ok, err := store.HasAtVersion("evm", []byte("z-genesis-only"), &v) + require.NoError(t, err) + require.True(t, ok) + value, err = store.GetAtVersion("evm", []byte("z-genesis-only"), &v) + require.NoError(t, err) + require.Equal(t, value, []byte("2")) + + value, err = store.GetAtVersion("evm", []byte("re-add-in-block3"), nil) + require.NoError(t, err) + require.Empty(t, value) + + ok, err = store.HasAtVersion("staking", key1, nil) + require.NoError(t, err) + require.True(t, ok) + + value, err = store.GetAtVersion("staking", key1, nil) + require.NoError(t, err) + require.Equal(t, value, []byte("value2")) + + v = 2 + value, err = store.GetAtVersion("staking", key1, &v) + require.NoError(t, err) + require.Equal(t, value, []byte("value2")) + + ok, err = store.HasAtVersion("staking", key1, &v) + require.NoError(t, err) + require.True(t, ok) + + v = 0 + value, err = store.GetAtVersion("staking", key1, &v) + require.NoError(t, err) + require.Equal(t, value, []byte("value1")) + + v = 1 + value, err = store.GetAtVersion("staking", key1, &v) + require.NoError(t, err) + require.Empty(t, value) + + ok, err = store.HasAtVersion("staking", key1, &v) + require.NoError(t, err) + require.False(t, ok) + + v = 0 + value, err = store.GetAtVersion("staking", key1, &v) + require.NoError(t, err) + require.Equal(t, value1, value) + value, err = store.GetAtVersion("staking", key1_subkey, &v) + require.NoError(t, err) + require.Equal(t, value1, value) +} + +type KVPair struct { + Key []byte + Value []byte +} + +func testIterator(t *testing.T, store VersionStore) { + SetupTestDB(t, store) + + expItems := [][]KVPair{ + { + KVPair{[]byte("delete-in-block2"), []byte("1")}, + KVPair{[]byte("modify-in-block2"), []byte("1")}, + KVPair{[]byte("re-add-in-block3"), []byte("1")}, + KVPair{[]byte("z-genesis-only"), []byte("2")}, + }, + { + KVPair{[]byte("add-in-block1"), []byte("1")}, + KVPair{[]byte("delete-in-block2"), []byte("1")}, + KVPair{[]byte("modify-in-block2"), []byte("1")}, + KVPair{[]byte("z-genesis-only"), []byte("2")}, + }, + { + KVPair{[]byte("add-in-block1"), []byte("1")}, + KVPair{[]byte("add-in-block2"), []byte("1")}, + KVPair{[]byte("modify-in-block2"), []byte("2")}, + KVPair{[]byte("z-genesis-only"), []byte("2")}, + }, + { + KVPair{[]byte("add-in-block1"), []byte("1")}, + KVPair{[]byte("add-in-block2"), []byte("1")}, + KVPair{[]byte("modify-in-block2"), []byte("2")}, + KVPair{[]byte("re-add-in-block3"), []byte("2")}, + KVPair{[]byte("z-genesis-only"), []byte("2")}, + }, + { + KVPair{[]byte("add-in-block1"), []byte("1")}, + KVPair{[]byte("add-in-block2"), []byte("1")}, + KVPair{[]byte("modify-in-block2"), []byte("2")}, + KVPair{[]byte("z-genesis-only"), []byte("2")}, + }, + } + for i, exp := range expItems { + t.Run(fmt.Sprintf("block-%d", i), func(t *testing.T) { + v := int64(i) + it, err := store.IteratorAtVersion("evm", nil, nil, &v) + require.NoError(t, err) + require.Equal(t, exp, consumeIterator(it)) + + it, err = store.ReverseIteratorAtVersion("evm", nil, nil, &v) + require.NoError(t, err) + actual := consumeIterator(it) + require.Equal(t, len(exp), len(actual)) + require.Equal(t, reversed(exp), actual) + }) + } + + it, err := store.IteratorAtVersion("evm", nil, nil, nil) + require.Equal(t, expItems[len(expItems)-1], consumeIterator(it)) + + it, err = store.ReverseIteratorAtVersion("evm", nil, nil, nil) + require.Equal(t, reversed(expItems[len(expItems)-1]), consumeIterator(it)) + + // with start parameter + v := int64(2) + it, err = store.IteratorAtVersion("evm", []byte("\xff"), nil, &v) + require.NoError(t, err) + require.Empty(t, consumeIterator(it)) + it, err = store.ReverseIteratorAtVersion("evm", nil, []byte("\x00"), &v) + require.NoError(t, err) + require.Empty(t, consumeIterator(it)) + + it, err = store.IteratorAtVersion("evm", []byte("modify-in-block2"), nil, &v) + require.NoError(t, err) + require.Equal(t, expItems[2][len(expItems[2])-2:], consumeIterator(it)) + + it, err = store.ReverseIteratorAtVersion("evm", nil, []byte("mp"), &v) + require.NoError(t, err) + require.Equal(t, + reversed(expItems[2][:len(expItems[2])-1]), + consumeIterator(it), + ) + + it, err = store.ReverseIteratorAtVersion("evm", nil, []byte("modify-in-block3"), &v) + require.NoError(t, err) + require.Equal(t, + reversed(expItems[2][:len(expItems[2])-1]), + consumeIterator(it), + ) + + // delete the last key, cover some edge cases + v = int64(len(expItems)) + err = store.PutAtVersion( + v, + []types.StoreKVPair{ + {StoreKey: "evm", Key: []byte("z-genesis-only"), Delete: true}, + }, + ) + require.NoError(t, err) + it, err = store.IteratorAtVersion("evm", nil, nil, &v) + require.NoError(t, err) + require.Equal(t, + expItems[v-1][:len(expItems[v-1])-1], + consumeIterator(it), + ) + v -= 1 + it, err = store.IteratorAtVersion("evm", nil, nil, &v) + require.NoError(t, err) + require.Equal(t, + expItems[v], + consumeIterator(it), + ) +} + +func testHeightInFuture(t *testing.T, store VersionStore) { + SetupTestDB(t, store) + + latest, err := store.GetLatestVersion() + require.NoError(t, err) + + v := latest + 1 + _, err = store.GetAtVersion("staking", key1, &v) + require.Error(t, err) + _, err = store.HasAtVersion("staking", key1, &v) + require.Error(t, err) + _, err = store.IteratorAtVersion("staking", nil, nil, &v) + require.Error(t, err) + _, err = store.ReverseIteratorAtVersion("staking", nil, nil, &v) + require.Error(t, err) +} + +func consumeIterator(it dbm.Iterator) []KVPair { + var result []KVPair + for ; it.Valid(); it.Next() { + result = append(result, KVPair{it.Key(), it.Value()}) + } + it.Close() + return result +} + +// reversed clone and reverse the slice +func reversed[S ~[]E, E any](s S) []E { + r := make([]E, len(s)) + for i, j := 0, len(s)-1; i <= j; i, j = i+1, j-1 { + r[i], r[j] = s[j], s[i] + } + return r +} diff --git a/versiondb/dbutils.go b/versiondb/dbutils.go new file mode 100644 index 0000000000..2334d0f2c1 --- /dev/null +++ b/versiondb/dbutils.go @@ -0,0 +1,84 @@ +package versiondb + +import ( + "encoding/binary" + "sort" + + "github.com/RoaringBitmap/roaring/roaring64" +) + +var ChunkLimit = uint64(1950) // threshold beyond which MDBX overflow pages appear: 4096 / 2 - (keySize + 8) + +// CutLeft - cut from bitmap `targetSize` bytes from left +// removing lft part from `bm` +// returns nil on zero cardinality +func CutLeft64(bm *roaring64.Bitmap, sizeLimit uint64) *roaring64.Bitmap { + if bm.GetCardinality() == 0 { + return nil + } + + sz := bm.GetSerializedSizeInBytes() + if sz <= sizeLimit { + lft := roaring64.New() + lft.AddRange(bm.Minimum(), bm.Maximum()+1) + lft.And(bm) + lft.RunOptimize() + bm.Clear() + return lft + } + + from := bm.Minimum() + minMax := bm.Maximum() - bm.Minimum() + to := sort.Search(int(minMax), func(i int) bool { // can be optimized to avoid "too small steps", but let's leave it for readability + lft := roaring64.New() // bitmap.Clear() method intentionally not used here, because then serialized size of bitmap getting bigger + lft.AddRange(from, from+uint64(i)+1) + lft.And(bm) + lft.RunOptimize() + return lft.GetSerializedSizeInBytes() > sizeLimit + }) + + lft := roaring64.New() + lft.AddRange(from, from+uint64(to)) // no +1 because sort.Search returns element which is just higher threshold - but we need lower + lft.And(bm) + bm.RemoveRange(from, from+uint64(to)) + lft.RunOptimize() + return lft +} + +func WalkChunks64(bm *roaring64.Bitmap, sizeLimit uint64, f func(chunk *roaring64.Bitmap, isLast bool) error) error { + for bm.GetCardinality() > 0 { + if err := f(CutLeft64(bm, sizeLimit), bm.GetCardinality() == 0); err != nil { + return err + } + } + return nil +} + +func WalkChunkWithKeys64(k []byte, m *roaring64.Bitmap, sizeLimit uint64, f func(chunkKey []byte, chunk *roaring64.Bitmap) error) error { + return WalkChunks64(m, sizeLimit, func(chunk *roaring64.Bitmap, isLast bool) error { + chunkKey := make([]byte, len(k)+8) + copy(chunkKey, k) + if isLast { + binary.BigEndian.PutUint64(chunkKey[len(k):], ^uint64(0)) + } else { + binary.BigEndian.PutUint64(chunkKey[len(k):], chunk.Maximum()) + } + return f(chunkKey, chunk) + }) +} + +// SeekInBitmap64 - returns value in bitmap which is >= n +func SeekInBitmap64(m *roaring64.Bitmap, n uint64) (found uint64, ok bool) { + if m == nil || m.IsEmpty() { + return 0, false + } + if n == 0 { + return m.Minimum(), true + } + searchRank := m.Rank(n - 1) + if searchRank >= m.GetCardinality() { + return 0, false + } + found, _ = m.Select(searchRank) + return found, true +} diff --git a/versiondb/multistore.go b/versiondb/multistore.go new file mode 100644 index 0000000000..4871558fd9 --- /dev/null +++ b/versiondb/multistore.go @@ -0,0 +1,135 @@ +package versiondb + +import ( + "io" + "sync" + + "github.com/cosmos/cosmos-sdk/store/cachemulti" + "github.com/cosmos/cosmos-sdk/store/mem" + "github.com/cosmos/cosmos-sdk/store/transient" + "github.com/cosmos/cosmos-sdk/store/types" + sdk "github.com/cosmos/cosmos-sdk/types" +) + +var _ sdk.MultiStore = (*MultiStore)(nil) + +type MultiStore struct { + versionDB VersionStore + storeKeys []types.StoreKey + + // transient or memory stores + transientStores map[types.StoreKey]types.KVStore + + traceWriter io.Writer + traceContext types.TraceContext + traceContextMutex sync.Mutex +} + +func NewMultiStore(versionDB VersionStore, storeKeys []types.StoreKey) *MultiStore { + return &MultiStore{versionDB: versionDB, storeKeys: storeKeys, transientStores: make(map[types.StoreKey]types.KVStore)} +} + +func (s *MultiStore) GetStoreType() types.StoreType { + return types.StoreTypeMulti +} + +func (s *MultiStore) cacheMultiStore(version *int64) sdk.CacheMultiStore { + stores := make(map[types.StoreKey]types.CacheWrapper, len(s.transientStores)+len(s.storeKeys)) + for k, v := range s.transientStores { + stores[k] = v + } + for _, k := range s.storeKeys { + stores[k] = NewKVStore(s.versionDB, k, version) + } + return cachemulti.NewStore(nil, stores, nil, s.traceWriter, s.getTracingContext()) +} + +func (s *MultiStore) CacheMultiStore() sdk.CacheMultiStore { + return s.cacheMultiStore(nil) +} + +func (s *MultiStore) CacheMultiStoreWithVersion(version int64) (sdk.CacheMultiStore, error) { + return s.cacheMultiStore(&version), nil +} + +// CacheWrap implements CacheWrapper/MultiStore/CommitStore. +func (s *MultiStore) CacheWrap() types.CacheWrap { + return s.CacheMultiStore().(types.CacheWrap) +} + +// CacheWrapWithTrace implements the CacheWrapper interface. +func (s *MultiStore) CacheWrapWithTrace(_ io.Writer, _ types.TraceContext) types.CacheWrap { + return s.CacheWrap() +} + +func (s *MultiStore) GetStore(storeKey types.StoreKey) sdk.Store { + return s.GetKVStore(storeKey) +} + +func (s *MultiStore) GetKVStore(storeKey types.StoreKey) sdk.KVStore { + store, ok := s.transientStores[storeKey] + if ok { + return store + } + return NewKVStore(s.versionDB, storeKey, nil) +} + +func (s *MultiStore) MountTransientStores(keys map[string]*types.TransientStoreKey) { + for _, key := range keys { + s.transientStores[key] = transient.NewStore() + } +} + +func (s *MultiStore) MountMemoryStores(keys map[string]*types.MemoryStoreKey) { + for _, key := range keys { + s.transientStores[key] = mem.NewStore() + } +} + +// SetTracer sets the tracer for the MultiStore that the underlying +// stores will utilize to trace operations. A MultiStore is returned. +func (s *MultiStore) SetTracer(w io.Writer) types.MultiStore { + s.traceWriter = w + return s +} + +// SetTracingContext updates the tracing context for the MultiStore by merging +// the given context with the existing context by key. Any existing keys will +// be overwritten. It is implied that the caller should update the context when +// necessary between tracing operations. It returns a modified MultiStore. +func (s *MultiStore) SetTracingContext(tc types.TraceContext) types.MultiStore { + s.traceContextMutex.Lock() + defer s.traceContextMutex.Unlock() + s.traceContext = s.traceContext.Merge(tc) + + return s +} + +func (s *MultiStore) getTracingContext() types.TraceContext { + s.traceContextMutex.Lock() + defer s.traceContextMutex.Unlock() + + if s.traceContext == nil { + return nil + } + + ctx := types.TraceContext{} + for k, v := range s.traceContext { + ctx[k] = v + } + + return ctx +} + +// TracingEnabled returns if tracing is enabled for the MultiStore. +func (s *MultiStore) TracingEnabled() bool { + return s.traceWriter != nil +} + +func (s *MultiStore) LatestVersion() int64 { + version, err := s.versionDB.GetLatestVersion() + if err != nil { + panic(err) + } + return version +} diff --git a/versiondb/store.go b/versiondb/store.go new file mode 100644 index 0000000000..906423fb9e --- /dev/null +++ b/versiondb/store.go @@ -0,0 +1,94 @@ +package versiondb + +import ( + "io" + "time" + + "github.com/cosmos/cosmos-sdk/store/cachekv" + "github.com/cosmos/cosmos-sdk/store/listenkv" + "github.com/cosmos/cosmos-sdk/store/tracekv" + "github.com/cosmos/cosmos-sdk/store/types" + "github.com/cosmos/cosmos-sdk/telemetry" +) + +var _ types.KVStore = (*Store)(nil) + +// Store Implements types.KVStore +type Store struct { + store VersionStore + storeKey types.StoreKey + version *int64 +} + +func NewKVStore(store VersionStore, storeKey types.StoreKey, version *int64) *Store { + return &Store{store, storeKey, version} +} + +// Implements Store. +func (st *Store) GetStoreType() types.StoreType { + // FIXME + return types.StoreTypeIAVL +} + +// Implements Store. +func (st *Store) CacheWrap() types.CacheWrap { + return cachekv.NewStore(st) +} + +// CacheWrapWithTrace implements the Store interface. +func (st *Store) CacheWrapWithTrace(w io.Writer, tc types.TraceContext) types.CacheWrap { + return cachekv.NewStore(tracekv.NewStore(st, w, tc)) +} + +// CacheWrapWithListeners implements the CacheWrapper interface. +func (st *Store) CacheWrapWithListeners(storeKey types.StoreKey, listeners []types.WriteListener) types.CacheWrap { + return cachekv.NewStore(listenkv.NewStore(st, storeKey, listeners)) +} + +// Implements types.KVStore. +func (st *Store) Get(key []byte) []byte { + defer telemetry.MeasureSince(time.Now(), "store", "iavl", "get") + value, err := st.store.GetAtVersion(st.storeKey.Name(), key, st.version) + if err != nil { + panic(err) + } + return value +} + +// Implements types.KVStore. +func (st *Store) Has(key []byte) (exists bool) { + defer telemetry.MeasureSince(time.Now(), "store", "iavl", "has") + has, err := st.store.HasAtVersion(st.storeKey.Name(), key, st.version) + if err != nil { + panic(err) + } + return has +} + +// Implements types.KVStore. +func (st *Store) Iterator(start, end []byte) types.Iterator { + itr, err := st.store.IteratorAtVersion(st.storeKey.Name(), start, end, st.version) + if err != nil { + panic(err) + } + return itr +} + +// Implements types.KVStore. +func (st *Store) ReverseIterator(start, end []byte) types.Iterator { + itr, err := st.store.ReverseIteratorAtVersion(st.storeKey.Name(), start, end, st.version) + if err != nil { + panic(err) + } + return itr +} + +// Implements types.KVStore. +func (st *Store) Set(key, value []byte) { + panic("write operation is not supported") +} + +// Implements types.KVStore. +func (st *Store) Delete(key []byte) { + panic("write operation is not supported") +} diff --git a/versiondb/streaming_service.go b/versiondb/streaming_service.go new file mode 100644 index 0000000000..e56aef0e18 --- /dev/null +++ b/versiondb/streaming_service.go @@ -0,0 +1,84 @@ +package versiondb + +import ( + "sort" + "strings" + "sync" + + abci "github.com/tendermint/tendermint/abci/types" + + "github.com/cosmos/cosmos-sdk/baseapp" + "github.com/cosmos/cosmos-sdk/store/types" + sdk "github.com/cosmos/cosmos-sdk/types" +) + +var _ baseapp.StreamingService = &StreamingService{} + +// StreamingService is a concrete implementation of StreamingService that accumulate the state changes in current block, +// writes the ordered changeset out to version storage. +type StreamingService struct { + listeners []*types.MemoryListener // the listeners that will be initialized with BaseApp + versionStore VersionStore + currentBlockNumber int64 // the current block number +} + +// NewStreamingService creates a new StreamingService for the provided writeDir, (optional) filePrefix, and storeKeys +func NewStreamingService(versionStore VersionStore, storeKeys []types.StoreKey) *StreamingService { + // sort by the storeKeys first + sort.SliceStable(storeKeys, func(i, j int) bool { + return strings.Compare(storeKeys[i].Name(), storeKeys[j].Name()) < 0 + }) + + listeners := make([]*types.MemoryListener, len(storeKeys)) + for i, key := range storeKeys { + listeners[i] = types.NewMemoryListener(key) + } + return &StreamingService{listeners, versionStore, 0} +} + +// Listeners satisfies the baseapp.StreamingService interface +func (fss *StreamingService) Listeners() map[types.StoreKey][]types.WriteListener { + listeners := make(map[types.StoreKey][]types.WriteListener, len(fss.listeners)) + for _, listener := range fss.listeners { + listeners[listener.StoreKey()] = []types.WriteListener{listener} + } + return listeners +} + +// ListenBeginBlock satisfies the baseapp.ABCIListener interface +// It sets the currentBlockNumber. +func (fss *StreamingService) ListenBeginBlock(ctx sdk.Context, req abci.RequestBeginBlock, res abci.ResponseBeginBlock) error { + fss.currentBlockNumber = req.GetHeader().Height + return nil +} + +// ListenDeliverTx satisfies the baseapp.ABCIListener interface +func (fss *StreamingService) ListenDeliverTx(ctx sdk.Context, req abci.RequestDeliverTx, res abci.ResponseDeliverTx) error { + return nil +} + +// ListenEndBlock satisfies the baseapp.ABCIListener interface +// It merge the state caches of all the listeners together, and write out to the versionStore. +func (fss *StreamingService) ListenEndBlock(ctx sdk.Context, req abci.RequestEndBlock, res abci.ResponseEndBlock) error { + return nil +} + +func (fss *StreamingService) ListenCommit(ctx sdk.Context, res abci.ResponseCommit) error { + // concat the state caches + var changeSet []types.StoreKVPair + for _, listener := range fss.listeners { + changeSet = append(changeSet, listener.PopStateCache()...) + } + + return fss.versionStore.PutAtVersion(fss.currentBlockNumber, changeSet) +} + +// Stream satisfies the baseapp.StreamingService interface +func (fss *StreamingService) Stream(wg *sync.WaitGroup) error { + return nil +} + +// Close satisfies the io.Closer interface, which satisfies the baseapp.StreamingService interface +func (fss *StreamingService) Close() error { + return nil +} diff --git a/versiondb/sync.go b/versiondb/sync.go new file mode 100644 index 0000000000..4c89590e4c --- /dev/null +++ b/versiondb/sync.go @@ -0,0 +1,31 @@ +package versiondb + +import ( + "bufio" + "io" + + protoio "github.com/gogo/protobuf/io" + + "github.com/cosmos/cosmos-sdk/store/types" +) + +const maxItemSize = 64000000 // SDK has no key/value size limit, so we set an arbitrary limit + +// ReadFileStreamer parse a binary stream dumped by file streamer to changeset, +// which can be feeded to version store. +func ReadFileStreamer(input *bufio.Reader) ([]types.StoreKVPair, error) { + var changeSet []types.StoreKVPair + reader := protoio.NewDelimitedReader(input, maxItemSize) + for { + var msg types.StoreKVPair + err := reader.ReadMsg(&msg) + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + changeSet = append(changeSet, msg) + } + return changeSet, nil +} diff --git a/versiondb/sync_test.go b/versiondb/sync_test.go new file mode 100644 index 0000000000..85498ee07f --- /dev/null +++ b/versiondb/sync_test.go @@ -0,0 +1,340 @@ +package versiondb + +import ( + "bufio" + "bytes" + "encoding/hex" + "strings" + "testing" + + "github.com/cosmos/cosmos-sdk/store/types" + "github.com/stretchr/testify/require" +) + +const data = ` +a2380aff030a20ff2dd95def59a0c5265f30768a1c6fbdab519c96fb98df +33d524a94575bac1e91292030a02080b120c63726f6e6f735f3737372d31 +1802220c08d9a39a9a0610b8d89eca022a480a202d8f46f61152696c6dc8 +0c6367c7ddc926dc3ed7ab23bd37fb7ca83eb00ba207122408011220df77 +0b6cb0a9694f550ae515d5fb0ebc08902e0fa47ceec816928e74fd8fee44 +322004943bc3709104b52f30f83ca3d30d8bf57551edd48797aef5cee50b +b532c15c3a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934c +a495991b7852b8554220f537a6e0561fa0edd12b30ec9b6479e659f6f1fa +1587e69556201bfaf4cf97404a20f537a6e0561fa0edd12b30ec9b6479e6 +59f6f1fa1587e69556201bfaf4cf97405220252fe7cf36dd1bb85dafc47a +08961df0cfd8c027defa5e01e958be121599db9d5a209048462db52bb809 +6c92f8733fcf26455a000a550d9f3748dddadcc2267917146220e3b0c442 +98fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8556a20 +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852 +b8557214ae7ccd8d599769209074b81d0c2f3f28624742b71a4612210a1d +0a142861b93c776d100695688250c5b1ba4d44ef2a791880a094a58d1d10 +0112210a1d0a14ae7ccd8d599769209074b81d0c2f3f28624742b71880a0 +94a58d1d100112ae100a630a0d636f696e5f726563656976656412360a08 +7265636569766572122a637263316d33683330776c767366386c6c727578 +7470756b64767379306b6d326b756d386c3061773437121a0a06616d6f75 +6e74121034313139343530383537377374616b650a5c0a08636f696e6261 +736512340a066d696e746572122a637263316d33683330776c767366386c +6c7275787470756b64767379306b6d326b756d386c3061773437121a0a06 +616d6f756e74121034313139343530383537377374616b650a5f0a0a636f +696e5f7370656e7412350a077370656e646572122a637263316d33683330 +776c767366386c6c7275787470756b64767379306b6d326b756d386c3061 +773437121a0a06616d6f756e74121034313139343530383537377374616b +650a630a0d636f696e5f726563656976656412360a087265636569766572 +122a637263313778706676616b6d32616d67393632796c73366638347a33 +6b656c6c3863356c3838656b6572121a0a06616d6f756e74121034313139 +343530383537377374616b650a95010a087472616e7366657212370a0972 +6563697069656e74122a637263313778706676616b6d32616d6739363279 +6c73366638347a336b656c6c3863356c3838656b657212340a0673656e64 +6572122a637263316d33683330776c767366386c6c7275787470756b6476 +7379306b6d326b756d386c3061773437121a0a06616d6f756e7412103431 +3139343530383537377374616b650a3f0a076d65737361676512340a0673 +656e646572122a637263316d33683330776c767366386c6c727578747075 +6b64767379306b6d326b756d386c30617734370aa2010a046d696e741224 +0a0c626f6e6465645f726174696f1214302e393939393939393739343032 +37343439353212210a09696e666c6174696f6e1214302e31323939393939 +3739373130313635333031123a0a11616e6e75616c5f70726f766973696f +6e7312253235393939393936343737353631363138382e37363031383234 +363033383333383838343312150a06616d6f756e74120b34313139343530 +383537370a5f0a0a636f696e5f7370656e7412350a077370656e64657212 +2a637263313778706676616b6d32616d67393632796c73366638347a336b +656c6c3863356c3838656b6572121a0a06616d6f756e7412103832333839 +3031393532307374616b650a630a0d636f696e5f72656365697665641236 +0a087265636569766572122a637263316a7636357333677271663676366a +6c33647034743663397439726b3939636438737037326d70121a0a06616d +6f756e74121038323338393031393532307374616b650a95010a08747261 +6e7366657212370a09726563697069656e74122a637263316a7636357333 +677271663676366a6c33647034743663397439726b393963643873703732 +6d7012340a0673656e646572122a637263313778706676616b6d32616d67 +393632796c73366638347a336b656c6c3863356c3838656b6572121a0a06 +616d6f756e74121038323338393031393532307374616b650a3f0a076d65 +737361676512340a0673656e646572122a637263313778706676616b6d32 +616d67393632796c73366638347a336b656c6c3863356c3838656b65720a +7f0a0f70726f706f7365725f726577617264122c0a06616d6f756e741222 +343131393435303937362e30303030303030303030303030303030303073 +74616b65123e0a0976616c696461746f72123163726376616c6f70657231 +326c756b75367578656868616b303270793472637a36357a753073776837 +776a36756c726c670a790a0a636f6d6d697373696f6e122b0a06616d6f75 +6e7412213431313934353039372e36303030303030303030303030303030 +30307374616b65123e0a0976616c696461746f72123163726376616c6f70 +657231326c756b75367578656868616b303270793472637a36357a753073 +776837776a36756c726c670a770a0772657761726473122c0a06616d6f75 +6e741222343131393435303937362e303030303030303030303030303030 +3030307374616b65123e0a0976616c696461746f72123163726376616c6f +70657231326c756b75367578656868616b303270793472637a36357a7530 +73776837776a36756c726c670a7a0a0a636f6d6d697373696f6e122c0a06 +616d6f756e741222333833313038393430372e3638303030303030303030 +303030303030307374616b65123e0a0976616c696461746f721231637263 +76616c6f70657231326c756b75367578656868616b303270793472637a36 +357a753073776837776a36756c726c670a780a0772657761726473122d0a +06616d6f756e74122333383331303839343037362e383030303030303030 +3030303030303030307374616b65123e0a0976616c696461746f72123163 +726376616c6f70657231326c756b75367578656868616b30327079347263 +7a36357a753073776837776a36756c726c670a7a0a0a636f6d6d69737369 +6f6e122c0a06616d6f756e741222333833313038393430372e3638303030 +303030303030303030303030307374616b65123e0a0976616c696461746f +72123163726376616c6f70657231387a367133386d68767473767972356d +616b38666a38733867346777376b6a6a7030653064680a780a0772657761 +726473122d0a06616d6f756e74122333383331303839343037362e383030 +3030303030303030303030303030307374616b65123e0a0976616c696461 +746f72123163726376616c6f70657231387a367133386d68767473767972 +356d616b38666a38733867346777376b6a6a7030653064680a250a0a6665 +655f6d61726b657412170a08626173655f666565120b3736353632353030 +3030301aa9100aa4040aa1040afa020aba020a2a2f636f736d6f732e7374 +616b696e672e763162657461312e4d736743726561746556616c69646174 +6f72128b020a070a056e6f646530123b0a12313030303030303030303030 +30303030303012123230303030303030303030303030303030301a113130 +3030303030303030303030303030301a0131222a63726331326c756b7536 +7578656868616b303270793472637a36357a753073776837776a73727730 +70702a3163726376616c6f70657231326c756b75367578656868616b3032 +70793472637a36357a753073776837776a36756c726c6732430a1d2f636f +736d6f732e63727970746f2e656432353531392e5075624b657912220a20 +9b86839433f4229b7b7b51554a25ed32549126eb80fe53497ae336a1e67b +66843a1c0a057374616b6512133130303030303030303030303030303030 +3030123b3439643130313933363334303730393664303337396438643863 +3833336438626633396334383866403139322e3136382e302e35343a3236 +363536125f0a570a4f0a282f65746865726d696e742e63727970746f2e76 +312e657468736563703235366b312e5075624b657912230a21026e710a62 +a342de0ed4d7c4532dcbcbbafbf19652ed67b237efab70e8b207efac1204 +0a020801120410c09a0c1a4119ffd1b342c44e183b8113561595186e1f72 +c7273ac4e9ccb04c2b8a4d31b3780eff63cb490102ed39f7482084b4ddb8 +4d672ee2607262a312724f0e5b2b79aa0012ff0b123612340a322f636f73 +6d6f732e7374616b696e672e763162657461312e4d736743726561746556 +616c696461746f72526573706f6e73651ae0055b7b226d73675f696e6465 +78223a302c226576656e7473223a5b7b2274797065223a22636f696e5f72 +65636569766564222c2261747472696275746573223a5b7b226b6579223a +227265636569766572222c2276616c7565223a22637263317479676d7333 +78686873337976343837706878336477346139356a6e3774376c6b393067 +6161227d2c7b226b6579223a22616d6f756e74222c2276616c7565223a22 +313030303030303030303030303030303030307374616b65227d5d7d2c7b +2274797065223a22636f696e5f7370656e74222c22617474726962757465 +73223a5b7b226b6579223a227370656e646572222c2276616c7565223a22 +63726331326c756b75367578656868616b303270793472637a36357a7530 +73776837776a737277307070227d2c7b226b6579223a22616d6f756e7422 +2c2276616c7565223a223130303030303030303030303030303030303073 +74616b65227d5d7d2c7b2274797065223a226372656174655f76616c6964 +61746f72222c2261747472696275746573223a5b7b226b6579223a227661 +6c696461746f72222c2276616c7565223a2263726376616c6f7065723132 +6c756b75367578656868616b303270793472637a36357a75307377683777 +6a36756c726c67227d2c7b226b6579223a22616d6f756e74222c2276616c +7565223a22313030303030303030303030303030303030307374616b6522 +7d5d7d2c7b2274797065223a226d657373616765222c2261747472696275 +746573223a5b7b226b6579223a22616374696f6e222c2276616c7565223a +222f636f736d6f732e7374616b696e672e763162657461312e4d73674372 +6561746556616c696461746f72227d2c7b226b6579223a226d6f64756c65 +222c2276616c7565223a227374616b696e67227d2c7b226b6579223a2273 +656e646572222c2276616c7565223a2263726331326c756b753675786568 +68616b303270793472637a36357a753073776837776a737277307070227d +5d7d5d7d5d28ffffffffffffffffff0130bca10d3a440a02747812050a03 +66656512370a096665655f7061796572122a63726331326c756b75367578 +656868616b303270793472637a36357a753073776837776a737277307070 +3a3d0a02747812370a076163635f736571122c63726331326c756b753675 +78656868616b303270793472637a36357a753073776837776a7372773070 +702f303a6b0a02747812650a097369676e6174757265125847662f527330 +4c455468673767524e57465a55596268397978796336784f6e4d73457772 +696b30787333674f2f32504c5351454337546e3353434345744e32345457 +6375346d427959714d53636b384f577974357167413d3a3f0a076d657373 +61676512340a06616374696f6e122a2f636f736d6f732e7374616b696e67 +2e763162657461312e4d736743726561746556616c696461746f723a670a +0a636f696e5f7370656e7412350a077370656e646572122a63726331326c +756b75367578656868616b303270793472637a36357a753073776837776a +73727730707012220a06616d6f756e741218313030303030303030303030 +303030303030307374616b653a6b0a0d636f696e5f726563656976656412 +360a087265636569766572122a637263317479676d733378686873337976 +343837706878336477346139356a6e3774376c6b393067616112220a0661 +6d6f756e741218313030303030303030303030303030303030307374616b +653a760a106372656174655f76616c696461746f72123e0a0976616c6964 +61746f72123163726376616c6f70657231326c756b75367578656868616b +303270793472637a36357a753073776837776a36756c726c6712220a0661 +6d6f756e741218313030303030303030303030303030303030307374616b +653a520a076d65737361676512110a066d6f64756c6512077374616b696e +6712340a0673656e646572122a63726331326c756b75367578656868616b +303270793472637a36357a753073776837776a7372773070701aa9100aa4 +040aa1040afa020aba020a2a2f636f736d6f732e7374616b696e672e7631 +62657461312e4d736743726561746556616c696461746f72128b020a070a +056e6f646531123b0a123130303030303030303030303030303030301212 +3230303030303030303030303030303030301a1131303030303030303030 +303030303030301a0131222a63726331387a367133386d68767473767972 +356d616b38666a38733867346777376b6a6a747367726e372a3163726376 +616c6f70657231387a367133386d68767473767972356d616b38666a3873 +3867346777376b6a6a70306530646832430a1d2f636f736d6f732e637279 +70746f2e656432353531392e5075624b657912220a2072b50cf0ed1863ff +c937af99b6ad779a2c223e59459eab7768bda7c2da6f836e3a1c0a057374 +616b65121331303030303030303030303030303030303030123b35396566 +333139663464383334396466626532613233653131356163353835356430 +303938613065403139322e3136382e302e35343a3236363536125f0a570a +4f0a282f65746865726d696e742e63727970746f2e76312e657468736563 +703235366b312e5075624b657912230a210242785a75074452d62a6ac222 +70ffb8fb01c9375d0ba72887ae800dc619315d1b12040a020801120410c0 +9a0c1a41e7019fd760970e02f8967aa0f9820c0b98de32d8e72601aa34fe +60df52356d19591eb2bd8516037d2c52c22170ca533abf72d50d4c7f770d +1d5e045df51ff89c0112ff0b123612340a322f636f736d6f732e7374616b +696e672e763162657461312e4d736743726561746556616c696461746f72 +526573706f6e73651ae0055b7b226d73675f696e646578223a302c226576 +656e7473223a5b7b2274797065223a22636f696e5f726563656976656422 +2c2261747472696275746573223a5b7b226b6579223a2272656365697665 +72222c2276616c7565223a22637263317479676d73337868687333797634 +3837706878336477346139356a6e3774376c6b3930676161227d2c7b226b +6579223a22616d6f756e74222c2276616c7565223a223130303030303030 +30303030303030303030307374616b65227d5d7d2c7b2274797065223a22 +636f696e5f7370656e74222c2261747472696275746573223a5b7b226b65 +79223a227370656e646572222c2276616c7565223a2263726331387a3671 +33386d68767473767972356d616b38666a38733867346777376b6a6a7473 +67726e37227d2c7b226b6579223a22616d6f756e74222c2276616c756522 +3a22313030303030303030303030303030303030307374616b65227d5d7d +2c7b2274797065223a226372656174655f76616c696461746f72222c2261 +747472696275746573223a5b7b226b6579223a2276616c696461746f7222 +2c2276616c7565223a2263726376616c6f70657231387a367133386d6876 +7473767972356d616b38666a38733867346777376b6a6a70306530646822 +7d2c7b226b6579223a22616d6f756e74222c2276616c7565223a22313030 +303030303030303030303030303030307374616b65227d5d7d2c7b227479 +7065223a226d657373616765222c2261747472696275746573223a5b7b22 +6b6579223a22616374696f6e222c2276616c7565223a222f636f736d6f73 +2e7374616b696e672e763162657461312e4d736743726561746556616c69 +6461746f72227d2c7b226b6579223a226d6f64756c65222c2276616c7565 +223a227374616b696e67227d2c7b226b6579223a2273656e646572222c22 +76616c7565223a2263726331387a367133386d68767473767972356d616b +38666a38733867346777376b6a6a747367726e37227d5d7d5d7d5d28ffff +ffffffffffffff0130fc8c0d3a440a02747812050a0366656512370a0966 +65655f7061796572122a63726331387a367133386d68767473767972356d +616b38666a38733867346777376b6a6a747367726e373a3d0a0274781237 +0a076163635f736571122c63726331387a367133386d6876747376797235 +6d616b38666a38733867346777376b6a6a747367726e372f303a6b0a0274 +7812650a097369676e61747572651258357747663132435844674c346c6e +71672b59494d43356a654d746a6e4a6747714e5035673331493162526c5a +48724b39685259446653785377694677796c4d3676334c564455782f6477 +30645867526439522f346e41453d3a3f0a076d65737361676512340a0661 +6374696f6e122a2f636f736d6f732e7374616b696e672e76316265746131 +2e4d736743726561746556616c696461746f723a670a0a636f696e5f7370 +656e7412350a077370656e646572122a63726331387a367133386d687674 +73767972356d616b38666a38733867346777376b6a6a747367726e371222 +0a06616d6f756e7412183130303030303030303030303030303030303073 +74616b653a6b0a0d636f696e5f726563656976656412360a087265636569 +766572122a637263317479676d7333786868733379763438377068783364 +77346139356a6e3774376c6b393067616112220a06616d6f756e74121831 +3030303030303030303030303030303030307374616b653a760a10637265 +6174655f76616c696461746f72123e0a0976616c696461746f7212316372 +6376616c6f70657231387a367133386d68767473767972356d616b38666a +38733867346777376b6a6a70306530646812220a06616d6f756e74121831 +3030303030303030303030303030303030307374616b653a520a076d6573 +7361676512110a066d6f64756c6512077374616b696e6712340a0673656e +646572122a63726331387a367133386d68767473767972356d616b38666a +38733867346777376b6a6a747367726e37220208022aec0212260a090880 +804010e0aeee26120e08a08d0612040880c60a188080401a090a07656432 +353531391a9a020a0b626c6f636b5f626c6f6f6d128a020a05626c6f6f6d +128002000000000000000000000000000000000000000000000000000000 +000000000000000000000000000000000000000000000000000000000000 +000000000000000000000000000000000000000000000000000000000000 +000000000000000000000000000000000000000000000000000000000000 +000000000000000000000000000000000000000000000000000000000000 +000000000000000000000000000000000000000000000000000000000000 +000000000000000000000000000000000000000000000000000000000000 +000000000000000000000000000000000000000000000000000000000000 +000000000000000000000000000000000000001a250a09626c6f636b5f67 +6173120b0a06686569676874120132120b0a06616d6f756e741201303222 +1220b995a75e975242574ff1730feaf1e9255743e86ff9d949b04e2906ad +fd10f824230a0462616e6b1a06007374616b652213323030303030303038 +32333839303139353230300a0462616e6b1a1b021493354845030274cd4b +f1686abd60ab28ec52e1a77374616b65220b383233383930313935323025 +0a0462616e6b10011a1b0214dc6f17bbec824fff8f86587966b2047db6ab +73677374616b65250a0462616e6b10011a1b0214f1829676db577682e944 +fc3493d451b67ff3e29f7374616b65270a0462616e6b1a1c037374616b65 +001493354845030274cd4bf1686abd60ab28ec52e1a7220100260a046261 +6e6b10011a1c037374616b650014dc6f17bbec824fff8f86587966b2047d +b6ab7367260a0462616e6b10011a1c037374616b650014f1829676db5776 +82e944fc3493d451b67ff3e29f3a0a0c646973747269627574696f6e1a01 +0022270a250a057374616b65121c31363437373830333930343030303030 +303030303030303030303030290a0c646973747269627574696f6e1a0101 +22160a14ae7ccd8d599769209074b81d0c2f3f28624742b7500a0c646973 +747269627574696f6e1a16021438b4089f7762e0c20e9bed8e991e074550 +ef5a5222280a260a057374616b65121d3338333130383934303736383030 +303030303030303030303030303030500a0c646973747269627574696f6e +1a16021457f96e6b86cdefdb3d412547816a82e3e0ebf9d222280a260a05 +7374616b65121d3432343330333435303532383030303030303030303030 +303030303030520a0c646973747269627574696f6e1a16061438b4089f77 +62e0c20e9bed8e991e074550ef5a52222a0a260a057374616b65121d3334 +343739383034363639313230303030303030303030303030303030100252 +0a0c646973747269627574696f6e1a16061457f96e6b86cdefdb3d412547 +816a82e3e0ebf9d2222a0a260a057374616b65121d333831383733313035 +343735323030303030303030303030303030303010024f0a0c6469737472 +69627574696f6e1a16071438b4089f7762e0c20e9bed8e991e074550ef5a +5222270a250a057374616b65121c33383331303839343037363830303030 +3030303030303030303030304f0a0c646973747269627574696f6e1a1607 +1457f96e6b86cdefdb3d412547816a82e3e0ebf9d222270a250a05737461 +6b65121c3432343330333435303532383030303030303030303030303030 +3030180a096665656d61726b65741a010122080000000000000000450a04 +6d696e741a0100223a0a1231323939393939373937313031363533303112 +243235393939393936343737353631363138383736303138323436303338 +333338383834332a0a06706172616d731a116665656d61726b65742f4261 +7365466565220d223736353632353030303030225b0a08736c617368696e +671a1601142861b93c776d100695688250c5b1ba4d44ef2a7922370a3163 +726376616c636f6e73313970736d6a307268643567716439746773666776 +7476643666347a7737326e656674666d6730180122005b0a08736c617368 +696e671a160114ae7ccd8d599769209074b81d0c2f3f28624742b722370a +3163726376616c636f6e7331346537766d7232656a61356a707972356871 +77736374656c397033797773346874333468797a18012200cd070a077374 +616b696e671a02503222bd070a92030a02080b120c63726f6e6f735f3737 +372d311802220c08d9a39a9a0610b8d89eca022a480a202d8f46f6115269 +6c6dc80c6367c7ddc926dc3ed7ab23bd37fb7ca83eb00ba2071224080112 +20df770b6cb0a9694f550ae515d5fb0ebc08902e0fa47ceec816928e74fd +8fee44322004943bc3709104b52f30f83ca3d30d8bf57551edd48797aef5 +cee50bb532c15c3a20e3b0c44298fc1c149afbf4c8996fb92427ae41e464 +9b934ca495991b7852b8554220f537a6e0561fa0edd12b30ec9b6479e659 +f6f1fa1587e69556201bfaf4cf97404a20f537a6e0561fa0edd12b30ec9b +6479e659f6f1fa1587e69556201bfaf4cf97405220252fe7cf36dd1bb85d +afc47a08961df0cfd8c027defa5e01e958be121599db9d5a209048462db5 +2bb8096c92f8733fcf26455a000a550d9f3748dddadcc2267917146220e3 +b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8 +556a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca49599 +1b7852b8557214ae7ccd8d599769209074b81d0c2f3f28624742b7129102 +0a3163726376616c6f70657231326c756b75367578656868616b30327079 +3472637a36357a753073776837776a36756c726c6712430a1d2f636f736d +6f732e63727970746f2e656432353531392e5075624b657912220a209b86 +839433f4229b7b7b51554a25ed32549126eb80fe53497ae336a1e67b6684 +20032a133130303030303030303030303030303030303032253130303030 +303030303030303030303030303030303030303030303030303030303030 +30303a070a056e6f6465304a00524b0a3b0a123130303030303030303030 +3030303030303012123230303030303030303030303030303030301a1131 +30303030303030303030303030303030120c08d1a39a9a0610a89eb0a602 +5a01311291020a3163726376616c6f70657231387a367133386d68767473 +767972356d616b38666a38733867346777376b6a6a70306530646812430a +1d2f636f736d6f732e63727970746f2e656432353531392e5075624b6579 +12220a2072b50cf0ed1863ffc937af99b6ad779a2c223e59459eab7768bd +a7c2da6f836e20032a133130303030303030303030303030303030303032 +253130303030303030303030303030303030303030303030303030303030 +30303030303030303a070a056e6f6465314a00524b0a3b0a123130303030 +303030303030303030303030301212323030303030303030303030303030 +3030301a113130303030303030303030303030303030120c08d1a39a9a06 +10a89eb0a6025a0131 +` + +func TestReadFileStreamer(t *testing.T) { + buf, err := hex.DecodeString(strings.Replace(data, "\n", "", -1)) + require.NoError(t, err) + + changeSet, err := ReadFileStreamer(bufio.NewReader(bytes.NewReader(buf))) + require.NoError(t, err) + + require.Equal(t, 21, len(changeSet)) + expItem := types.StoreKVPair{StoreKey: "bank", Delete: false, Key: []uint8{0x0, 0x73, 0x74, 0x61, 0x6b, 0x65}, Value: []uint8{0x32, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x38, 0x32, 0x33, 0x38, 0x39, 0x30, 0x31, 0x39, 0x35, 0x32, 0x30}} + require.Equal(t, expItem, changeSet[0]) +} diff --git a/versiondb/tmdb/history.go b/versiondb/tmdb/history.go new file mode 100644 index 0000000000..0b0438b3ec --- /dev/null +++ b/versiondb/tmdb/history.go @@ -0,0 +1,67 @@ +package tmdb + +import ( + "bytes" + + "github.com/RoaringBitmap/roaring/roaring64" + + "github.com/crypto-org-chain/cronos/versiondb" + dbm "github.com/tendermint/tm-db" +) + +// GetHistoryIndex returns the history index bitmap. +func GetHistoryIndex(db dbm.DB, key []byte) (*roaring64.Bitmap, error) { + // try to seek the first chunk whose maximum is bigger or equal to the target height. + bz, err := db.Get(key) + if err != nil { + return nil, err + } + if len(bz) == 0 { + return nil, nil + } + m := roaring64.New() + _, err = m.ReadFrom(bytes.NewReader(bz)) + if err != nil { + return nil, err + } + return m, nil +} + +// SeekHistoryIndex locate the minimal version that changed the key and is larger than the target version, +// using the returned version can find the value for the target version in changeset table. +// If not found, return -1 +func SeekHistoryIndex(db dbm.DB, key []byte, version uint64) (int64, error) { + m, err := GetHistoryIndex(db, key) + if err != nil { + return -1, err + } + found, ok := versiondb.SeekInBitmap64(m, version+1) + if !ok { + return -1, nil + } + return int64(found), nil +} + +// WriteHistoryIndex set the block height to the history bitmap. +// it try to set to the last chunk, if the last chunk exceeds chunk limit, split it. +func WriteHistoryIndex(db dbm.DB, batch dbm.Batch, key []byte, height uint64) error { + bz, err := db.Get(key) + if err != nil { + return err + } + + m := roaring64.New() + if len(bz) > 0 { + _, err = m.ReadFrom(bytes.NewReader(bz)) + if err != nil { + return err + } + } + m.Add(height) + m.RunOptimize() + bz, err = m.ToBytes() + if err != nil { + return err + } + return batch.Set(key, bz) +} diff --git a/versiondb/tmdb/iterator.go b/versiondb/tmdb/iterator.go new file mode 100644 index 0000000000..bdbc9a6e6d --- /dev/null +++ b/versiondb/tmdb/iterator.go @@ -0,0 +1,183 @@ +package tmdb + +import ( + "bytes" + + "github.com/RoaringBitmap/roaring/roaring64" + "github.com/cosmos/cosmos-sdk/store/types" + "github.com/crypto-org-chain/cronos/versiondb" + dbm "github.com/tendermint/tm-db" +) + +type Iterator struct { + storeKey string + version int64 + + start, end []byte + + plain, history types.Iterator + changesetDB dbm.DB + + key, value []byte + + reverse bool + status int + err error +} + +var _ types.Iterator = (*Iterator)(nil) + +func NewIterator(storeKey string, version int64, plainDB, historyDB types.KVStore, changesetDB dbm.DB, start, end []byte, reverse bool) (types.Iterator, error) { + var plain, history types.Iterator + + if reverse { + plain = plainDB.ReverseIterator(start, end) + } else { + plain = plainDB.Iterator(start, end) + } + + if reverse { + history = historyDB.ReverseIterator(start, end) + } else { + history = historyDB.Iterator(start, end) + } + iter := &Iterator{ + storeKey: storeKey, version: version, + reverse: reverse, + start: start, end: end, + plain: plain, history: history, + changesetDB: changesetDB, + } + iter.err = iter.resolve() + return iter, nil +} + +// Domain implements types.Iterator. +func (iter *Iterator) Domain() ([]byte, []byte) { + return iter.start, iter.end +} + +func (iter *Iterator) Valid() bool { + return iter.err == nil && len(iter.key) > 0 +} + +func (iter *Iterator) Next() { + switch iter.status { + case -2: + return + case 0: + iter.plain.Next() + iter.history.Next() + case 1: + iter.history.Next() + case -1: + iter.plain.Next() + } + iter.err = iter.resolve() +} + +func (iter *Iterator) Key() []byte { + return iter.key +} + +func (iter *Iterator) Value() []byte { + return iter.value +} + +func (iter *Iterator) Close() error { + err1 := iter.plain.Close() + err2 := iter.history.Close() + if err1 != nil { + return err1 + } + if err2 != nil { + return err2 + } + return nil +} + +func (iter *Iterator) Error() error { + return iter.err +} + +func (iter *Iterator) getFromHistory(key []byte, bz []byte, getLatestValue func() []byte) ([]byte, error) { + m := roaring64.New() + _, err := m.ReadFrom(bytes.NewReader(bz)) + if err != nil { + return nil, err + } + found, ok := versiondb.SeekInBitmap64(m, uint64(iter.version)+1) + if !ok { + // not changed, use the latest one + return getLatestValue(), nil + } + changesetKey := ChangesetKey(found, prependStoreKey(iter.storeKey, key)) + return iter.changesetDB.Get(changesetKey) +} + +func (iter *Iterator) resolve() (err error) { + for { + var pkey, hkey []byte + if iter.plain.Valid() { + pkey = iter.plain.Key() + } + if iter.history.Valid() { + hkey = iter.history.Key() + } + + iter.status = compareKey(pkey, hkey, iter.reverse) + switch iter.status { + case -2: + // end of iteration + iter.key = nil + iter.value = nil + return nil + case 0: + // find the historial value, or fallback to latest one. + iter.key = hkey + iter.value, err = iter.getFromHistory(hkey, iter.history.Value(), func() []byte { + return iter.plain.Value() + }) + if len(iter.value) > 0 { + return + } + iter.plain.Next() + iter.history.Next() + case 1: + // plain state exhausted or history cursor lag behind + // the key is deleted in plain state, use the history state. + iter.key = hkey + iter.value, err = iter.getFromHistory(hkey, iter.history.Value(), func() []byte { + return nil + }) + if len(iter.value) > 0 { + return + } + iter.history.Next() + case -1: + // history state exhausted or plain cursor lag behind + // the key don't exist in history state, use the plain state value. + iter.key = pkey + iter.value = iter.plain.Value() + return + } + } +} + +// compareKey is similar to bytes.Compare, but it treat empty slice as biggest value. +func compareKey(k1, k2 []byte, reverse bool) int { + switch { + case len(k1) == 0 && len(k2) == 0: + return -2 + case len(k1) == 0: + return 1 + case len(k2) == 0: + return -1 + default: + result := bytes.Compare(k1, k2) + if reverse { + result = -result + } + return result + } +} diff --git a/versiondb/tmdb/store.go b/versiondb/tmdb/store.go new file mode 100644 index 0000000000..d86b2eddbe --- /dev/null +++ b/versiondb/tmdb/store.go @@ -0,0 +1,266 @@ +package tmdb + +import ( + "bytes" + "errors" + "fmt" + + "github.com/cosmos/cosmos-sdk/store/dbadapter" + "github.com/cosmos/cosmos-sdk/store/prefix" + "github.com/cosmos/cosmos-sdk/store/types" + sdk "github.com/cosmos/cosmos-sdk/types" + gogotypes "github.com/cosmos/gogoproto/types" + "github.com/crypto-org-chain/cronos/versiondb" + dbm "github.com/tendermint/tm-db" +) + +const latestVersionKey = "s/latest" + +var _ versiondb.VersionStore = (*Store)(nil) + +// Store implements `VersionStore`. +type Store struct { + // latest key-value pairs + plainDB dbm.DB + // history bitmap index of keys + historyDB dbm.DB + // changesets of each blocks + changesetDB dbm.DB +} + +func NewStore(plainDB, historyDB, changesetDB dbm.DB) *Store { + return &Store{plainDB, historyDB, changesetDB} +} + +// PutAtVersion implements VersionStore interface +// TODO reduce allocation within iterations. +func (s *Store) PutAtVersion(version int64, changeSet []types.StoreKVPair) error { + plainBatch := s.plainDB.NewBatch() + defer plainBatch.Close() + historyBatch := s.historyDB.NewBatch() + defer historyBatch.Close() + changesetBatch := s.changesetDB.NewBatch() + defer changesetBatch.Close() + + for _, pair := range changeSet { + key := prependStoreKey(pair.StoreKey, pair.Key) + + if version == 0 { + // genesis state is written into plain state directly + if pair.Delete { + return errors.New("can't delete at genesis") + } + if err := plainBatch.Set(key, pair.Value); err != nil { + return err + } + continue + } + + original, err := s.plainDB.Get(key) + if err != nil { + return err + } + if bytes.Equal(original, pair.Value) { + // do nothing if the value is not changed + continue + } + + // write history index + if err := WriteHistoryIndex(s.historyDB, historyBatch, key, uint64(version)); err != nil { + return err + } + + // write the old value to changeset + if len(original) > 0 { + changesetKey := append(sdk.Uint64ToBigEndian(uint64(version)), key...) + if err := changesetBatch.Set(changesetKey, original); err != nil { + return err + } + } + + // write the new value to plain state + if pair.Delete { + if err := plainBatch.Delete(key); err != nil { + return err + } + } else { + if err := plainBatch.Set(key, pair.Value); err != nil { + return err + } + } + } + + // write latest version to plain state + if err := s.setLatestVersion(plainBatch, version); err != nil { + return err + } + + if err := changesetBatch.WriteSync(); err != nil { + return err + } + if err := historyBatch.WriteSync(); err != nil { + return err + } + return plainBatch.WriteSync() +} + +// GetAtVersion implements VersionStore interface +func (s *Store) GetAtVersion(storeKey string, key []byte, version *int64) ([]byte, error) { + rawKey := prependStoreKey(storeKey, key) + if version == nil { + return s.plainDB.Get(rawKey) + } + + height := *version + + // optimize for latest version + latest, err := s.GetLatestVersion() + if err != nil { + return nil, err + } + if height > latest { + return nil, fmt.Errorf("height %d is in the future", height) + } + if latest == height { + return s.plainDB.Get(rawKey) + } + + found, err := SeekHistoryIndex(s.historyDB, rawKey, uint64(height)) + if err != nil { + return nil, err + } + if found < 0 { + // there's no change records found after the target version, query the latest state. + return s.plainDB.Get(rawKey) + } + // get from changeset + changesetKey := ChangesetKey(uint64(found), rawKey) + return s.changesetDB.Get(changesetKey) +} + +// HasAtVersion implements VersionStore interface +func (s *Store) HasAtVersion(storeKey string, key []byte, version *int64) (bool, error) { + rawKey := prependStoreKey(storeKey, key) + if version == nil { + return s.plainDB.Has(rawKey) + } + + height := *version + + // optimize for latest version + latest, err := s.GetLatestVersion() + if err != nil { + return false, err + } + if height > latest { + return false, fmt.Errorf("height %d is in the future", height) + } + if latest == height { + return s.plainDB.Has(rawKey) + } + + found, err := SeekHistoryIndex(s.historyDB, rawKey, uint64(height)) + if err != nil { + return false, err + } + if found < 0 { + // there's no change records after the target version, query the latest state. + return s.plainDB.Has(rawKey) + } + // get from changeset + changesetKey := ChangesetKey(uint64(found), rawKey) + return s.changesetDB.Has(changesetKey) +} + +// IteratorAtVersion implements VersionStore interface +func (s *Store) IteratorAtVersion(storeKey string, start, end []byte, version *int64) (types.Iterator, error) { + storePrefix := StoreKeyPrefix(storeKey) + prefixPlain := prefix.NewStore(dbadapter.Store{DB: s.plainDB}, storePrefix) + if version == nil { + return prefixPlain.Iterator(start, end), nil + } + + // optimize for latest version + height := *version + latest, err := s.GetLatestVersion() + if err != nil { + return nil, err + } + if height > latest { + return nil, fmt.Errorf("height %d is in the future", height) + } + if latest == height { + return prefixPlain.Iterator(start, end), nil + } + + prefixHistory := prefix.NewStore(dbadapter.Store{DB: s.historyDB}, storePrefix) + return NewIterator(storeKey, height, prefixPlain, prefixHistory, s.changesetDB, start, end, false) +} + +// ReverseIteratorAtVersion implements VersionStore interface +func (s *Store) ReverseIteratorAtVersion(storeKey string, start, end []byte, version *int64) (types.Iterator, error) { + storePrefix := StoreKeyPrefix(storeKey) + prefixPlain := prefix.NewStore(dbadapter.Store{DB: s.plainDB}, storePrefix) + if version == nil { + return prefixPlain.ReverseIterator(start, end), nil + } + + // optimize for latest version + height := *version + latest, err := s.GetLatestVersion() + if err != nil { + return nil, err + } + if height > latest { + return nil, fmt.Errorf("height %d is in the future", height) + } + if latest == height { + return prefixPlain.ReverseIterator(start, end), nil + } + + prefixHistory := prefix.NewStore(dbadapter.Store{DB: s.historyDB}, storePrefix) + return NewIterator(storeKey, height, prefixPlain, prefixHistory, s.changesetDB, start, end, true) +} + +// GetLatestVersion returns the latest version stored in plain state, +// it's committed after the changesets, so the data for this version is guaranteed to be persisted. +// returns -1 if the key don't exists. +func (s *Store) GetLatestVersion() (int64, error) { + bz, err := s.plainDB.Get([]byte(latestVersionKey)) + if err != nil { + return -1, err + } else if bz == nil { + return -1, nil + } + + var latestVersion int64 + + if err := gogotypes.StdInt64Unmarshal(&latestVersion, bz); err != nil { + return -1, err + } + + return latestVersion, nil +} + +func (s *Store) setLatestVersion(plainBatch dbm.Batch, version int64) error { + // write latest version to plain state + bz, err := gogotypes.StdInt64Marshal(version) + if err != nil { + return err + } + return plainBatch.Set([]byte(latestVersionKey), bz) +} + +// ChangesetKey build key changeset db +func ChangesetKey(version uint64, key []byte) []byte { + return append(sdk.Uint64ToBigEndian(version), key...) +} + +func StoreKeyPrefix(storeKey string) []byte { + return []byte("s/k:" + storeKey + "/") +} + +// prependStoreKey prepends storeKey to the key +func prependStoreKey(storeKey string, key []byte) []byte { + return append(StoreKeyPrefix(storeKey), key...) +} diff --git a/versiondb/tmdb/store_test.go b/versiondb/tmdb/store_test.go new file mode 100644 index 0000000000..24453198ff --- /dev/null +++ b/versiondb/tmdb/store_test.go @@ -0,0 +1,14 @@ +package tmdb + +import ( + "testing" + + "github.com/crypto-org-chain/cronos/versiondb" + dbm "github.com/tendermint/tm-db" +) + +func TestTMDB(t *testing.T) { + versiondb.Run(t, func() versiondb.VersionStore { + return NewStore(dbm.NewMemDB(), dbm.NewMemDB(), dbm.NewMemDB()) + }) +} diff --git a/versiondb/types.go b/versiondb/types.go new file mode 100644 index 0000000000..fab4a6a21d --- /dev/null +++ b/versiondb/types.go @@ -0,0 +1,21 @@ +package versiondb + +import ( + "github.com/cosmos/cosmos-sdk/store/types" +) + +// VersionStore is a versioned storage of a flat key-value pairs. +// it don't need to support merkle proof, so could be implemented in a much more efficient way. +// `nil` version means the latest version. +type VersionStore interface { + GetAtVersion(storeKey string, key []byte, version *int64) ([]byte, error) + HasAtVersion(storeKey string, key []byte, version *int64) (bool, error) + IteratorAtVersion(storeKey string, start, end []byte, version *int64) (types.Iterator, error) + ReverseIteratorAtVersion(storeKey string, start, end []byte, version *int64) (types.Iterator, error) + GetLatestVersion() (int64, error) + + // Persist the change set of a block, + // the `changeSet` should be ordered by (storeKey, key), + // the version should be latest version plus one. + PutAtVersion(version int64, changeSet []types.StoreKVPair) error +} From 7d1201a2af6a9d8da6ac3064497f68468d61085b Mon Sep 17 00:00:00 2001 From: HuangYi Date: Mon, 28 Nov 2022 13:26:18 +0800 Subject: [PATCH 02/15] fix streamer config --- app/app.go | 104 ++++++--------- client/flags.go | 2 +- cmd/cronosd/cmd/root.go | 4 +- go.mod | 34 ++--- go.sum | 95 ++++++++------ gomod2nix.toml | 71 +++++----- integration_tests/configs/default.jsonnet | 7 +- integration_tests/poetry.lock | 153 ++++++++++++++-------- integration_tests/pyproject.toml | 2 +- 9 files changed, 260 insertions(+), 212 deletions(-) diff --git a/app/app.go b/app/app.go index 8d423a3a44..a67cd2c8e9 100644 --- a/app/app.go +++ b/app/app.go @@ -1,12 +1,11 @@ package app import ( + "fmt" "io" "net/http" "os" "path/filepath" - "strings" - "sync" "github.com/crypto-org-chain/cronos/x/cronos" "github.com/crypto-org-chain/cronos/x/cronos/middleware" @@ -29,7 +28,7 @@ import ( "github.com/cosmos/cosmos-sdk/server/api" "github.com/cosmos/cosmos-sdk/server/config" servertypes "github.com/cosmos/cosmos-sdk/server/types" - "github.com/cosmos/cosmos-sdk/store/streaming/file" + "github.com/cosmos/cosmos-sdk/store/streaming" storetypes "github.com/cosmos/cosmos-sdk/store/types" "github.com/cosmos/cosmos-sdk/testutil/testdata" sdk "github.com/cosmos/cosmos-sdk/types" @@ -150,8 +149,6 @@ const ( // // NOTE: In the SDK, the default value is 255. AddrLen = 20 - - FileStreamerDirectory = "file_streamer" ) // this line is used by starport scaffolding # stargate/wasm/app/enabledProposals @@ -351,65 +348,50 @@ func New( tkeys := sdk.NewTransientStoreKeys(paramstypes.TStoreKey, evmtypes.TransientKey, feemarkettypes.TransientKey) memKeys := sdk.NewMemoryStoreKeys(capabilitytypes.MemStoreKey) - // configure state listening capabilities using AppOptions - // we are doing nothing with the returned streamingServices and waitGroup in this case - // Only support file streamer right now. - streamers := cast.ToString(appOpts.Get(cronosappclient.FlagStreamers)) - if strings.Contains(streamers, "file") { - streamingDir := filepath.Join(cast.ToString(appOpts.Get(flags.FlagHome)), "data", FileStreamerDirectory) - if err := os.MkdirAll(streamingDir, os.ModePerm); err != nil { - panic(err) - } - - // default to exposing all - exposeStoreKeys := make([]storetypes.StoreKey, 0, len(keys)) - for _, storeKey := range keys { - exposeStoreKeys = append(exposeStoreKeys, storeKey) - } - service, err := file.NewStreamingService(streamingDir, "", exposeStoreKeys, appCodec, false) - if err != nil { - panic(err) - } - bApp.SetStreamingService(service) - - wg := new(sync.WaitGroup) - if err := service.Stream(wg); err != nil { - panic(err) - } + // load state streaming if enabled + if _, _, err := streaming.LoadStreamingServices(bApp, appOpts, appCodec, logger, keys, homePath); err != nil { + fmt.Printf("failed to load state streaming: %s", err) + os.Exit(1) } - if strings.Contains(streamers, "versiondb") { - rootDir := cast.ToString(appOpts.Get(flags.FlagHome)) - dataDir := filepath.Join(rootDir, "data", "versiondb") - if err := os.MkdirAll(dataDir, os.ModePerm); err != nil { - panic(err) - } - backendType := server.GetAppDBBackend(appOpts) - plainDB, err := dbm.NewDB("plain", backendType, dataDir) - if err != nil { - panic(err) - } - historyDB, err := dbm.NewDB("history", backendType, dataDir) - if err != nil { - panic(err) - } - changesetDB, err := dbm.NewDB("changeset", backendType, dataDir) - if err != nil { - panic(err) - } - versionDB := tmdb.NewStore(plainDB, historyDB, changesetDB) - - // default to exposing all - exposeStoreKeys := make([]storetypes.StoreKey, 0, len(keys)) - for _, storeKey := range keys { - exposeStoreKeys = append(exposeStoreKeys, storeKey) + // configure state listening capabilities using AppOptions + // we are doing nothing with the returned streamingServices and waitGroup in this case + streamers := cast.ToStringSlice(appOpts.Get(cronosappclient.FlagStreamers)) + for _, streamerName := range streamers { + if streamerName == "versiondb" { + rootDir := cast.ToString(appOpts.Get(flags.FlagHome)) + dataDir := filepath.Join(rootDir, "data", "versiondb") + if err := os.MkdirAll(dataDir, os.ModePerm); err != nil { + panic(err) + } + backendType := server.GetAppDBBackend(appOpts) + plainDB, err := dbm.NewDB("plain", backendType, dataDir) + if err != nil { + panic(err) + } + historyDB, err := dbm.NewDB("history", backendType, dataDir) + if err != nil { + panic(err) + } + changesetDB, err := dbm.NewDB("changeset", backendType, dataDir) + if err != nil { + panic(err) + } + versionDB := tmdb.NewStore(plainDB, historyDB, changesetDB) + + // default to exposing all + exposeStoreKeys := make([]storetypes.StoreKey, 0, len(keys)) + for _, storeKey := range keys { + exposeStoreKeys = append(exposeStoreKeys, storeKey) + } + service := versiondb.NewStreamingService(versionDB, exposeStoreKeys) + bApp.SetStreamingService(service) + qms := versiondb.NewMultiStore(versionDB, exposeStoreKeys) + qms.MountTransientStores(tkeys) + qms.MountMemoryStores(memKeys) + bApp.SetQueryMultiStore(qms) + break } - service := versiondb.NewStreamingService(versionDB, exposeStoreKeys) - bApp.SetStreamingService(service) - qms := versiondb.NewMultiStore(versionDB, exposeStoreKeys) - qms.MountTransientStores(tkeys) - qms.MountMemoryStores(memKeys) - bApp.SetQueryMultiStore(qms) } app := &App{ diff --git a/client/flags.go b/client/flags.go index 96926045f6..87ea0d120c 100644 --- a/client/flags.go +++ b/client/flags.go @@ -1,3 +1,3 @@ package client -const FlagStreamers = "streamers" +const FlagStreamers = "store.streamers" diff --git a/cmd/cronosd/cmd/root.go b/cmd/cronosd/cmd/root.go index 39f88d7486..b65230df01 100644 --- a/cmd/cronosd/cmd/root.go +++ b/cmd/cronosd/cmd/root.go @@ -45,7 +45,6 @@ import ( ethermint "github.com/evmos/ethermint/types" "github.com/crypto-org-chain/cronos/app" - cronosclient "github.com/crypto-org-chain/cronos/client" // this line is used by starport scaffolding # stargate/root/import ) @@ -148,7 +147,6 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig params.EncodingConfig) { func addModuleInitFlags(startCmd *cobra.Command) { crisis.AddModuleInitFlags(startCmd) cronos.AddModuleInitFlags(startCmd) - startCmd.Flags().String(cronosclient.FlagStreamers, "", "Enable streamers, only file streamer is supported right now") // this line is used by starport scaffolding # stargate/root/initFlags } @@ -269,7 +267,7 @@ func (a appCreator) newApp(logger log.Logger, db dbm.DB, traceStore io.Writer, a baseapp.SetIndexEvents(cast.ToStringSlice(appOpts.Get(server.FlagIndexEvents))), baseapp.SetSnapshot(snapshotStore, snapshotOptions), baseapp.SetIAVLCacheSize(cast.ToInt(appOpts.Get(server.FlagIAVLCacheSize))), - baseapp.SetIAVLDisableFastNode(cast.ToBool(appOpts.Get(server.FlagIAVLFastNode))), + baseapp.SetIAVLDisableFastNode(cast.ToBool(appOpts.Get(server.FlagDisableIAVLFastNode))), ) } diff --git a/go.mod b/go.mod index 40fa2911d7..a72ea75ed9 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/RoaringBitmap/roaring v1.2.1 github.com/armon/go-metrics v0.4.1 github.com/cosmos/cosmos-sdk v0.46.3 + github.com/cosmos/gogoproto v1.4.3 github.com/cosmos/ibc-go/v5 v5.0.0 github.com/ethereum/go-ethereum v1.10.19 github.com/evmos/ethermint v0.6.1-0.20221003153722-491c3da7ebd7 @@ -17,14 +18,14 @@ require ( github.com/peggyjv/gravity-bridge/module/v2 v2.0.0-20220420162017-838c0d25e974 github.com/rakyll/statik v0.1.7 github.com/spf13/cast v1.5.0 - github.com/spf13/cobra v1.5.0 + github.com/spf13/cobra v1.6.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.0 - github.com/tendermint/tendermint v0.34.22 + github.com/tendermint/tendermint v0.34.24 github.com/tendermint/tm-db v0.6.7 - google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc - google.golang.org/grpc v1.50.0 - google.golang.org/protobuf v1.28.1 + google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a + google.golang.org/grpc v1.50.1 + google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8 gopkg.in/yaml.v2 v2.4.0 ) @@ -63,7 +64,7 @@ require ( github.com/cosmos/cosmos-proto v1.0.0-alpha7 // indirect github.com/cosmos/go-bip39 v1.0.0 // indirect github.com/cosmos/gorocksdb v1.2.0 // indirect - github.com/cosmos/iavl v0.19.3 // indirect + github.com/cosmos/iavl v0.19.4 // indirect github.com/cosmos/ledger-cosmos-go v0.11.1 // indirect github.com/cosmos/ledger-go v0.9.2 // indirect github.com/creachadair/taskgroup v0.3.2 // indirect @@ -95,7 +96,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.2 // indirect github.com/google/flatbuffers v2.0.0+incompatible // indirect - github.com/google/go-cmp v0.5.8 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/google/orderedcode v0.0.1 // indirect github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect @@ -124,7 +125,7 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmhodges/levigo v1.0.0 // indirect - github.com/klauspost/compress v1.15.9 // indirect + github.com/klauspost/compress v1.15.11 // indirect github.com/lib/pq v1.10.6 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/magiconair/properties v1.8.6 // indirect @@ -140,7 +141,6 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/mtibben/percent v0.2.1 // indirect - github.com/natefinch/atomic v1.0.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.5 // indirect @@ -150,7 +150,7 @@ require ( github.com/prometheus/client_golang v1.12.2 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.34.0 // indirect - github.com/prometheus/procfs v0.7.3 // indirect + github.com/prometheus/procfs v0.8.0 // indirect github.com/prometheus/tsdb v0.7.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/regen-network/cosmos-proto v0.3.1 // indirect @@ -176,14 +176,14 @@ require ( github.com/zondax/hid v0.9.1-0.20220302062450-5552068d2266 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.opencensus.io v0.23.0 // indirect - golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503 // indirect + golang.org/x/crypto v0.1.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect - golang.org/x/net v0.0.0-20220812174116-3211cb980234 // indirect + golang.org/x/net v0.1.0 // indirect golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2 // indirect - golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect - golang.org/x/sys v0.0.0-20220818161305-2296e01440c6 // indirect - golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0 // indirect + golang.org/x/sys v0.1.0 // indirect + golang.org/x/term v0.1.0 // indirect + golang.org/x/text v0.4.0 // indirect golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect google.golang.org/api v0.93.0 // indirect google.golang.org/appengine v1.6.7 // indirect @@ -195,7 +195,7 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/yihuang/cosmos-sdk v0.43.0-beta1.0.20221014023203-2c6b9d06b12d + github.com/cosmos/cosmos-sdk => github.com/yihuang/cosmos-sdk v0.43.0-beta1.0.20221128050842-6df48c3ab38b github.com/ethereum/go-ethereum => github.com/ethereum/go-ethereum v1.10.19 // Fix upstream GHSA-h395-qcrw-5vmq vulnerability. diff --git a/go.sum b/go.sum index 1e5eb4568d..bc3497abfe 100644 --- a/go.sum +++ b/go.sum @@ -91,10 +91,12 @@ github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= +github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/RoaringBitmap/roaring v1.2.1 h1:58/LJlg/81wfEHd5L9qsHduznOIhyv4qb1yWcSvVq9A= +github.com/RoaringBitmap/roaring v1.2.1/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= @@ -154,6 +156,8 @@ github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1U github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d/go.mod h1:d3C0AkH6BRcvO8T0UEPu53cnw4IbV63x1bEjildYhO0= @@ -182,6 +186,8 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= +github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b h1:6+ZFm0flnudZzdSE0JxlhR2hKnGPcNB35BjQf4RYQDY= +github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= @@ -241,15 +247,15 @@ github.com/cosmos/btcutil v1.0.4 h1:n7C2ngKXo7UC9gNyMNLbzqz7Asuf+7Qv4gnX/rOdQ44= github.com/cosmos/btcutil v1.0.4/go.mod h1:Ffqc8Hn6TJUdDgHBwIZLtrLQC1KdJ9jGJl/TvgUaxbU= github.com/cosmos/cosmos-proto v1.0.0-alpha7 h1:yqYUOHF2jopwZh4dVQp3xgqwftE5/2hkrwIV6vkUbO0= github.com/cosmos/cosmos-proto v1.0.0-alpha7/go.mod h1:dosO4pSAbJF8zWCzCoTWP7nNsjcvSUBQmniFxDg5daw= -github.com/cosmos/cosmos-sdk v0.46.2 h1:3dUNqbLas94ud5aTcJKCwxVOmNXpuGBtVQTMrYczTwY= -github.com/cosmos/cosmos-sdk v0.46.2/go.mod h1:0aUPGPU6PWaDEaHNjtgrpNhgxo9bAUrQ7BO7XCvFOfs= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY= github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw= +github.com/cosmos/gogoproto v1.4.3 h1:RP3yyVREh9snv/lsOvmsAPQt8f44LgL281X0IOIhhcI= +github.com/cosmos/gogoproto v1.4.3/go.mod h1:0hLIG5TR7IvV1fme1HCFKjfzW9X2x0Mo+RooWXCnOWU= github.com/cosmos/gorocksdb v1.2.0 h1:d0l3jJG8M4hBouIZq0mDUHZ+zjOx044J3nGRskwTb4Y= github.com/cosmos/gorocksdb v1.2.0/go.mod h1:aaKvKItm514hKfNJpUJXnnOWeBnk2GL4+Qw9NHizILw= -github.com/cosmos/iavl v0.19.3 h1:cESO0OwTTxQm5rmyESKW+zESheDUYI7CcZDWWDwnuxg= -github.com/cosmos/iavl v0.19.3/go.mod h1:X9PKD3J0iFxdmgNLa7b2LYWdsGd90ToV5cAONApkEPw= +github.com/cosmos/iavl v0.19.4 h1:t82sN+Y0WeqxDLJRSpNd8YFX5URIrT+p8n6oJbJ2Dok= +github.com/cosmos/iavl v0.19.4/go.mod h1:X9PKD3J0iFxdmgNLa7b2LYWdsGd90ToV5cAONApkEPw= github.com/cosmos/ibc-go/v5 v5.0.0 h1:MkObdarpoICPHXoRg/Ne9NRix4j7eQlJZq74/uzH3Zc= github.com/cosmos/ibc-go/v5 v5.0.0/go.mod h1:Wqsguq98Iuns8tgTv8+xaGYbC+Q8zJfbpjzT6IgMJbs= github.com/cosmos/ledger-cosmos-go v0.11.1 h1:9JIYsGnXP613pb2vPjFeMMjBI5lEDsEaF6oYorTy6J4= @@ -284,8 +290,9 @@ github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= -github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= +github.com/dgraph-io/badger/v3 v3.2103.2 h1:dpyM5eCJAtQCBcMCZcT4UBZchuTJgCywerHHgmxfxM8= +github.com/dgraph-io/badger/v3 v3.2103.2/go.mod h1:RHo4/GmYcKKh5Lxu63wLEMHJ70Pac2JqZRYGhlyAo2M= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= @@ -302,7 +309,7 @@ github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/ github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/docker v1.6.2/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/dop251/goja v0.0.0-20220405120441-9037c2b61cbf h1:Yt+4K30SdjOkRoRRm3vYNQgR+/ZIy0RmeUDZo7Y8zeQ= github.com/dop251/goja v0.0.0-20220405120441-9037c2b61cbf/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= @@ -463,9 +470,12 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v2.0.0+incompatible h1:dicJ2oXwypfwUGnB2/TYWYEKiuk9eYQlQO/AnOHl5mI= +github.com/google/flatbuffers v2.0.0+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -479,8 +489,9 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa h1:Q75Upo5UN4JbPFURXZ8nLKYUvF85dyFRop/vQ0Rv+64= @@ -673,8 +684,8 @@ github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.11.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= +github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= @@ -771,6 +782,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -810,7 +823,7 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.20.2 h1:8uQq0zMgLEfa0vRrrBgaJF2gyW9Da9BmfGV+OyUzfkY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec= +github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= github.com/opencontainers/runc v1.1.3 h1:vIXrkId+0/J2Ymu2m7VjGvbSlAId9XNRPhn2p4b+d8w= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= @@ -888,8 +901,9 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= @@ -949,8 +963,8 @@ github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= -github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= +github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= +github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -992,10 +1006,8 @@ github.com/tendermint/crypto v0.0.0-20191022145703-50d29ede1e15 h1:hqAk8riJvK4RM github.com/tendermint/crypto v0.0.0-20191022145703-50d29ede1e15/go.mod h1:z4YtwM70uOnk8h0pjJYlj3zdYwi9l03By6iAIF5j/Pk= github.com/tendermint/go-amino v0.16.0 h1:GyhmgQKvqF82e2oZeuMSp9JTN0N09emoSZlb2lyGa2E= github.com/tendermint/go-amino v0.16.0/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME= -github.com/tendermint/tendermint v0.34.22 h1:XMhtC8s8QqJO4l/dn+TkQvevTRSow3Vixjclr41o+2Q= -github.com/tendermint/tendermint v0.34.22/go.mod h1:YpP5vBEAKUT4g6oyfjKgFeZmdB/GjkJAxfF+cgmJg6Y= -github.com/tendermint/tm-db v0.6.7 h1:fE00Cbl0jayAoqlExN6oyQJ7fR/ZtoVOmvPJ//+shu8= -github.com/tendermint/tm-db v0.6.7/go.mod h1:byQDzFkZV1syXr/ReXS808NxA2xvyuuVgXOJ/088L6I= +github.com/tendermint/tendermint v0.34.24 h1:879MKKJWYYPJEMMKME+DWUTY4V9f/FBpnZDI82ky+4k= +github.com/tendermint/tendermint v0.34.24/go.mod h1:rXVrl4OYzmIa1I91av3iLv2HS0fGSiucyW9J4aMTpKI= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= @@ -1036,6 +1048,12 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= +github.com/yihuang/cosmos-sdk v0.43.0-beta1.0.20221128050842-6df48c3ab38b h1:h4gWiEuVB9B8PfY3KL5lwdtObndvcIwti0MOFkEtaqI= +github.com/yihuang/cosmos-sdk v0.43.0-beta1.0.20221128050842-6df48c3ab38b/go.mod h1:B2j/SQkKvs/hUbXTmKxGwBnShJHvlxtVEbvTYNPwfrU= +github.com/yihuang/mdbx-go v0.0.0-20221010042614-b72b4f091d88 h1:RqQjImUPy0joQDaP/Xv8+YXrHYqfR3fJpbKLBchVddY= +github.com/yihuang/mdbx-go v0.0.0-20221010042614-b72b4f091d88/go.mod h1:T2fsoJDVppxfAPTLd1svUgH1kpPmeXdPESmroSHcL1E= +github.com/yihuang/tm-db v0.0.0-20221006023748-f6214ae9454d h1:+HeYUyE9B0Zg/eYVj9LIZfriP7B71akpRvCmv1oIIbw= +github.com/yihuang/tm-db v0.0.0-20221006023748-f6214ae9454d/go.mod h1:a8QLaVn+zo5zgX2qMrNSETPO/8uBRPxvLsqL+urelEU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1091,8 +1109,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503 h1:vJ2V3lFLg+bBhgroYuRfyN583UzVveQmIXjc8T/y3to= -golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1138,6 +1156,7 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1200,8 +1219,8 @@ golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220812174116-3211cb980234 h1:RDqmgfe7SvlMWoqC3xwQ2blLO3fcWcxMa3eBLRdRW7E= -golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1236,8 +1255,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde h1:ejfdSekXMDxDLbRrJMwUk6KnSLZ2McaUCVcIKM+N6jc= -golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0 h1:cu5kTvlzcw1Q5S9f5ip1/cpiB4nXvw1XYzFPGgzLUOY= +golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1341,13 +1360,13 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220818161305-2296e01440c6 h1:Sx/u41w+OwrInGdEckYmEuU5gHoGSL4QbDz3S9s6j4U= -golang.org/x/sys v0.0.0-20220818161305-2296e01440c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 h1:Q5284mrmYTpACcm+eAKjKJH48BBwSyfJqmmGDTtT8Vc= -golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1356,8 +1375,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1430,6 +1450,7 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1584,8 +1605,8 @@ google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljW google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc h1:Nf+EdcTLHR8qDNN/KfkQL0u0ssxt9OhbaWCl5C0ucEI= -google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a h1:GH6UPn3ixhWcKDhpnEC55S75cerLPdpp3hrhfKYjZgw= +google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1624,8 +1645,8 @@ google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11 google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.50.0 h1:fPVVDxY9w++VjTZsYvXWqEf9Rqar/e+9zYfxKK+W+YU= -google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY= +google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1641,8 +1662,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8 h1:KR8+MyP7/qOlV+8Af01LtjL04bu7on42eVsxT4EyBQk= +google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/gomod2nix.toml b/gomod2nix.toml index b98b5ed14b..a99a296035 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -107,21 +107,21 @@ schema = 3 version = "v1.0.0-alpha7" hash = "sha256-2wCH+toTF2A6MfFjOa13muEH5oBCcxAhZEqirNOrBA0=" [mod."github.com/cosmos/cosmos-sdk"] - version = "v0.43.0-beta1.0.20221014023203-2c6b9d06b12d" - hash = "sha256-t/QEOJgATr82KFXes+AzNlNL6w2tGHfiwZAE4PdO5GE=" + version = "v0.43.0-beta1.0.20221128050842-6df48c3ab38b" + hash = "sha256-SxpPaUvHOZ5TYN5VnjwAROwXHqPwNYBdst2Rfs4TaKU=" replaced = "github.com/yihuang/cosmos-sdk" [mod."github.com/cosmos/go-bip39"] version = "v1.0.0" hash = "sha256-Qm2aC2vaS8tjtMUbHmlBSagOSqbduEEDwc51qvQaBmA=" [mod."github.com/cosmos/gogoproto"] - version = "v1.4.2" - hash = "sha256-hOY+mhPDYWcSYSdth2AW7IONdgicqQir0z/1XrXt9NY=" + version = "v1.4.3" + hash = "sha256-Y/NL76ay/oAl8mS3skkK5ula0/xudqbwW1o22lZjKRg=" [mod."github.com/cosmos/gorocksdb"] version = "v1.2.0" hash = "sha256-209TcVuXc5s/TcOvNlaQ1HEJAUDTEK3nxPhs+d8TEcY=" [mod."github.com/cosmos/iavl"] - version = "v0.19.3" - hash = "sha256-rmW2KoKmm4YXmYIYE1vXMOCaDyP1ym0qsW224PYc9rg=" + version = "v0.19.4" + hash = "sha256-EmpRZ48pjPFq/fIHneut9Vyo5QJATfb3ZO7KzWnqs9g=" [mod."github.com/cosmos/ibc-go/v5"] version = "v5.0.0" hash = "sha256-sDZdmuGohaaBF7bxrjo9PWJnmoF+VOkjySYhsFixPz4=" @@ -233,8 +233,8 @@ schema = 3 version = "v2.0.0+incompatible" hash = "sha256-4Db9FdOL60Da4H1+K4Qv02w4omxdsh3uzpmY1vtqHeA=" [mod."github.com/google/go-cmp"] - version = "v0.5.8" - hash = "sha256-8zkIo+Sr1NXMnj3PNmvjX2sZKnAKWXOFvmnX7D9bwxQ=" + version = "v0.5.9" + hash = "sha256-lQc4O00R3QSMGs9LP8Sy7A9kj0cqV5rrUdpnGeipIyg=" [mod."github.com/google/orderedcode"] version = "v0.0.1" hash = "sha256-KrExYovtUQrHGI1mPQf57jGw8soz7eWOC2xqEaV0uGk=" @@ -326,8 +326,8 @@ schema = 3 version = "v1.0.0" hash = "sha256-xEd0mDBeq3eR/GYeXjoTVb2sPs8sTCosn5ayWkcgENI=" [mod."github.com/klauspost/compress"] - version = "v1.15.9" - hash = "sha256-ctPxlVq0c/SoNPVsP66RjxUjTHys3diW2Apxjyc9WdE=" + version = "v1.15.11" + hash = "sha256-9MXm0TObg6DyqnYMIw3IChrorHc2ILf5djZYoM0e1J0=" [mod."github.com/lib/pq"] version = "v1.10.6" hash = "sha256-8EhFwY/9YH5L/fd6l2beOnC3VvpegRAmCCsnDVJBqBM=" @@ -373,9 +373,6 @@ schema = 3 [mod."github.com/mtibben/percent"] version = "v0.2.1" hash = "sha256-Zj1lpCP6mKQ0UUTMs2By4LC414ou+iJzKkK+eBHfEcc=" - [mod."github.com/natefinch/atomic"] - version = "v1.0.1" - hash = "sha256-fbOVHCwRNI8PFjC4o0YXpKZO0JU2aWTfH5c7WXXKMHg=" [mod."github.com/olekukonko/tablewriter"] version = "v0.0.5" hash = "sha256-/5i70IkH/qSW5KjGzv8aQNKh9tHoz98tqtL0K2DMFn4=" @@ -408,8 +405,8 @@ schema = 3 version = "v0.34.0" hash = "sha256-M+v+7DntUBmiQNzfNmG3aLLufbl0XBQOubtYoNTzJDA=" [mod."github.com/prometheus/procfs"] - version = "v0.7.3" - hash = "sha256-ik0WpnpSjMwifPYfQTfu/eb5ilNj+eLJF0d5Dftp8A8=" + version = "v0.8.0" + hash = "sha256-hgrilokQsXCOCCvwgOSfuErxoFAQpXM/+zNJKcMVHyM=" [mod."github.com/prometheus/tsdb"] version = "v0.7.1" hash = "sha256-BPz7YJbfMZgeR+u9YaeWeipVzHIS73EdgXD7VSJSLbA=" @@ -444,8 +441,8 @@ schema = 3 version = "v1.5.0" hash = "sha256-Pdp+wC5FWqyJKzyYHb7JCcV9BoJk/sxQw6nLyuLJvuQ=" [mod."github.com/spf13/cobra"] - version = "v1.5.0" - hash = "sha256-rcyHWrxshA5DVpxrSba5X4NjppqOGrJ64QkUKKnfW2E=" + version = "v1.6.0" + hash = "sha256-BidkXU9dFuU3Ah8Hl0PbuDe/EHrTr0B1JLSsdFgCXyI=" [mod."github.com/spf13/jwalterweatherman"] version = "v1.1.0" hash = "sha256-62BQtqTLF/eVrTOr7pUXE7AiHRjOVC8jQs3/Ehmflfs=" @@ -477,8 +474,8 @@ schema = 3 version = "v0.16.0" hash = "sha256-JW4zO/0vMzf1dXLePOqaMtiLUZgNbuIseh9GV+jQlf0=" [mod."github.com/tendermint/tendermint"] - version = "v0.34.22" - hash = "sha256-4p4cpyCWjBbNQUpYN2gDJvnyj+Pov9hw5uRjHrrO++Y=" + version = "v0.34.24" + hash = "sha256-3HFTv4XgN535RDaJ5OwUS+fnJHgkmLTwU7CNU2ilxEQ=" [mod."github.com/tendermint/tm-db"] version = "v0.0.0-20221006023748-f6214ae9454d" hash = "sha256-mtTVR3f3A9CmcyJBXTeurQuHGiVA5hIzqlxskz1M1qk=" @@ -510,29 +507,29 @@ schema = 3 version = "v0.23.0" hash = "sha256-R3O9GyNtv6j0ic7s+2xkLLaLzbJEop0Otj1nJDFBjsg=" [mod."golang.org/x/crypto"] - version = "v0.0.0-20220824171710-5757bc0c5503" - hash = "sha256-eGAblX40HAKw5jayFnN0SueZut7uQr7bHTzbv9YrR2g=" + version = "v0.1.0" + hash = "sha256-0oZWBSiW5Pd/2a1p2beuoelDe0CpfXZhrg/qPduJlYs=" [mod."golang.org/x/exp"] version = "v0.0.0-20220722155223-a9213eeb770e" hash = "sha256-kNgzydWRpjm0sZl4uXEs3LX5L0xjJtJRAFf/CTlYUN4=" [mod."golang.org/x/net"] - version = "v0.0.0-20220812174116-3211cb980234" - hash = "sha256-v/Qep/W6lw6IurR+R2V0AN/MlQUROLnEaWeeuwY8sg8=" + version = "v0.1.0" + hash = "sha256-SrThFBg6sqGYpiN1E3d1SilJxbKkQhSZXPmAFoMAA/I=" [mod."golang.org/x/oauth2"] version = "v0.0.0-20220622183110-fd043fe589d2" hash = "sha256-VLffpTpx3DlUzXB8mKiJfFzm4ZmgnLSUuLB5Ir0WQUg=" [mod."golang.org/x/sync"] - version = "v0.0.0-20220819030929-7fc1605a5dde" - hash = "sha256-5EOxO8FRdaLW9v/DhwBmWiT2G34A2ofxSCaC7ovvpb0=" + version = "v0.0.0-20220929204114-8fcdb60fdcc0" + hash = "sha256-Hygjq9euZ0qz6TvHYQwOZEjNiTbTh1nSLRAWZ6KFGR8=" [mod."golang.org/x/sys"] - version = "v0.0.0-20220818161305-2296e01440c6" - hash = "sha256-gCzekaAZ7aPg4EUAwL3USir4BNp/Etv5OWGekAGVZ8w=" + version = "v0.1.0" + hash = "sha256-nZbEJ/2PuWrDLD4ujeVvcFGoIsfVoIH/Lcp4FjD7hpU=" [mod."golang.org/x/term"] - version = "v0.0.0-20220722155259-a9ba230a4035" - hash = "sha256-9uM1OONzbsa6bz2iKk767hAaCuafi58bdTF7at03fWY=" + version = "v0.1.0" + hash = "sha256-UWnNsJIj5nXsuzlPWQ1NyHQuHStaDacMVkFbJ4pnxXk=" [mod."golang.org/x/text"] - version = "v0.3.7" - hash = "sha256-XH2pUzzQx95O0rak00grQvfACfL+EmZiV7ZzJBkX+XY=" + version = "v0.4.0" + hash = "sha256-JvyMygdmTvWg7xhbnUB9MMk6WcYXJt8DAj4DYl82Pys=" [mod."golang.org/x/xerrors"] version = "v0.0.0-20220609144429-65e65417b02f" hash = "sha256-tl8pv3oddbz2+KoIp7PFDKsxjQF8ocjPF8XPsY3sw38=" @@ -543,14 +540,14 @@ schema = 3 version = "v1.6.7" hash = "sha256-zIxGRHiq4QBvRqkrhMGMGCaVL4iM4TtlYpAi/hrivS4=" [mod."google.golang.org/genproto"] - version = "v0.0.0-20220822174746-9e6da59bd2fc" - hash = "sha256-yqs8RLNZJOaeT2QVhVpZdokXaapQa841VEBkh9xd/yU=" + version = "v0.0.0-20221014213838-99cd37c6964a" + hash = "sha256-zSdk2kbcfWaaJfHxLULI9v38lvEaJb8koC5c59aVZZI=" [mod."google.golang.org/grpc"] - version = "v1.50.0" - hash = "sha256-ep8UAToLpWoc0VLRGNQFzwzsuL+Yjd7emncmIOc2t8o=" + version = "v1.50.1" + hash = "sha256-38nk4qIme+fE57SsCqNxtCZnc8fyzzi4Sb60uDTT2KE=" [mod."google.golang.org/protobuf"] - version = "v1.28.1" - hash = "sha256-sTJYgvlv5is7vHNxcuigF2lNASp0QonhUgnrguhfHSU=" + version = "v1.28.2-0.20220831092852-f930b1dc76e8" + hash = "sha256-li5hXlXwTJ5LIZ8bVki1AZ6UFI2gXHl33JwdX1dOrtM=" [mod."gopkg.in/ini.v1"] version = "v1.67.0" hash = "sha256-V10ahGNGT+NLRdKUyRg1dos5RxLBXBk1xutcnquc/+4=" diff --git a/integration_tests/configs/default.jsonnet b/integration_tests/configs/default.jsonnet index 23b183f7d5..ac6cf543f2 100644 --- a/integration_tests/configs/default.jsonnet +++ b/integration_tests/configs/default.jsonnet @@ -2,14 +2,14 @@ dotenv: '../../scripts/.env', 'cronos_777-1': { cmd: 'cronosd', - 'start-flags': '--trace --streamers versiondb,file', + 'start-flags': '--trace', config: { mempool: { version: 'v1', }, }, 'app-config': { - 'app-db-backend': 'rocksdb', + // 'app-db-backend': 'rocksdb', 'minimum-gas-prices': '0basetcro', 'index-events': ['ethereum_tx.ethereumTxHash'], 'json-rpc': { @@ -20,6 +20,9 @@ 'block-range-cap': 10000, 'logs-cap': 10000, }, + store: { + streamers: ['file', 'versiondb'], + }, }, validators: [{ coins: '1000000000000000000stake,10000000000000000000000basetcro', diff --git a/integration_tests/poetry.lock b/integration_tests/poetry.lock index 81c27ee182..8e9f159796 100644 --- a/integration_tests/poetry.lock +++ b/integration_tests/poetry.lock @@ -100,7 +100,6 @@ mypy-extensions = ">=0.4.3" pathspec = ">=0.9.0" platformdirs = ">=2" tomli = {version = ">=1.1.0", markers = "python_full_version < \"3.11.0a7\""} -typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} [package.extras] colorama = ["colorama (>=0.4.3)"] @@ -473,21 +472,6 @@ category = "main" optional = false python-versions = ">=3.5" -[[package]] -name = "importlib-resources" -version = "5.7.1" -description = "Read resources from Python packages" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} - -[package.extras] -docs = ["jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx"] -testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - [[package]] name = "iniconfig" version = "1.1.1" @@ -551,7 +535,6 @@ python-versions = ">=3.7" [package.dependencies] attrs = ">=17.4.0" -importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""} pyrsistent = ">=0.14.0,<0.17.0 || >0.17.0,<0.17.1 || >0.17.1,<0.17.2 || >0.17.2" [package.extras] @@ -739,6 +722,14 @@ python-versions = ">=3.6.8" [package.extras] diagrams = ["jinja2", "railroad-diagrams"] +[[package]] +name = "pyroaring" +version = "0.3.4" +description = "Fast and lightweight set for unsigned 32 bits integers." +category = "main" +optional = false +python-versions = "*" + [[package]] name = "pyrsistent" version = "0.18.1" @@ -781,9 +772,9 @@ tomlkit = "^0.7.0" [package.source] type = "git" -url = "https://github.com/crypto-com/pystarport.git" +url = "https://github.com/yihuang/pystarport.git" reference = "main" -resolved_reference = "c3028081386e8489171607e0aed6e68d1ff87d43" +resolved_reference = "e6a75ce65a1ea07a6b402d4b6ddca291503089d9" [[package]] name = "pytest" @@ -906,6 +897,46 @@ lint = ["flake8 (==3.4.1)"] rust-backend = ["rusty-rlp (>=0.2.1,<0.3)"] test = ["hypothesis (==5.19.0)", "pytest (>=6.2.5,<7)", "tox (>=2.9.1,<3)"] +[[package]] +name = "roaring64" +version = "0.1.0" +description = "roaring64 wrapper of pyroaring implemented in pure python" +category = "main" +optional = false +python-versions = "^3.10" +develop = false + +[package.dependencies] +pyroaring = "^0.3.3" + +[package.source] +type = "git" +url = "https://github.com/yihuang/python-roaring64.git" +reference = "main" +resolved_reference = "7ceafa7beef9a0a078f2f511358f86d55550b958" + +[[package]] +name = "rocksdb" +version = "0.9.1" +description = "" +category = "main" +optional = false +python-versions = "*" +develop = false + +[package.dependencies] +setuptools = ">=25" + +[package.extras] +doc = ["sphinx", "sphinx_rtd_theme"] +test = ["pytest"] + +[package.source] +type = "git" +url = "https://github.com/HathorNetwork/python-rocksdb.git" +reference = "master" +resolved_reference = "947f68a80d97c4a5621ee681ae01602ebd883f3a" + [[package]] name = "setuptools" version = "65.5.0" @@ -994,14 +1025,6 @@ category = "main" optional = false python-versions = ">=3.5" -[[package]] -name = "typing-extensions" -version = "4.2.0" -description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" -optional = false -python-versions = ">=3.7" - [[package]] name = "urllib3" version = "1.26.9" @@ -1086,22 +1109,10 @@ python-versions = ">=3.6" idna = ">=2.0" multidict = ">=4.0" -[[package]] -name = "zipp" -version = "3.8.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx"] -testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - [metadata] lock-version = "1.1" -python-versions = "^3.8" -content-hash = "e40ace5d601ae1cbdcfa816c195b32eaed7c3267fc988a4e0219b4a5b549100b" +python-versions = "^3.10" +content-hash = "ff24ebe9c0d682d71698a957082ca483863691538895b93d2a4a8b9a652ccf29" [metadata.files] aiohttp = [ @@ -1510,10 +1521,6 @@ idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, ] -importlib-resources = [ - {file = "importlib_resources-5.7.1-py3-none-any.whl", hash = "sha256:e447dc01619b1e951286f3929be820029d48c75eb25d265c28b92a16548212b8"}, - {file = "importlib_resources-5.7.1.tar.gz", hash = "sha256:b6062987dfc51f0fcb809187cffbd60f35df7acb4589091f154214af6d0d49d3"}, -] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, @@ -1716,6 +1723,52 @@ pyparsing = [ {file = "pyparsing-3.0.8-py3-none-any.whl", hash = "sha256:ef7b523f6356f763771559412c0d7134753f037822dad1b16945b7b846f7ad06"}, {file = "pyparsing-3.0.8.tar.gz", hash = "sha256:7bf433498c016c4314268d95df76c81b842a4cb2b276fa3312cfb1e1d85f6954"}, ] +pyroaring = [ + {file = "pyroaring-0.3.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c752689fb1af66bc62380b3101426c6a357a85d73492acc5a56fbfe9dd85f571"}, + {file = "pyroaring-0.3.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:fa2d254b20ef2e8a094fd2c84317ad2e133e5ec40089fbbf1f798cc255dfd467"}, + {file = "pyroaring-0.3.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cf7e4b01a62ffc8ab15d443e90afa29226f7cc7a6814e60cca1be7e4ff20c4fc"}, + {file = "pyroaring-0.3.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20b7a8d445ad974ba8d17239d1729478930de8f2dae980566b55383f55280256"}, + {file = "pyroaring-0.3.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a10bb61c6d0128d0e92f4e16c5d995e23cbf7a1641a3ada59676d5e5ca004f39"}, + {file = "pyroaring-0.3.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:782f453e5a9fb1bea0e108c1d2b3e9920e41c97590b528e359a6203df3fe3bfd"}, + {file = "pyroaring-0.3.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6932a4eb7fbd2673e22f4251a643ed997ec86d90f16644fbb833493e35ad19ac"}, + {file = "pyroaring-0.3.4-cp310-cp310-win32.whl", hash = "sha256:25b3bc512069777fbf05f533c248cab23f91dc99bd9f9f2cd72260a9fca525a4"}, + {file = "pyroaring-0.3.4-cp310-cp310-win_amd64.whl", hash = "sha256:1b7b8bcfd4011c13f38ffb3a96553201cb44675f16d9cc5d9a8d4560dc40c89d"}, + {file = "pyroaring-0.3.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b522eba5140631e892c5727590b18cd4b93f849e5ae5fbe9e5498c94563ac3d3"}, + {file = "pyroaring-0.3.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:40af83423015c46f0ec29201fa3a855d216fc241b6148cbc2479b4180cf558e9"}, + {file = "pyroaring-0.3.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c5c0f79b1202564e8131d1e9dcd1efb7cca026562007de5a0bd6948cdcacb90"}, + {file = "pyroaring-0.3.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a595aafbfa541431b61a740d51c3a6d5d579650e95cd6854526a692fb32df7a"}, + {file = "pyroaring-0.3.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0c2451e2c534c4df53a1df7afdb994f2387da0cf7f48df3059d332d085a79bb8"}, + {file = "pyroaring-0.3.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9c37613e3916860f344bb475883b2b8f2832ac8c816836e2c81ae131d987a9b3"}, + {file = "pyroaring-0.3.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a7c6ac7cbd0e799aeaea69e686160a7cb2c3d1b4804d5bf28e1816b37575cb21"}, + {file = "pyroaring-0.3.4-cp311-cp311-win32.whl", hash = "sha256:c3de6d36002a8561738baacc3e64b77102d597bcec536f82a9d20ec1b5e2d2db"}, + {file = "pyroaring-0.3.4-cp311-cp311-win_amd64.whl", hash = "sha256:5064a8fd58937e942bad5c797348dcf2c539796bbbf1e9ed24b9dc55c249d577"}, + {file = "pyroaring-0.3.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ed54e3394cb762bda5846d58407aa2fb371ff8eb6e518bb3703802660aef86b1"}, + {file = "pyroaring-0.3.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b0b7b4b407b178cf0076832ec1eba1a38ea88e869a410f09d3748a2f9816aa6"}, + {file = "pyroaring-0.3.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f503303333ef58333f04acf63b8b6ff80ded2064d840bd4e96fda570c9b6c52"}, + {file = "pyroaring-0.3.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c9d0ac2e47d201299d579e570114ae7b46f4fb49e292c796ad74e80211c48f89"}, + {file = "pyroaring-0.3.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a047cb76fd8d46f7fcac9d67566885a2920ec65a9ddf090e7e29e3496d9ab849"}, + {file = "pyroaring-0.3.4-cp37-cp37m-win32.whl", hash = "sha256:5f446a2817c540ddac125f69a161c56349cbd43981cea303370cdcd76a35660e"}, + {file = "pyroaring-0.3.4-cp37-cp37m-win_amd64.whl", hash = "sha256:1037b5408699acafb5d49868f9dbc43239df385a3d6fe945f9ed4429e8adf440"}, + {file = "pyroaring-0.3.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6e217c983ae6f24728f36bb2ec95464fb2a74d65e8a1e866a7eb2f412c61d00a"}, + {file = "pyroaring-0.3.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:fcfc7c9fd0d2f33d0297434bfb76a2bf6bc951fb0cbe86e1824451cb75a787e9"}, + {file = "pyroaring-0.3.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8a22e3d3d24e0daf640a2aad5b8fe86e99ad1a0c49ca0605130b19cf8df934a0"}, + {file = "pyroaring-0.3.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ac83a7f486631c111897a13de1d4ddd166c071980f794cd2ef7f12e96c41f95"}, + {file = "pyroaring-0.3.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9046804a12b4337c0320a02f78a73e3c5a9f2e2390bd4b5a59fefccec27d1626"}, + {file = "pyroaring-0.3.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:41ec889b0474838237e0f2edcb3aeeb12bf478fcaab6a7e9fc541b1e45e790cf"}, + {file = "pyroaring-0.3.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8088c458bcf8fa8fdf7e174f6d1d3f5c46ef14183ecfdd07dd9741eb6a931a99"}, + {file = "pyroaring-0.3.4-cp38-cp38-win32.whl", hash = "sha256:bbcbf5956a29135dd206398e61c2a0434e8e7d13c77e0a2fafa897b0ae020e97"}, + {file = "pyroaring-0.3.4-cp38-cp38-win_amd64.whl", hash = "sha256:9f38871aff1660af2c043ab16faca61ef8bd1a0079ca31ac8a0088ee5b81b1f9"}, + {file = "pyroaring-0.3.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b05d72e239628eea6cf4e757b284169b470cfa83c24ce3b31ab8b374ce2f2b5e"}, + {file = "pyroaring-0.3.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3567563493f9b609df0d46b34a3d4dab7c8f43cad34bbfb22202f63f688fe627"}, + {file = "pyroaring-0.3.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1f42dd009cbdf5aec8743e9b00cf8e23e83afef38f6d75ab51a2512c578fda79"}, + {file = "pyroaring-0.3.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf32c2b62eeb69f76afea290c1b59b432eab7766751c0d53a2ad957c14f707f2"}, + {file = "pyroaring-0.3.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:060c1459c74dc1abd93303e7396d0d4682d0bbefa5c1f1d44c8d19814ce2dd31"}, + {file = "pyroaring-0.3.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4d82a7f03e60f5b2404b917f91b7792a0c03207809ddb8ef16fa36679fa1923b"}, + {file = "pyroaring-0.3.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3a7484ed3fe76acf34b1bfb51b8fd2a153acea6f7270d64fd4be637ae51ebe90"}, + {file = "pyroaring-0.3.4-cp39-cp39-win32.whl", hash = "sha256:c0880fe9f0155cd028662838cde8799aabca588842727fb9495487174bb2477d"}, + {file = "pyroaring-0.3.4-cp39-cp39-win_amd64.whl", hash = "sha256:b68c169b169d90c4e58610e4a352db9530e6de951ee0ac422a7501fcc1e10b0f"}, + {file = "pyroaring-0.3.4.tar.gz", hash = "sha256:0ef2fabf2808b93eb1d884fd637ea96f2c1bfb877797295b5331fd572794e2ce"}, +] pyrsistent = [ {file = "pyrsistent-0.18.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:df46c854f490f81210870e509818b729db4488e1f30f2a1ce1698b2295a878d1"}, {file = "pyrsistent-0.18.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d45866ececf4a5fff8742c25722da6d4c9e180daa7b405dc0a2a2790d668c26"}, @@ -1836,6 +1889,8 @@ rlp = [ {file = "rlp-3.0.0-py2.py3-none-any.whl", hash = "sha256:d2a963225b3f26795c5b52310e0871df9824af56823d739511583ef459895a7d"}, {file = "rlp-3.0.0.tar.gz", hash = "sha256:63b0465d2948cd9f01de449d7adfb92d207c1aef3982f20310f8009be4a507e8"}, ] +roaring64 = [] +rocksdb = [] setuptools = [ {file = "setuptools-65.5.0-py3-none-any.whl", hash = "sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356"}, {file = "setuptools-65.5.0.tar.gz", hash = "sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17"}, @@ -1871,10 +1926,6 @@ toolz = [ {file = "toolz-0.11.2-py3-none-any.whl", hash = "sha256:a5700ce83414c64514d82d60bcda8aabfde092d1c1a8663f9200c07fdcc6da8f"}, {file = "toolz-0.11.2.tar.gz", hash = "sha256:6b312d5e15138552f1bda8a4e66c30e236c831b612b2bf0005f8a1df10a4bc33"}, ] -typing-extensions = [ - {file = "typing_extensions-4.2.0-py3-none-any.whl", hash = "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708"}, - {file = "typing_extensions-4.2.0.tar.gz", hash = "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"}, -] urllib3 = [ {file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"}, {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"}, @@ -2014,7 +2065,3 @@ yarl = [ {file = "yarl-1.7.2-cp39-cp39-win_amd64.whl", hash = "sha256:797c2c412b04403d2da075fb93c123df35239cd7b4cc4e0cd9e5839b73f52c58"}, {file = "yarl-1.7.2.tar.gz", hash = "sha256:45399b46d60c253327a460e99856752009fcee5f5d3c80b2f7c0cae1c38d56dd"}, ] -zipp = [ - {file = "zipp-3.8.0-py3-none-any.whl", hash = "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"}, - {file = "zipp-3.8.0.tar.gz", hash = "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad"}, -] diff --git a/integration_tests/pyproject.toml b/integration_tests/pyproject.toml index 54fb3b3f60..5eb3283d89 100644 --- a/integration_tests/pyproject.toml +++ b/integration_tests/pyproject.toml @@ -20,7 +20,7 @@ python-dateutil = "^2.8.1" web3 = "^6.0.0b6" eth-bloom = "^1.0.4" python-dotenv = "^0.19.2" -pystarport = { git = "https://github.com/crypto-com/pystarport.git", branch = "main" } +pystarport = { git = "https://github.com/yihuang/pystarport.git", branch = "main" } websockets = "^10.3" toml = "^0.10.2" pysha3 = "^1.0.2" From 66e50d4b53466e5f068443b2a0271aaf08d74d34 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Mon, 28 Nov 2022 16:21:31 +0800 Subject: [PATCH 03/15] fix streamer decoding --- integration_tests/test_streamer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integration_tests/test_streamer.py b/integration_tests/test_streamer.py index f0128d1e29..d003e4c0ed 100644 --- a/integration_tests/test_streamer.py +++ b/integration_tests/test_streamer.py @@ -17,8 +17,10 @@ def decode_stream_file(data, entry_cls=StoreKVPairs): """ StoreKVPairs, StoreKVPairs, ... """ + assert int.from_bytes(data[:8], "big") + 8 == len(data), "incomplete file" + items = [] - offset = 0 + offset = 8 while offset < len(data): size, n = decode_primitive(data[offset:], "uint64") offset += n From 850ed6677272d2232a643d8785a6983fc249c788 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Mon, 28 Nov 2022 16:33:57 +0800 Subject: [PATCH 04/15] fix linters --- app/app.go | 4 ++-- client/flags.go | 3 --- nix/testenv.nix | 2 +- versiondb/backend_test_utils.go | 15 +++++++-------- 4 files changed, 10 insertions(+), 14 deletions(-) delete mode 100644 client/flags.go diff --git a/app/app.go b/app/app.go index a67cd2c8e9..f85b2ed081 100644 --- a/app/app.go +++ b/app/app.go @@ -123,7 +123,7 @@ import ( gravitytypes "github.com/peggyjv/gravity-bridge/module/v2/x/gravity/types" // this line is used by starport scaffolding # stargate/app/moduleImport - cronosappclient "github.com/crypto-org-chain/cronos/client" + "github.com/crypto-org-chain/cronos/versiondb" "github.com/crypto-org-chain/cronos/versiondb/tmdb" cronosclient "github.com/crypto-org-chain/cronos/x/cronos/client" @@ -356,7 +356,7 @@ func New( // configure state listening capabilities using AppOptions // we are doing nothing with the returned streamingServices and waitGroup in this case - streamers := cast.ToStringSlice(appOpts.Get(cronosappclient.FlagStreamers)) + streamers := cast.ToStringSlice(appOpts.Get("store.streamers")) for _, streamerName := range streamers { if streamerName == "versiondb" { rootDir := cast.ToString(appOpts.Get(flags.FlagHome)) diff --git a/client/flags.go b/client/flags.go deleted file mode 100644 index 87ea0d120c..0000000000 --- a/client/flags.go +++ /dev/null @@ -1,3 +0,0 @@ -package client - -const FlagStreamers = "store.streamers" diff --git a/nix/testenv.nix b/nix/testenv.nix index 0f84ddc388..2e2298d50b 100644 --- a/nix/testenv.nix +++ b/nix/testenv.nix @@ -15,7 +15,7 @@ poetry2nix.mkPoetryEnv { flake8-black = [ "setuptools" ]; multiaddr = [ "setuptools" ]; rocksdb = [ "setuptools" "cython" "pkgconfig" ]; - pyroaring = [ "setuptools" ]; + pyroaring = [ "setuptools" "cython" ]; roaring64 = [ "poetry" ]; }; in diff --git a/versiondb/backend_test_utils.go b/versiondb/backend_test_utils.go index a5cff0cb00..507ab87ec8 100644 --- a/versiondb/backend_test_utils.go +++ b/versiondb/backend_test_utils.go @@ -11,12 +11,9 @@ import ( ) var ( - key1 = []byte("key1") - key2 = []byte("key2") - value1 = []byte("value1") - value2 = []byte("value2") - value3 = []byte("value3") - key1_subkey = []byte("key1/subkey") + key1 = []byte("key1") + value1 = []byte("value1") + key1Subkey = []byte("key1/subkey") ) func SetupTestDB(t *testing.T, store VersionStore) { @@ -122,7 +119,7 @@ func testBasics(t *testing.T, store VersionStore) { value, err = store.GetAtVersion("staking", key1, &v) require.NoError(t, err) require.Equal(t, value1, value) - value, err = store.GetAtVersion("staking", key1_subkey, &v) + value, err = store.GetAtVersion("staking", key1Subkey, &v) require.NoError(t, err) require.Equal(t, value1, value) } @@ -184,9 +181,11 @@ func testIterator(t *testing.T, store VersionStore) { } it, err := store.IteratorAtVersion("evm", nil, nil, nil) + require.NoError(t, err) require.Equal(t, expItems[len(expItems)-1], consumeIterator(it)) it, err = store.ReverseIteratorAtVersion("evm", nil, nil, nil) + require.NoError(t, err) require.Equal(t, reversed(expItems[len(expItems)-1]), consumeIterator(it)) // with start parameter @@ -231,7 +230,7 @@ func testIterator(t *testing.T, store VersionStore) { expItems[v-1][:len(expItems[v-1])-1], consumeIterator(it), ) - v -= 1 + v-- it, err = store.IteratorAtVersion("evm", nil, nil, &v) require.NoError(t, err) require.Equal(t, From 1f6eebb4b18cd89398b08badc46bfc1962fe4c55 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Mon, 28 Nov 2022 18:00:16 +0800 Subject: [PATCH 05/15] home path --- app/app.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/app.go b/app/app.go index f85b2ed081..f3297ce970 100644 --- a/app/app.go +++ b/app/app.go @@ -22,7 +22,6 @@ import ( dbm "github.com/tendermint/tm-db" "github.com/cosmos/cosmos-sdk/baseapp" - "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/client/grpc/tmservice" "github.com/cosmos/cosmos-sdk/codec" "github.com/cosmos/cosmos-sdk/server/api" @@ -359,8 +358,7 @@ func New( streamers := cast.ToStringSlice(appOpts.Get("store.streamers")) for _, streamerName := range streamers { if streamerName == "versiondb" { - rootDir := cast.ToString(appOpts.Get(flags.FlagHome)) - dataDir := filepath.Join(rootDir, "data", "versiondb") + dataDir := filepath.Join(homePath, "data", "versiondb") if err := os.MkdirAll(dataDir, os.ModePerm); err != nil { panic(err) } From 856b18fdf8387a9789b16bd800d1becf2a475711 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Mon, 28 Nov 2022 18:14:26 +0800 Subject: [PATCH 06/15] fix integration test --- integration_tests/configs/pruned-node.jsonnet | 8 ++++++-- integration_tests/configs/state_benchmark.jsonnet | 2 +- integration_tests/test_upgrade.py | 1 + integration_tests/test_upgrade_gravity.py | 1 + 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/integration_tests/configs/pruned-node.jsonnet b/integration_tests/configs/pruned-node.jsonnet index 6e87b2288b..3c9cdc4cdc 100644 --- a/integration_tests/configs/pruned-node.jsonnet +++ b/integration_tests/configs/pruned-node.jsonnet @@ -2,13 +2,17 @@ local config = import 'default.jsonnet'; config { 'cronos_777-1'+: { - // don't enable versiondb, since it don't do pruning right now - 'start-flags': '--trace --streamers file', + 'start-flags': '--trace', 'app-config'+: { pruning: 'everything', 'state-sync'+: { 'snapshot-interval': 0, }, + store+: { + // don't enable versiondb, since it don't do pruning right now + streamers: ['file'], + }, + }, genesis+: { app_state+: { diff --git a/integration_tests/configs/state_benchmark.jsonnet b/integration_tests/configs/state_benchmark.jsonnet index ba8dbf1741..8ae37bdbca 100644 --- a/integration_tests/configs/state_benchmark.jsonnet +++ b/integration_tests/configs/state_benchmark.jsonnet @@ -2,7 +2,7 @@ local config = import 'default.jsonnet'; config { 'cronos_777-1'+: { - 'start-flags': '--trace --streamers file,versiondb', + 'start-flags': '--trace', 'app-config'+: { 'app-db-backend': 'rocksdb', 'state-sync'+: { diff --git a/integration_tests/test_upgrade.py b/integration_tests/test_upgrade.py index ad3a84fe43..28673bb9e7 100644 --- a/integration_tests/test_upgrade.py +++ b/integration_tests/test_upgrade.py @@ -82,6 +82,7 @@ def custom_cronos(tmp_path_factory): ) +@pytest.mark.skip(reason="no way of currently testing this") def test_cosmovisor_upgrade(custom_cronos: Cronos): """ - propose an upgrade and pass it diff --git a/integration_tests/test_upgrade_gravity.py b/integration_tests/test_upgrade_gravity.py index fce44a4b9d..d4f3d88dd9 100644 --- a/integration_tests/test_upgrade_gravity.py +++ b/integration_tests/test_upgrade_gravity.py @@ -74,6 +74,7 @@ def custom_cronos(tmp_path_factory): ) +@pytest.mark.skip(reason="no way of currently testing this") def test_cosmovisor_upgrade_gravity(custom_cronos: Cronos): """ - propose an upgrade and pass it From 7b93785578e9fc8bb21a60dde88634d86beee104 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Mon, 28 Nov 2022 18:34:08 +0800 Subject: [PATCH 07/15] fix unit test --- integration_tests/poetry.lock | 6 +- integration_tests/pyproject.toml | 2 +- versiondb/sync_test.go | 322 +------------------------------ 3 files changed, 10 insertions(+), 320 deletions(-) diff --git a/integration_tests/poetry.lock b/integration_tests/poetry.lock index 8e9f159796..2a72101f1f 100644 --- a/integration_tests/poetry.lock +++ b/integration_tests/poetry.lock @@ -772,9 +772,9 @@ tomlkit = "^0.7.0" [package.source] type = "git" -url = "https://github.com/yihuang/pystarport.git" +url = "https://github.com/crypto-com/pystarport.git" reference = "main" -resolved_reference = "e6a75ce65a1ea07a6b402d4b6ddca291503089d9" +resolved_reference = "08c5d488e620f6f0b0fd95b55b44731d327c6977" [[package]] name = "pytest" @@ -1112,7 +1112,7 @@ multidict = ">=4.0" [metadata] lock-version = "1.1" python-versions = "^3.10" -content-hash = "ff24ebe9c0d682d71698a957082ca483863691538895b93d2a4a8b9a652ccf29" +content-hash = "9fe9a7aecc2e2597e0e03623b64c541aebe70034638a47456967626b6a8a3398" [metadata.files] aiohttp = [ diff --git a/integration_tests/pyproject.toml b/integration_tests/pyproject.toml index 5eb3283d89..54fb3b3f60 100644 --- a/integration_tests/pyproject.toml +++ b/integration_tests/pyproject.toml @@ -20,7 +20,7 @@ python-dateutil = "^2.8.1" web3 = "^6.0.0b6" eth-bloom = "^1.0.4" python-dotenv = "^0.19.2" -pystarport = { git = "https://github.com/yihuang/pystarport.git", branch = "main" } +pystarport = { git = "https://github.com/crypto-com/pystarport.git", branch = "main" } websockets = "^10.3" toml = "^0.10.2" pysha3 = "^1.0.2" diff --git a/versiondb/sync_test.go b/versiondb/sync_test.go index 85498ee07f..23b86e58d4 100644 --- a/versiondb/sync_test.go +++ b/versiondb/sync_test.go @@ -8,330 +8,20 @@ import ( "testing" "github.com/cosmos/cosmos-sdk/store/types" + sdk "github.com/cosmos/cosmos-sdk/types" "github.com/stretchr/testify/require" ) -const data = ` -a2380aff030a20ff2dd95def59a0c5265f30768a1c6fbdab519c96fb98df -33d524a94575bac1e91292030a02080b120c63726f6e6f735f3737372d31 -1802220c08d9a39a9a0610b8d89eca022a480a202d8f46f61152696c6dc8 -0c6367c7ddc926dc3ed7ab23bd37fb7ca83eb00ba207122408011220df77 -0b6cb0a9694f550ae515d5fb0ebc08902e0fa47ceec816928e74fd8fee44 -322004943bc3709104b52f30f83ca3d30d8bf57551edd48797aef5cee50b -b532c15c3a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934c -a495991b7852b8554220f537a6e0561fa0edd12b30ec9b6479e659f6f1fa -1587e69556201bfaf4cf97404a20f537a6e0561fa0edd12b30ec9b6479e6 -59f6f1fa1587e69556201bfaf4cf97405220252fe7cf36dd1bb85dafc47a -08961df0cfd8c027defa5e01e958be121599db9d5a209048462db52bb809 -6c92f8733fcf26455a000a550d9f3748dddadcc2267917146220e3b0c442 -98fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8556a20 -e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852 -b8557214ae7ccd8d599769209074b81d0c2f3f28624742b71a4612210a1d -0a142861b93c776d100695688250c5b1ba4d44ef2a791880a094a58d1d10 -0112210a1d0a14ae7ccd8d599769209074b81d0c2f3f28624742b71880a0 -94a58d1d100112ae100a630a0d636f696e5f726563656976656412360a08 -7265636569766572122a637263316d33683330776c767366386c6c727578 -7470756b64767379306b6d326b756d386c3061773437121a0a06616d6f75 -6e74121034313139343530383537377374616b650a5c0a08636f696e6261 -736512340a066d696e746572122a637263316d33683330776c767366386c -6c7275787470756b64767379306b6d326b756d386c3061773437121a0a06 -616d6f756e74121034313139343530383537377374616b650a5f0a0a636f -696e5f7370656e7412350a077370656e646572122a637263316d33683330 -776c767366386c6c7275787470756b64767379306b6d326b756d386c3061 -773437121a0a06616d6f756e74121034313139343530383537377374616b -650a630a0d636f696e5f726563656976656412360a087265636569766572 -122a637263313778706676616b6d32616d67393632796c73366638347a33 -6b656c6c3863356c3838656b6572121a0a06616d6f756e74121034313139 -343530383537377374616b650a95010a087472616e7366657212370a0972 -6563697069656e74122a637263313778706676616b6d32616d6739363279 -6c73366638347a336b656c6c3863356c3838656b657212340a0673656e64 -6572122a637263316d33683330776c767366386c6c7275787470756b6476 -7379306b6d326b756d386c3061773437121a0a06616d6f756e7412103431 -3139343530383537377374616b650a3f0a076d65737361676512340a0673 -656e646572122a637263316d33683330776c767366386c6c727578747075 -6b64767379306b6d326b756d386c30617734370aa2010a046d696e741224 -0a0c626f6e6465645f726174696f1214302e393939393939393739343032 -37343439353212210a09696e666c6174696f6e1214302e31323939393939 -3739373130313635333031123a0a11616e6e75616c5f70726f766973696f -6e7312253235393939393936343737353631363138382e37363031383234 -363033383333383838343312150a06616d6f756e74120b34313139343530 -383537370a5f0a0a636f696e5f7370656e7412350a077370656e64657212 -2a637263313778706676616b6d32616d67393632796c73366638347a336b -656c6c3863356c3838656b6572121a0a06616d6f756e7412103832333839 -3031393532307374616b650a630a0d636f696e5f72656365697665641236 -0a087265636569766572122a637263316a7636357333677271663676366a -6c33647034743663397439726b3939636438737037326d70121a0a06616d -6f756e74121038323338393031393532307374616b650a95010a08747261 -6e7366657212370a09726563697069656e74122a637263316a7636357333 -677271663676366a6c33647034743663397439726b393963643873703732 -6d7012340a0673656e646572122a637263313778706676616b6d32616d67 -393632796c73366638347a336b656c6c3863356c3838656b6572121a0a06 -616d6f756e74121038323338393031393532307374616b650a3f0a076d65 -737361676512340a0673656e646572122a637263313778706676616b6d32 -616d67393632796c73366638347a336b656c6c3863356c3838656b65720a -7f0a0f70726f706f7365725f726577617264122c0a06616d6f756e741222 -343131393435303937362e30303030303030303030303030303030303073 -74616b65123e0a0976616c696461746f72123163726376616c6f70657231 -326c756b75367578656868616b303270793472637a36357a753073776837 -776a36756c726c670a790a0a636f6d6d697373696f6e122b0a06616d6f75 -6e7412213431313934353039372e36303030303030303030303030303030 -30307374616b65123e0a0976616c696461746f72123163726376616c6f70 -657231326c756b75367578656868616b303270793472637a36357a753073 -776837776a36756c726c670a770a0772657761726473122c0a06616d6f75 -6e741222343131393435303937362e303030303030303030303030303030 -3030307374616b65123e0a0976616c696461746f72123163726376616c6f -70657231326c756b75367578656868616b303270793472637a36357a7530 -73776837776a36756c726c670a7a0a0a636f6d6d697373696f6e122c0a06 -616d6f756e741222333833313038393430372e3638303030303030303030 -303030303030307374616b65123e0a0976616c696461746f721231637263 -76616c6f70657231326c756b75367578656868616b303270793472637a36 -357a753073776837776a36756c726c670a780a0772657761726473122d0a -06616d6f756e74122333383331303839343037362e383030303030303030 -3030303030303030307374616b65123e0a0976616c696461746f72123163 -726376616c6f70657231326c756b75367578656868616b30327079347263 -7a36357a753073776837776a36756c726c670a7a0a0a636f6d6d69737369 -6f6e122c0a06616d6f756e741222333833313038393430372e3638303030 -303030303030303030303030307374616b65123e0a0976616c696461746f -72123163726376616c6f70657231387a367133386d68767473767972356d -616b38666a38733867346777376b6a6a7030653064680a780a0772657761 -726473122d0a06616d6f756e74122333383331303839343037362e383030 -3030303030303030303030303030307374616b65123e0a0976616c696461 -746f72123163726376616c6f70657231387a367133386d68767473767972 -356d616b38666a38733867346777376b6a6a7030653064680a250a0a6665 -655f6d61726b657412170a08626173655f666565120b3736353632353030 -3030301aa9100aa4040aa1040afa020aba020a2a2f636f736d6f732e7374 -616b696e672e763162657461312e4d736743726561746556616c69646174 -6f72128b020a070a056e6f646530123b0a12313030303030303030303030 -30303030303012123230303030303030303030303030303030301a113130 -3030303030303030303030303030301a0131222a63726331326c756b7536 -7578656868616b303270793472637a36357a753073776837776a73727730 -70702a3163726376616c6f70657231326c756b75367578656868616b3032 -70793472637a36357a753073776837776a36756c726c6732430a1d2f636f -736d6f732e63727970746f2e656432353531392e5075624b657912220a20 -9b86839433f4229b7b7b51554a25ed32549126eb80fe53497ae336a1e67b -66843a1c0a057374616b6512133130303030303030303030303030303030 -3030123b3439643130313933363334303730393664303337396438643863 -3833336438626633396334383866403139322e3136382e302e35343a3236 -363536125f0a570a4f0a282f65746865726d696e742e63727970746f2e76 -312e657468736563703235366b312e5075624b657912230a21026e710a62 -a342de0ed4d7c4532dcbcbbafbf19652ed67b237efab70e8b207efac1204 -0a020801120410c09a0c1a4119ffd1b342c44e183b8113561595186e1f72 -c7273ac4e9ccb04c2b8a4d31b3780eff63cb490102ed39f7482084b4ddb8 -4d672ee2607262a312724f0e5b2b79aa0012ff0b123612340a322f636f73 -6d6f732e7374616b696e672e763162657461312e4d736743726561746556 -616c696461746f72526573706f6e73651ae0055b7b226d73675f696e6465 -78223a302c226576656e7473223a5b7b2274797065223a22636f696e5f72 -65636569766564222c2261747472696275746573223a5b7b226b6579223a -227265636569766572222c2276616c7565223a22637263317479676d7333 -78686873337976343837706878336477346139356a6e3774376c6b393067 -6161227d2c7b226b6579223a22616d6f756e74222c2276616c7565223a22 -313030303030303030303030303030303030307374616b65227d5d7d2c7b -2274797065223a22636f696e5f7370656e74222c22617474726962757465 -73223a5b7b226b6579223a227370656e646572222c2276616c7565223a22 -63726331326c756b75367578656868616b303270793472637a36357a7530 -73776837776a737277307070227d2c7b226b6579223a22616d6f756e7422 -2c2276616c7565223a223130303030303030303030303030303030303073 -74616b65227d5d7d2c7b2274797065223a226372656174655f76616c6964 -61746f72222c2261747472696275746573223a5b7b226b6579223a227661 -6c696461746f72222c2276616c7565223a2263726376616c6f7065723132 -6c756b75367578656868616b303270793472637a36357a75307377683777 -6a36756c726c67227d2c7b226b6579223a22616d6f756e74222c2276616c -7565223a22313030303030303030303030303030303030307374616b6522 -7d5d7d2c7b2274797065223a226d657373616765222c2261747472696275 -746573223a5b7b226b6579223a22616374696f6e222c2276616c7565223a -222f636f736d6f732e7374616b696e672e763162657461312e4d73674372 -6561746556616c696461746f72227d2c7b226b6579223a226d6f64756c65 -222c2276616c7565223a227374616b696e67227d2c7b226b6579223a2273 -656e646572222c2276616c7565223a2263726331326c756b753675786568 -68616b303270793472637a36357a753073776837776a737277307070227d -5d7d5d7d5d28ffffffffffffffffff0130bca10d3a440a02747812050a03 -66656512370a096665655f7061796572122a63726331326c756b75367578 -656868616b303270793472637a36357a753073776837776a737277307070 -3a3d0a02747812370a076163635f736571122c63726331326c756b753675 -78656868616b303270793472637a36357a753073776837776a7372773070 -702f303a6b0a02747812650a097369676e6174757265125847662f527330 -4c455468673767524e57465a55596268397978796336784f6e4d73457772 -696b30787333674f2f32504c5351454337546e3353434345744e32345457 -6375346d427959714d53636b384f577974357167413d3a3f0a076d657373 -61676512340a06616374696f6e122a2f636f736d6f732e7374616b696e67 -2e763162657461312e4d736743726561746556616c696461746f723a670a -0a636f696e5f7370656e7412350a077370656e646572122a63726331326c -756b75367578656868616b303270793472637a36357a753073776837776a -73727730707012220a06616d6f756e741218313030303030303030303030 -303030303030307374616b653a6b0a0d636f696e5f726563656976656412 -360a087265636569766572122a637263317479676d733378686873337976 -343837706878336477346139356a6e3774376c6b393067616112220a0661 -6d6f756e741218313030303030303030303030303030303030307374616b -653a760a106372656174655f76616c696461746f72123e0a0976616c6964 -61746f72123163726376616c6f70657231326c756b75367578656868616b -303270793472637a36357a753073776837776a36756c726c6712220a0661 -6d6f756e741218313030303030303030303030303030303030307374616b -653a520a076d65737361676512110a066d6f64756c6512077374616b696e -6712340a0673656e646572122a63726331326c756b75367578656868616b -303270793472637a36357a753073776837776a7372773070701aa9100aa4 -040aa1040afa020aba020a2a2f636f736d6f732e7374616b696e672e7631 -62657461312e4d736743726561746556616c696461746f72128b020a070a -056e6f646531123b0a123130303030303030303030303030303030301212 -3230303030303030303030303030303030301a1131303030303030303030 -303030303030301a0131222a63726331387a367133386d68767473767972 -356d616b38666a38733867346777376b6a6a747367726e372a3163726376 -616c6f70657231387a367133386d68767473767972356d616b38666a3873 -3867346777376b6a6a70306530646832430a1d2f636f736d6f732e637279 -70746f2e656432353531392e5075624b657912220a2072b50cf0ed1863ff -c937af99b6ad779a2c223e59459eab7768bda7c2da6f836e3a1c0a057374 -616b65121331303030303030303030303030303030303030123b35396566 -333139663464383334396466626532613233653131356163353835356430 -303938613065403139322e3136382e302e35343a3236363536125f0a570a -4f0a282f65746865726d696e742e63727970746f2e76312e657468736563 -703235366b312e5075624b657912230a210242785a75074452d62a6ac222 -70ffb8fb01c9375d0ba72887ae800dc619315d1b12040a020801120410c0 -9a0c1a41e7019fd760970e02f8967aa0f9820c0b98de32d8e72601aa34fe -60df52356d19591eb2bd8516037d2c52c22170ca533abf72d50d4c7f770d -1d5e045df51ff89c0112ff0b123612340a322f636f736d6f732e7374616b -696e672e763162657461312e4d736743726561746556616c696461746f72 -526573706f6e73651ae0055b7b226d73675f696e646578223a302c226576 -656e7473223a5b7b2274797065223a22636f696e5f726563656976656422 -2c2261747472696275746573223a5b7b226b6579223a2272656365697665 -72222c2276616c7565223a22637263317479676d73337868687333797634 -3837706878336477346139356a6e3774376c6b3930676161227d2c7b226b -6579223a22616d6f756e74222c2276616c7565223a223130303030303030 -30303030303030303030307374616b65227d5d7d2c7b2274797065223a22 -636f696e5f7370656e74222c2261747472696275746573223a5b7b226b65 -79223a227370656e646572222c2276616c7565223a2263726331387a3671 -33386d68767473767972356d616b38666a38733867346777376b6a6a7473 -67726e37227d2c7b226b6579223a22616d6f756e74222c2276616c756522 -3a22313030303030303030303030303030303030307374616b65227d5d7d -2c7b2274797065223a226372656174655f76616c696461746f72222c2261 -747472696275746573223a5b7b226b6579223a2276616c696461746f7222 -2c2276616c7565223a2263726376616c6f70657231387a367133386d6876 -7473767972356d616b38666a38733867346777376b6a6a70306530646822 -7d2c7b226b6579223a22616d6f756e74222c2276616c7565223a22313030 -303030303030303030303030303030307374616b65227d5d7d2c7b227479 -7065223a226d657373616765222c2261747472696275746573223a5b7b22 -6b6579223a22616374696f6e222c2276616c7565223a222f636f736d6f73 -2e7374616b696e672e763162657461312e4d736743726561746556616c69 -6461746f72227d2c7b226b6579223a226d6f64756c65222c2276616c7565 -223a227374616b696e67227d2c7b226b6579223a2273656e646572222c22 -76616c7565223a2263726331387a367133386d68767473767972356d616b -38666a38733867346777376b6a6a747367726e37227d5d7d5d7d5d28ffff -ffffffffffffff0130fc8c0d3a440a02747812050a0366656512370a0966 -65655f7061796572122a63726331387a367133386d68767473767972356d -616b38666a38733867346777376b6a6a747367726e373a3d0a0274781237 -0a076163635f736571122c63726331387a367133386d6876747376797235 -6d616b38666a38733867346777376b6a6a747367726e372f303a6b0a0274 -7812650a097369676e61747572651258357747663132435844674c346c6e -71672b59494d43356a654d746a6e4a6747714e5035673331493162526c5a -48724b39685259446653785377694677796c4d3676334c564455782f6477 -30645867526439522f346e41453d3a3f0a076d65737361676512340a0661 -6374696f6e122a2f636f736d6f732e7374616b696e672e76316265746131 -2e4d736743726561746556616c696461746f723a670a0a636f696e5f7370 -656e7412350a077370656e646572122a63726331387a367133386d687674 -73767972356d616b38666a38733867346777376b6a6a747367726e371222 -0a06616d6f756e7412183130303030303030303030303030303030303073 -74616b653a6b0a0d636f696e5f726563656976656412360a087265636569 -766572122a637263317479676d7333786868733379763438377068783364 -77346139356a6e3774376c6b393067616112220a06616d6f756e74121831 -3030303030303030303030303030303030307374616b653a760a10637265 -6174655f76616c696461746f72123e0a0976616c696461746f7212316372 -6376616c6f70657231387a367133386d68767473767972356d616b38666a -38733867346777376b6a6a70306530646812220a06616d6f756e74121831 -3030303030303030303030303030303030307374616b653a520a076d6573 -7361676512110a066d6f64756c6512077374616b696e6712340a0673656e -646572122a63726331387a367133386d68767473767972356d616b38666a -38733867346777376b6a6a747367726e37220208022aec0212260a090880 -804010e0aeee26120e08a08d0612040880c60a188080401a090a07656432 -353531391a9a020a0b626c6f636b5f626c6f6f6d128a020a05626c6f6f6d -128002000000000000000000000000000000000000000000000000000000 -000000000000000000000000000000000000000000000000000000000000 -000000000000000000000000000000000000000000000000000000000000 -000000000000000000000000000000000000000000000000000000000000 -000000000000000000000000000000000000000000000000000000000000 -000000000000000000000000000000000000000000000000000000000000 -000000000000000000000000000000000000000000000000000000000000 -000000000000000000000000000000000000000000000000000000000000 -000000000000000000000000000000000000001a250a09626c6f636b5f67 -6173120b0a06686569676874120132120b0a06616d6f756e741201303222 -1220b995a75e975242574ff1730feaf1e9255743e86ff9d949b04e2906ad -fd10f824230a0462616e6b1a06007374616b652213323030303030303038 -32333839303139353230300a0462616e6b1a1b021493354845030274cd4b -f1686abd60ab28ec52e1a77374616b65220b383233383930313935323025 -0a0462616e6b10011a1b0214dc6f17bbec824fff8f86587966b2047db6ab -73677374616b65250a0462616e6b10011a1b0214f1829676db577682e944 -fc3493d451b67ff3e29f7374616b65270a0462616e6b1a1c037374616b65 -001493354845030274cd4bf1686abd60ab28ec52e1a7220100260a046261 -6e6b10011a1c037374616b650014dc6f17bbec824fff8f86587966b2047d -b6ab7367260a0462616e6b10011a1c037374616b650014f1829676db5776 -82e944fc3493d451b67ff3e29f3a0a0c646973747269627574696f6e1a01 -0022270a250a057374616b65121c31363437373830333930343030303030 -303030303030303030303030290a0c646973747269627574696f6e1a0101 -22160a14ae7ccd8d599769209074b81d0c2f3f28624742b7500a0c646973 -747269627574696f6e1a16021438b4089f7762e0c20e9bed8e991e074550 -ef5a5222280a260a057374616b65121d3338333130383934303736383030 -303030303030303030303030303030500a0c646973747269627574696f6e -1a16021457f96e6b86cdefdb3d412547816a82e3e0ebf9d222280a260a05 -7374616b65121d3432343330333435303532383030303030303030303030 -303030303030520a0c646973747269627574696f6e1a16061438b4089f77 -62e0c20e9bed8e991e074550ef5a52222a0a260a057374616b65121d3334 -343739383034363639313230303030303030303030303030303030100252 -0a0c646973747269627574696f6e1a16061457f96e6b86cdefdb3d412547 -816a82e3e0ebf9d2222a0a260a057374616b65121d333831383733313035 -343735323030303030303030303030303030303010024f0a0c6469737472 -69627574696f6e1a16071438b4089f7762e0c20e9bed8e991e074550ef5a -5222270a250a057374616b65121c33383331303839343037363830303030 -3030303030303030303030304f0a0c646973747269627574696f6e1a1607 -1457f96e6b86cdefdb3d412547816a82e3e0ebf9d222270a250a05737461 -6b65121c3432343330333435303532383030303030303030303030303030 -3030180a096665656d61726b65741a010122080000000000000000450a04 -6d696e741a0100223a0a1231323939393939373937313031363533303112 -243235393939393936343737353631363138383736303138323436303338 -333338383834332a0a06706172616d731a116665656d61726b65742f4261 -7365466565220d223736353632353030303030225b0a08736c617368696e -671a1601142861b93c776d100695688250c5b1ba4d44ef2a7922370a3163 -726376616c636f6e73313970736d6a307268643567716439746773666776 -7476643666347a7737326e656674666d6730180122005b0a08736c617368 -696e671a160114ae7ccd8d599769209074b81d0c2f3f28624742b722370a -3163726376616c636f6e7331346537766d7232656a61356a707972356871 -77736374656c397033797773346874333468797a18012200cd070a077374 -616b696e671a02503222bd070a92030a02080b120c63726f6e6f735f3737 -372d311802220c08d9a39a9a0610b8d89eca022a480a202d8f46f6115269 -6c6dc80c6367c7ddc926dc3ed7ab23bd37fb7ca83eb00ba2071224080112 -20df770b6cb0a9694f550ae515d5fb0ebc08902e0fa47ceec816928e74fd -8fee44322004943bc3709104b52f30f83ca3d30d8bf57551edd48797aef5 -cee50bb532c15c3a20e3b0c44298fc1c149afbf4c8996fb92427ae41e464 -9b934ca495991b7852b8554220f537a6e0561fa0edd12b30ec9b6479e659 -f6f1fa1587e69556201bfaf4cf97404a20f537a6e0561fa0edd12b30ec9b -6479e659f6f1fa1587e69556201bfaf4cf97405220252fe7cf36dd1bb85d -afc47a08961df0cfd8c027defa5e01e958be121599db9d5a209048462db5 -2bb8096c92f8733fcf26455a000a550d9f3748dddadcc2267917146220e3 -b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8 -556a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca49599 -1b7852b8557214ae7ccd8d599769209074b81d0c2f3f28624742b7129102 -0a3163726376616c6f70657231326c756b75367578656868616b30327079 -3472637a36357a753073776837776a36756c726c6712430a1d2f636f736d -6f732e63727970746f2e656432353531392e5075624b657912220a209b86 -839433f4229b7b7b51554a25ed32549126eb80fe53497ae336a1e67b6684 -20032a133130303030303030303030303030303030303032253130303030 -303030303030303030303030303030303030303030303030303030303030 -30303a070a056e6f6465304a00524b0a3b0a123130303030303030303030 -3030303030303012123230303030303030303030303030303030301a1131 -30303030303030303030303030303030120c08d1a39a9a0610a89eb0a602 -5a01311291020a3163726376616c6f70657231387a367133386d68767473 -767972356d616b38666a38733867346777376b6a6a70306530646812430a -1d2f636f736d6f732e63727970746f2e656432353531392e5075624b6579 -12220a2072b50cf0ed1863ffc937af99b6ad779a2c223e59459eab7768bd -a7c2da6f836e20032a133130303030303030303030303030303030303032 -253130303030303030303030303030303030303030303030303030303030 -30303030303030303a070a056e6f6465314a00524b0a3b0a123130303030 -303030303030303030303030301212323030303030303030303030303030 -3030301a113130303030303030303030303030303030120c08d1a39a9a06 -10a89eb0a6025a0131 -` +const data = "0000000000000873230a0462616e6b1a06007374616b65221332303030303030303832333839303139353230300a0462616e6b1a1b021493354845030274cd4bf1686abd60ab28ec52e1a77374616b65220b3832333839303139353230250a0462616e6b10011a1b0214dc6f17bbec824fff8f86587966b2047db6ab73677374616b65250a0462616e6b10011a1b0214f1829676db577682e944fc3493d451b67ff3e29f7374616b65270a0462616e6b1a1c037374616b65001493354845030274cd4bf1686abd60ab28ec52e1a7220100260a0462616e6b10011a1c037374616b650014dc6f17bbec824fff8f86587966b2047db6ab7367260a0462616e6b10011a1c037374616b650014f1829676db577682e944fc3493d451b67ff3e29f3a0a0c646973747269627574696f6e1a010022270a250a057374616b65121c31363437373830333930343030303030303030303030303030303030290a0c646973747269627574696f6e1a010122160a14c352ad46450b10f1763ab28ade60aeecf4a276e7500a0c646973747269627574696f6e1a16021438b4089f7762e0c20e9bed8e991e074550ef5a5222280a260a057374616b65121d3432343330333435303532383030303030303030303030303030303030500a0c646973747269627574696f6e1a16021457f96e6b86cdefdb3d412547816a82e3e0ebf9d222280a260a057374616b65121d3338333130383934303736383030303030303030303030303030303030520a0c646973747269627574696f6e1a16061438b4089f7762e0c20e9bed8e991e074550ef5a52222a0a260a057374616b65121d33383138373331303534373532303030303030303030303030303030301002520a0c646973747269627574696f6e1a16061457f96e6b86cdefdb3d412547816a82e3e0ebf9d2222a0a260a057374616b65121d333434373938303436363931323030303030303030303030303030303010024f0a0c646973747269627574696f6e1a16071438b4089f7762e0c20e9bed8e991e074550ef5a5222270a250a057374616b65121c343234333033343530353238303030303030303030303030303030304f0a0c646973747269627574696f6e1a16071457f96e6b86cdefdb3d412547816a82e3e0ebf9d222270a250a057374616b65121c33383331303839343037363830303030303030303030303030303030180a096665656d61726b65741a010122080000000000000000450a046d696e741a0100223a0a1231323939393939373937313031363533303112243235393939393936343737353631363138383736303138323436303338333338383834332a0a06706172616d731a116665656d61726b65742f42617365466565220d223736363136313830393832225b0a08736c617368696e671a1601145c0f70af16fa6e056d5a722651c586b24f10908322370a3163726376616c636f6e7331747338687074636b6c666871326d323677676e39723376786b663833707979726c7067616832180122005b0a08736c617368696e671a160114c352ad46450b10f1763ab28ade60aeecf4a276e722370a3163726376616c636f6e73316364663236336a39707667307a6133366b32396475633977616e363279616838736b71686d7418012200cb070a077374616b696e671a02503222bb070a92030a02080b120c63726f6e6f735f3737372d311802220c08ebdc919c0610f89ce7db022a480a204f4280f40de2dea51618996576a960ee849cf8c605af6ea7912b7d462dfbf21c1224080112207b8624266881e12487a887ec8b2a5744d496422c245f33925f2b33ecd0c0ce8b3220b17b7bc6e2317e48eeab99202baf538fee726ac57f81d8f99e1cd94fa8ac48e03a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855422095e4fff352417c37f09c26c5c89e1939fb995f1d477b272d0f450ed1e6b779d34a2095e4fff352417c37f09c26c5c89e1939fb995f1d477b272d0f450ed1e6b779d35220252fe7cf36dd1bb85dafc47a08961df0cfd8c027defa5e01e958be121599db9d5a20a911dcd4c68befef4cf9798ee12c94da9ee2d60b5c43f620814d1e59ef3ad0316220e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8556a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8557214c352ad46450b10f1763ab28ade60aeecf4a276e71290020a3163726376616c6f70657231387a367133386d68767473767972356d616b38666a38733867346777376b6a6a70306530646812430a1d2f636f736d6f732e63727970746f2e656432353531392e5075624b657912220a20cdc66f7519ba5a3101c5b62a65bd4a7a767e9cadb22504e25bf1fdf78433109220032a13313030303030303030303030303030303030303225313030303030303030303030303030303030303030303030303030303030303030303030303a070a056e6f6465314a00524a0a3b0a1231303030303030303030303030303030303012123230303030303030303030303030303030301a113130303030303030303030303030303030120b08e4dc919c0610f0b180535a01311290020a3163726376616c6f70657231326c756b75367578656868616b303270793472637a36357a753073776837776a36756c726c6712430a1d2f636f736d6f732e63727970746f2e656432353531392e5075624b657912220a20b7eda2d527f8ecf441ad339841632bfe87b289de17d95d309fc091f7065ff75520032a13313030303030303030303030303030303030303225313030303030303030303030303030303030303030303030303030303030303030303030303a070a056e6f6465304a00524a0a3b0a1231303030303030303030303030303030303012123230303030303030303030303030303030301a113130303030303030303030303030303030120b08e4dc919c0610f0b180535a0131" func TestReadFileStreamer(t *testing.T) { buf, err := hex.DecodeString(strings.Replace(data, "\n", "", -1)) require.NoError(t, err) - changeSet, err := ReadFileStreamer(bufio.NewReader(bytes.NewReader(buf))) + size := sdk.BigEndianToUint64(buf[:8]) + require.Equal(t, size+8, uint64(len(buf))) + + changeSet, err := ReadFileStreamer(bufio.NewReader(bytes.NewReader(buf[8:]))) require.NoError(t, err) require.Equal(t, 21, len(changeSet)) From 234ad32c8d4226b1f120a1ebbddce0f04f707f77 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Tue, 29 Nov 2022 10:38:46 +0800 Subject: [PATCH 08/15] fix upgrade test --- integration_tests/configs/cosmovisor.jsonnet | 1 + integration_tests/poetry.lock | 2 +- integration_tests/test_upgrade.py | 1 - integration_tests/test_upgrade_gravity.py | 1 - 4 files changed, 2 insertions(+), 3 deletions(-) diff --git a/integration_tests/configs/cosmovisor.jsonnet b/integration_tests/configs/cosmovisor.jsonnet index 053e557720..208cf4e834 100644 --- a/integration_tests/configs/cosmovisor.jsonnet +++ b/integration_tests/configs/cosmovisor.jsonnet @@ -4,6 +4,7 @@ config { 'cronos_777-1'+: { 'app-config'+: { 'minimum-gas-prices': '100000000000basetcro', + store:: super.store, }, genesis+: { app_state+: { diff --git a/integration_tests/poetry.lock b/integration_tests/poetry.lock index 2a72101f1f..c1e906d137 100644 --- a/integration_tests/poetry.lock +++ b/integration_tests/poetry.lock @@ -774,7 +774,7 @@ tomlkit = "^0.7.0" type = "git" url = "https://github.com/crypto-com/pystarport.git" reference = "main" -resolved_reference = "08c5d488e620f6f0b0fd95b55b44731d327c6977" +resolved_reference = "ff645505d5052e11ecc56b6933af8f5f2a6b4373" [[package]] name = "pytest" diff --git a/integration_tests/test_upgrade.py b/integration_tests/test_upgrade.py index 28673bb9e7..ad3a84fe43 100644 --- a/integration_tests/test_upgrade.py +++ b/integration_tests/test_upgrade.py @@ -82,7 +82,6 @@ def custom_cronos(tmp_path_factory): ) -@pytest.mark.skip(reason="no way of currently testing this") def test_cosmovisor_upgrade(custom_cronos: Cronos): """ - propose an upgrade and pass it diff --git a/integration_tests/test_upgrade_gravity.py b/integration_tests/test_upgrade_gravity.py index d4f3d88dd9..fce44a4b9d 100644 --- a/integration_tests/test_upgrade_gravity.py +++ b/integration_tests/test_upgrade_gravity.py @@ -74,7 +74,6 @@ def custom_cronos(tmp_path_factory): ) -@pytest.mark.skip(reason="no way of currently testing this") def test_cosmovisor_upgrade_gravity(custom_cronos: Cronos): """ - propose an upgrade and pass it From 14be5b672f167e2076aa7f0d1173dece381aa49c Mon Sep 17 00:00:00 2001 From: HuangYi Date: Wed, 7 Dec 2022 16:44:09 +0800 Subject: [PATCH 09/15] update to upstream --- app/app.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- gomod2nix.toml | 6 +++--- versiondb/streaming_service.go | 10 +++++----- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/app/app.go b/app/app.go index f3297ce970..e9018efa04 100644 --- a/app/app.go +++ b/app/app.go @@ -348,7 +348,7 @@ func New( memKeys := sdk.NewMemoryStoreKeys(capabilitytypes.MemStoreKey) // load state streaming if enabled - if _, _, err := streaming.LoadStreamingServices(bApp, appOpts, appCodec, logger, keys, homePath); err != nil { + if _, _, err := streaming.LoadStreamingServices(bApp, appOpts, appCodec, keys); err != nil { fmt.Printf("failed to load state streaming: %s", err) os.Exit(1) } diff --git a/go.mod b/go.mod index a72ea75ed9..4d8390ca40 100644 --- a/go.mod +++ b/go.mod @@ -195,7 +195,7 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/yihuang/cosmos-sdk v0.43.0-beta1.0.20221128050842-6df48c3ab38b + github.com/cosmos/cosmos-sdk => github.com/cosmos/cosmos-sdk v0.46.7-0.20221206163158-45d2f08e0e1d github.com/ethereum/go-ethereum => github.com/ethereum/go-ethereum v1.10.19 // Fix upstream GHSA-h395-qcrw-5vmq vulnerability. diff --git a/go.sum b/go.sum index bc3497abfe..e507b872ec 100644 --- a/go.sum +++ b/go.sum @@ -247,6 +247,8 @@ github.com/cosmos/btcutil v1.0.4 h1:n7C2ngKXo7UC9gNyMNLbzqz7Asuf+7Qv4gnX/rOdQ44= github.com/cosmos/btcutil v1.0.4/go.mod h1:Ffqc8Hn6TJUdDgHBwIZLtrLQC1KdJ9jGJl/TvgUaxbU= github.com/cosmos/cosmos-proto v1.0.0-alpha7 h1:yqYUOHF2jopwZh4dVQp3xgqwftE5/2hkrwIV6vkUbO0= github.com/cosmos/cosmos-proto v1.0.0-alpha7/go.mod h1:dosO4pSAbJF8zWCzCoTWP7nNsjcvSUBQmniFxDg5daw= +github.com/cosmos/cosmos-sdk v0.46.7-0.20221206163158-45d2f08e0e1d h1:Np+fiB2/FUV/lPhFVecebV+nO9qsQgILx/hWcVeKfiI= +github.com/cosmos/cosmos-sdk v0.46.7-0.20221206163158-45d2f08e0e1d/go.mod h1:B2j/SQkKvs/hUbXTmKxGwBnShJHvlxtVEbvTYNPwfrU= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY= github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw= @@ -1048,8 +1050,6 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= -github.com/yihuang/cosmos-sdk v0.43.0-beta1.0.20221128050842-6df48c3ab38b h1:h4gWiEuVB9B8PfY3KL5lwdtObndvcIwti0MOFkEtaqI= -github.com/yihuang/cosmos-sdk v0.43.0-beta1.0.20221128050842-6df48c3ab38b/go.mod h1:B2j/SQkKvs/hUbXTmKxGwBnShJHvlxtVEbvTYNPwfrU= github.com/yihuang/mdbx-go v0.0.0-20221010042614-b72b4f091d88 h1:RqQjImUPy0joQDaP/Xv8+YXrHYqfR3fJpbKLBchVddY= github.com/yihuang/mdbx-go v0.0.0-20221010042614-b72b4f091d88/go.mod h1:T2fsoJDVppxfAPTLd1svUgH1kpPmeXdPESmroSHcL1E= github.com/yihuang/tm-db v0.0.0-20221006023748-f6214ae9454d h1:+HeYUyE9B0Zg/eYVj9LIZfriP7B71akpRvCmv1oIIbw= diff --git a/gomod2nix.toml b/gomod2nix.toml index a99a296035..e44d00d658 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -107,9 +107,9 @@ schema = 3 version = "v1.0.0-alpha7" hash = "sha256-2wCH+toTF2A6MfFjOa13muEH5oBCcxAhZEqirNOrBA0=" [mod."github.com/cosmos/cosmos-sdk"] - version = "v0.43.0-beta1.0.20221128050842-6df48c3ab38b" - hash = "sha256-SxpPaUvHOZ5TYN5VnjwAROwXHqPwNYBdst2Rfs4TaKU=" - replaced = "github.com/yihuang/cosmos-sdk" + version = "v0.46.7-0.20221206163158-45d2f08e0e1d" + hash = "sha256-4j5nR0VxlxhBeCQS4RvWngQZfZIy2NVNxyTFwmmwGTk=" + replaced = "github.com/cosmos/cosmos-sdk" [mod."github.com/cosmos/go-bip39"] version = "v1.0.0" hash = "sha256-Qm2aC2vaS8tjtMUbHmlBSagOSqbduEEDwc51qvQaBmA=" diff --git a/versiondb/streaming_service.go b/versiondb/streaming_service.go index e56aef0e18..f61732333d 100644 --- a/versiondb/streaming_service.go +++ b/versiondb/streaming_service.go @@ -1,6 +1,7 @@ package versiondb import ( + "context" "sort" "strings" "sync" @@ -9,7 +10,6 @@ import ( "github.com/cosmos/cosmos-sdk/baseapp" "github.com/cosmos/cosmos-sdk/store/types" - sdk "github.com/cosmos/cosmos-sdk/types" ) var _ baseapp.StreamingService = &StreamingService{} @@ -47,23 +47,23 @@ func (fss *StreamingService) Listeners() map[types.StoreKey][]types.WriteListene // ListenBeginBlock satisfies the baseapp.ABCIListener interface // It sets the currentBlockNumber. -func (fss *StreamingService) ListenBeginBlock(ctx sdk.Context, req abci.RequestBeginBlock, res abci.ResponseBeginBlock) error { +func (fss *StreamingService) ListenBeginBlock(ctx context.Context, req abci.RequestBeginBlock, res abci.ResponseBeginBlock) error { fss.currentBlockNumber = req.GetHeader().Height return nil } // ListenDeliverTx satisfies the baseapp.ABCIListener interface -func (fss *StreamingService) ListenDeliverTx(ctx sdk.Context, req abci.RequestDeliverTx, res abci.ResponseDeliverTx) error { +func (fss *StreamingService) ListenDeliverTx(ctx context.Context, req abci.RequestDeliverTx, res abci.ResponseDeliverTx) error { return nil } // ListenEndBlock satisfies the baseapp.ABCIListener interface // It merge the state caches of all the listeners together, and write out to the versionStore. -func (fss *StreamingService) ListenEndBlock(ctx sdk.Context, req abci.RequestEndBlock, res abci.ResponseEndBlock) error { +func (fss *StreamingService) ListenEndBlock(ctx context.Context, req abci.RequestEndBlock, res abci.ResponseEndBlock) error { return nil } -func (fss *StreamingService) ListenCommit(ctx sdk.Context, res abci.ResponseCommit) error { +func (fss *StreamingService) ListenCommit(ctx context.Context, res abci.ResponseCommit) error { // concat the state caches var changeSet []types.StoreKVPair for _, listener := range fss.listeners { From b686422b65300ca5b6da19cf027701fde02e3905 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Wed, 7 Dec 2022 18:01:10 +0800 Subject: [PATCH 10/15] fix gravity test --- integration_tests/configs/cosmovisor_gravity.jsonnet | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/configs/cosmovisor_gravity.jsonnet b/integration_tests/configs/cosmovisor_gravity.jsonnet index 25ef6beb3c..86d4b300af 100644 --- a/integration_tests/configs/cosmovisor_gravity.jsonnet +++ b/integration_tests/configs/cosmovisor_gravity.jsonnet @@ -5,6 +5,7 @@ config { 'cmd-flags': '--unsafe-experimental', 'app-config'+: { 'minimum-gas-prices': '100000000000basetcro', + store:: super.store, }, genesis+: { app_state+: { From eef8781c3928415b8f01ae5125e0306b359249b7 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Fri, 9 Dec 2022 10:31:23 +0800 Subject: [PATCH 11/15] revert mdbx from this PR --- Makefile | 4 ---- default.nix | 2 +- go.mod | 11 ++--------- go.sum | 14 +++---------- gomod2nix.toml | 24 ++++++++--------------- integration_tests/configs/default.jsonnet | 2 +- 6 files changed, 15 insertions(+), 42 deletions(-) diff --git a/Makefile b/Makefile index e92f965247..94ea62b5e1 100644 --- a/Makefile +++ b/Makefile @@ -74,10 +74,6 @@ ifeq (boltdb,$(findstring boltdb,$(COSMOS_BUILD_OPTIONS))) BUILD_TAGS += boltdb endif -ifeq (mdbx,$(findstring mdbx,$(COSMOS_BUILD_OPTIONS))) - BUILD_TAGS += mdbx -endif - ifeq (,$(findstring nostrip,$(COSMOS_BUILD_OPTIONS))) ldflags += -w -s endif diff --git a/default.nix b/default.nix index 846e83eea0..0dd65d1259 100644 --- a/default.nix +++ b/default.nix @@ -8,7 +8,7 @@ let version = "v0.9.0"; pname = "cronosd"; - tags = [ "ledger" "netgo" network "mdbx" ] + tags = [ "ledger" "netgo" network ] ++ lib.lists.optionals (rocksdb != null) [ "rocksdb" "rocksdb_build" ]; ldflags = lib.concatStringsSep "\n" ([ "-X github.com/cosmos/cosmos-sdk/version.Name=cronos" diff --git a/go.mod b/go.mod index 4d8390ca40..3048a6be54 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,6 @@ require ( github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect - github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect @@ -73,8 +72,9 @@ require ( github.com/deckarep/golang-set v1.8.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect - github.com/dgraph-io/badger/v3 v3.2103.2 // indirect + github.com/dgraph-io/badger/v2 v2.2007.4 // indirect github.com/dgraph-io/ristretto v0.1.0 // indirect + github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect github.com/dop251/goja v0.0.0-20220405120441-9037c2b61cbf // indirect github.com/dustin/go-humanize v1.0.0 // indirect @@ -95,7 +95,6 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.2 // indirect - github.com/google/flatbuffers v2.0.0+incompatible // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/orderedcode v0.0.1 // indirect github.com/google/uuid v1.3.0 // indirect @@ -170,7 +169,6 @@ require ( github.com/tendermint/go-amino v0.16.0 // indirect github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect - github.com/torquem-ch/mdbx-go v0.26.0 // indirect github.com/tyler-smith/go-bip39 v1.1.0 // indirect github.com/ulikunitz/xz v0.5.8 // indirect github.com/zondax/hid v0.9.1-0.20220302062450-5552068d2266 // indirect @@ -205,11 +203,6 @@ replace ( github.com/peggyjv/gravity-bridge/module/v2 => github.com/crypto-org-chain/gravity-bridge/module/v2 v2.0.1-0.20221027085649-2107c6bd6bc4 - // https://github.com/tendermint/tm-db/pull/297 - github.com/tendermint/tm-db => github.com/yihuang/tm-db v0.0.0-20221006023748-f6214ae9454d - - github.com/torquem-ch/mdbx-go => github.com/yihuang/mdbx-go v0.0.0-20221010042614-b72b4f091d88 - // TODO: remove after fixed https://github.com/cosmos/cosmos-sdk/issues/11364 github.com/zondax/hid => github.com/zondax/hid v0.9.0 ) diff --git a/go.sum b/go.sum index e507b872ec..3be31013c8 100644 --- a/go.sum +++ b/go.sum @@ -186,8 +186,6 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= -github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b h1:6+ZFm0flnudZzdSE0JxlhR2hKnGPcNB35BjQf4RYQDY= -github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= @@ -292,9 +290,8 @@ github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= +github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= -github.com/dgraph-io/badger/v3 v3.2103.2 h1:dpyM5eCJAtQCBcMCZcT4UBZchuTJgCywerHHgmxfxM8= -github.com/dgraph-io/badger/v3 v3.2103.2/go.mod h1:RHo4/GmYcKKh5Lxu63wLEMHJ70Pac2JqZRYGhlyAo2M= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= @@ -475,9 +472,6 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/flatbuffers v2.0.0+incompatible h1:dicJ2oXwypfwUGnB2/TYWYEKiuk9eYQlQO/AnOHl5mI= -github.com/google/flatbuffers v2.0.0+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -1010,6 +1004,8 @@ github.com/tendermint/go-amino v0.16.0 h1:GyhmgQKvqF82e2oZeuMSp9JTN0N09emoSZlb2l github.com/tendermint/go-amino v0.16.0/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME= github.com/tendermint/tendermint v0.34.24 h1:879MKKJWYYPJEMMKME+DWUTY4V9f/FBpnZDI82ky+4k= github.com/tendermint/tendermint v0.34.24/go.mod h1:rXVrl4OYzmIa1I91av3iLv2HS0fGSiucyW9J4aMTpKI= +github.com/tendermint/tm-db v0.6.7 h1:fE00Cbl0jayAoqlExN6oyQJ7fR/ZtoVOmvPJ//+shu8= +github.com/tendermint/tm-db v0.6.7/go.mod h1:byQDzFkZV1syXr/ReXS808NxA2xvyuuVgXOJ/088L6I= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= @@ -1050,10 +1046,6 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= -github.com/yihuang/mdbx-go v0.0.0-20221010042614-b72b4f091d88 h1:RqQjImUPy0joQDaP/Xv8+YXrHYqfR3fJpbKLBchVddY= -github.com/yihuang/mdbx-go v0.0.0-20221010042614-b72b4f091d88/go.mod h1:T2fsoJDVppxfAPTLd1svUgH1kpPmeXdPESmroSHcL1E= -github.com/yihuang/tm-db v0.0.0-20221006023748-f6214ae9454d h1:+HeYUyE9B0Zg/eYVj9LIZfriP7B71akpRvCmv1oIIbw= -github.com/yihuang/tm-db v0.0.0-20221006023748-f6214ae9454d/go.mod h1:a8QLaVn+zo5zgX2qMrNSETPO/8uBRPxvLsqL+urelEU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/gomod2nix.toml b/gomod2nix.toml index e44d00d658..408f0be0d3 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -76,9 +76,6 @@ schema = 3 [mod."github.com/btcsuite/btcutil"] version = "v1.0.3-0.20201208143702-a53e38424cce" hash = "sha256-4kasJReFcj25JRHx9dJMct3yDkHqVoHGUx5cu45Msfo=" - [mod."github.com/c2h5oh/datasize"] - version = "v0.0.0-20220606134207-859f65c6625b" - hash = "sha256-1uH+D3w0Y/B3poXm545XGrT4S4c+msTbj7gKgu9pbPM=" [mod."github.com/cenkalti/backoff/v4"] version = "v4.1.3" hash = "sha256-u6MEDopHoTWAZoVvvXOKnAg++xre53YgQx0gmf6t2KU=" @@ -149,12 +146,15 @@ schema = 3 [mod."github.com/desertbit/timer"] version = "v0.0.0-20180107155436-c41aec40b27f" hash = "sha256-abLOtEcomAqCWLphd2X6WkD/ED764w6sa6unox4BXss=" - [mod."github.com/dgraph-io/badger/v3"] - version = "v3.2103.2" - hash = "sha256-F6pvsaSKwXOl9RfnUQFqAl6xpCVu9+rthQgOxhKVk1g=" + [mod."github.com/dgraph-io/badger/v2"] + version = "v2.2007.4" + hash = "sha256-+KwqZJZpViv8S3TqUVvPXrFoMgWFyS3NoLsi4RR5fGk=" [mod."github.com/dgraph-io/ristretto"] version = "v0.1.0" hash = "sha256-01jneg1+1x8tTfUTBZ+6mHkQaqXVnPYxLJyJhJQcvt4=" + [mod."github.com/dgryski/go-farm"] + version = "v0.0.0-20200201041132-a6ae2369ad13" + hash = "sha256-aOMlPwFY36bLiiIx4HonbCYRAhagk5N6HAWN7Ygif+E=" [mod."github.com/dlclark/regexp2"] version = "v1.4.1-0.20201116162257-a2a8dda75c91" hash = "sha256-VNNMZIc7NkDg3DVLnqeJNM/KZqkkaZu2/HTLBL8X2xE=" @@ -229,9 +229,6 @@ schema = 3 [mod."github.com/google/btree"] version = "v1.1.2" hash = "sha256-K7V2obq3pLM71Mg0vhhHtZ+gtaubwXPQx3xcIyZDCjM=" - [mod."github.com/google/flatbuffers"] - version = "v2.0.0+incompatible" - hash = "sha256-4Db9FdOL60Da4H1+K4Qv02w4omxdsh3uzpmY1vtqHeA=" [mod."github.com/google/go-cmp"] version = "v0.5.9" hash = "sha256-lQc4O00R3QSMGs9LP8Sy7A9kj0cqV5rrUdpnGeipIyg=" @@ -477,19 +474,14 @@ schema = 3 version = "v0.34.24" hash = "sha256-3HFTv4XgN535RDaJ5OwUS+fnJHgkmLTwU7CNU2ilxEQ=" [mod."github.com/tendermint/tm-db"] - version = "v0.0.0-20221006023748-f6214ae9454d" - hash = "sha256-mtTVR3f3A9CmcyJBXTeurQuHGiVA5hIzqlxskz1M1qk=" - replaced = "github.com/yihuang/tm-db" + version = "v0.6.7" + hash = "sha256-hl/3RrBrpkk2zA6dmrNlIYKs1/GfqegSscDSkA5Pjlo=" [mod."github.com/tklauser/go-sysconf"] version = "v0.3.10" hash = "sha256-Zf2NsgM9+HeM949vCce4HQtSbfUiFpeiQ716yKcFyx4=" [mod."github.com/tklauser/numcpus"] version = "v0.4.0" hash = "sha256-ndE82nOb3agubhEV7aRzEqqTlN4DPbKFHEm2+XZLn8k=" - [mod."github.com/torquem-ch/mdbx-go"] - version = "v0.0.0-20221010042614-b72b4f091d88" - hash = "sha256-HWsrhzSGoYgEUUziQPHEtQGwBQ/upg3/f1fY2NGiC0g=" - replaced = "github.com/yihuang/mdbx-go" [mod."github.com/tyler-smith/go-bip39"] version = "v1.1.0" hash = "sha256-3YhWBtSwRLGwm7vNwqumphZG3uLBW1vwT9QkQ8JuSjU=" diff --git a/integration_tests/configs/default.jsonnet b/integration_tests/configs/default.jsonnet index ac6cf543f2..a0351d612c 100644 --- a/integration_tests/configs/default.jsonnet +++ b/integration_tests/configs/default.jsonnet @@ -9,7 +9,7 @@ }, }, 'app-config': { - // 'app-db-backend': 'rocksdb', + 'app-db-backend': 'rocksdb', 'minimum-gas-prices': '0basetcro', 'index-events': ['ethereum_tx.ethereumTxHash'], 'json-rpc': { From 611e2f3cdfc7d1d54c3e7369a1289128e4fa5a22 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Fri, 9 Dec 2022 10:37:34 +0800 Subject: [PATCH 12/15] remove FIXME --- versiondb/store.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/versiondb/store.go b/versiondb/store.go index 906423fb9e..283e846496 100644 --- a/versiondb/store.go +++ b/versiondb/store.go @@ -11,6 +11,8 @@ import ( "github.com/cosmos/cosmos-sdk/telemetry" ) +const StoreTypeVersionDB = 100 + var _ types.KVStore = (*Store)(nil) // Store Implements types.KVStore @@ -26,8 +28,8 @@ func NewKVStore(store VersionStore, storeKey types.StoreKey, version *int64) *St // Implements Store. func (st *Store) GetStoreType() types.StoreType { - // FIXME - return types.StoreTypeIAVL + // should have effect, just define an unique indentifier, don't be conflicts with cosmos-sdk's builtin ones. + return StoreTypeVersionDB } // Implements Store. From bde66c1e23d535bb6c2d6ddd9162ff40e62c13a4 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Fri, 9 Dec 2022 10:41:34 +0800 Subject: [PATCH 13/15] remove unused functions --- versiondb/dbutils.go | 84 -------------------------------------------- versiondb/utils.go | 19 ++++++++++ 2 files changed, 19 insertions(+), 84 deletions(-) delete mode 100644 versiondb/dbutils.go create mode 100644 versiondb/utils.go diff --git a/versiondb/dbutils.go b/versiondb/dbutils.go deleted file mode 100644 index 2334d0f2c1..0000000000 --- a/versiondb/dbutils.go +++ /dev/null @@ -1,84 +0,0 @@ -package versiondb - -import ( - "encoding/binary" - "sort" - - "github.com/RoaringBitmap/roaring/roaring64" -) - -var ChunkLimit = uint64(1950) // threshold beyond which MDBX overflow pages appear: 4096 / 2 - (keySize + 8) - -// CutLeft - cut from bitmap `targetSize` bytes from left -// removing lft part from `bm` -// returns nil on zero cardinality -func CutLeft64(bm *roaring64.Bitmap, sizeLimit uint64) *roaring64.Bitmap { - if bm.GetCardinality() == 0 { - return nil - } - - sz := bm.GetSerializedSizeInBytes() - if sz <= sizeLimit { - lft := roaring64.New() - lft.AddRange(bm.Minimum(), bm.Maximum()+1) - lft.And(bm) - lft.RunOptimize() - bm.Clear() - return lft - } - - from := bm.Minimum() - minMax := bm.Maximum() - bm.Minimum() - to := sort.Search(int(minMax), func(i int) bool { // can be optimized to avoid "too small steps", but let's leave it for readability - lft := roaring64.New() // bitmap.Clear() method intentionally not used here, because then serialized size of bitmap getting bigger - lft.AddRange(from, from+uint64(i)+1) - lft.And(bm) - lft.RunOptimize() - return lft.GetSerializedSizeInBytes() > sizeLimit - }) - - lft := roaring64.New() - lft.AddRange(from, from+uint64(to)) // no +1 because sort.Search returns element which is just higher threshold - but we need lower - lft.And(bm) - bm.RemoveRange(from, from+uint64(to)) - lft.RunOptimize() - return lft -} - -func WalkChunks64(bm *roaring64.Bitmap, sizeLimit uint64, f func(chunk *roaring64.Bitmap, isLast bool) error) error { - for bm.GetCardinality() > 0 { - if err := f(CutLeft64(bm, sizeLimit), bm.GetCardinality() == 0); err != nil { - return err - } - } - return nil -} - -func WalkChunkWithKeys64(k []byte, m *roaring64.Bitmap, sizeLimit uint64, f func(chunkKey []byte, chunk *roaring64.Bitmap) error) error { - return WalkChunks64(m, sizeLimit, func(chunk *roaring64.Bitmap, isLast bool) error { - chunkKey := make([]byte, len(k)+8) - copy(chunkKey, k) - if isLast { - binary.BigEndian.PutUint64(chunkKey[len(k):], ^uint64(0)) - } else { - binary.BigEndian.PutUint64(chunkKey[len(k):], chunk.Maximum()) - } - return f(chunkKey, chunk) - }) -} - -// SeekInBitmap64 - returns value in bitmap which is >= n -func SeekInBitmap64(m *roaring64.Bitmap, n uint64) (found uint64, ok bool) { - if m == nil || m.IsEmpty() { - return 0, false - } - if n == 0 { - return m.Minimum(), true - } - searchRank := m.Rank(n - 1) - if searchRank >= m.GetCardinality() { - return 0, false - } - found, _ = m.Select(searchRank) - return found, true -} diff --git a/versiondb/utils.go b/versiondb/utils.go new file mode 100644 index 0000000000..c6200caf94 --- /dev/null +++ b/versiondb/utils.go @@ -0,0 +1,19 @@ +package versiondb + +import "github.com/RoaringBitmap/roaring/roaring64" + +// SeekInBitmap64 - returns value in bitmap which is >= n +func SeekInBitmap64(m *roaring64.Bitmap, n uint64) (found uint64, ok bool) { + if m == nil || m.IsEmpty() { + return 0, false + } + if n == 0 { + return m.Minimum(), true + } + searchRank := m.Rank(n - 1) + if searchRank >= m.GetCardinality() { + return 0, false + } + found, _ = m.Select(searchRank) + return found, true +} From 659a562c8d982c07f6371105f6a4740f934d916e Mon Sep 17 00:00:00 2001 From: yihuang Date: Fri, 9 Dec 2022 10:45:15 +0800 Subject: [PATCH 14/15] Update integration_tests/test_benchmark_storage.py Co-authored-by: mmsqe Signed-off-by: yihuang --- integration_tests/test_benchmark_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_benchmark_storage.py b/integration_tests/test_benchmark_storage.py index 41668a94ac..e1782c4b11 100644 --- a/integration_tests/test_benchmark_storage.py +++ b/integration_tests/test_benchmark_storage.py @@ -36,7 +36,7 @@ def test_benchmark_storage(custom_cronos): def task(acct, acct_i): for i in range(iterations): seed = i * 10 + acct_i - tx = contract.functions.batch_set(seed, n, n * parity).buildTransaction( + tx = contract.functions.batch_set(seed, n, n * parity).build_transaction( {"from": acct.address, "gas": gas} ) print(send_transaction(w3, tx, acct.key)) From f46daef54bf6d36299b388947bb82dcb38975988 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Fri, 9 Dec 2022 12:04:28 +0800 Subject: [PATCH 15/15] remove TODO --- versiondb/tmdb/store.go | 1 - 1 file changed, 1 deletion(-) diff --git a/versiondb/tmdb/store.go b/versiondb/tmdb/store.go index d86b2eddbe..e59914827a 100644 --- a/versiondb/tmdb/store.go +++ b/versiondb/tmdb/store.go @@ -33,7 +33,6 @@ func NewStore(plainDB, historyDB, changesetDB dbm.DB) *Store { } // PutAtVersion implements VersionStore interface -// TODO reduce allocation within iterations. func (s *Store) PutAtVersion(version int64, changeSet []types.StoreKVPair) error { plainBatch := s.plainDB.NewBatch() defer plainBatch.Close()