Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Trie storage statistics component #5401

Merged
merged 45 commits into from
Nov 6, 2023
Merged

Trie storage statistics component #5401

Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
b509c81
storage statistics component in storage package
ssd04 Jul 5, 2023
94db95c
addapt storage statistics component in statusCoreComponents
ssd04 Jul 5, 2023
bdcb008
Revert "addapt storage statistics component in statusCoreComponents"
ssd04 Jul 5, 2023
4e02971
use stats component in trie storage manager
ssd04 Jul 6, 2023
5ba1393
refactor trie statistics component
ssd04 Jul 6, 2023
1be0bd0
update storer with stats interface
ssd04 Jul 7, 2023
1a7db42
fix trie package tests
ssd04 Jul 7, 2023
2471b82
refactor to cast storagemanager in constructor
ssd04 Jul 14, 2023
261c492
fix epoch start unit tests
ssd04 Jul 17, 2023
088a9ac
Merge branch 'rc/v1.6.0' into trie-storage-statistics-component
ssd04 Jul 17, 2023
1b5b5de
fix test in epochstart and integration tests
ssd04 Jul 17, 2023
68cba11
add missing comments
ssd04 Jul 18, 2023
a352567
fix benchmarks integration test
ssd04 Jul 18, 2023
c990e1d
fix linter issue
ssd04 Jul 18, 2023
8aeac06
Merge branch 'rc/v1.6.0' into trie-storage-statistics-component
ssd04 Jul 26, 2023
c0afd2a
fixes after review
ssd04 Aug 7, 2023
8703e00
update trie stub
ssd04 Aug 7, 2023
239929d
Merge branch 'rc/v1.6.0' into trie-storage-statistics-component
ssd04 Sep 4, 2023
7a8fbb4
fix unit tests after merge
ssd04 Sep 4, 2023
c646309
move state stats collector in common
ssd04 Sep 6, 2023
dc3b665
Merge branch 'rc/v1.6.0' into trie-storage-statistics-component
ssd04 Oct 5, 2023
d8e86ed
cleanup trie stats handler
ssd04 Oct 6, 2023
185ea39
separate stats for sync and snapshot
ssd04 Oct 9, 2023
082a009
use atomic map from sync package
ssd04 Oct 10, 2023
f6dc8ed
renamings and comments + fix unit tests
ssd04 Oct 10, 2023
1caef49
fix unit tests - epochstart boostrap and factory
ssd04 Oct 10, 2023
3f566ec
fix integration tests
ssd04 Oct 10, 2023
aa16716
more unit tests + remove old code
ssd04 Oct 10, 2023
77f1f4a
snapshot manager stats
ssd04 Oct 10, 2023
20c0b03
fixes after review
ssd04 Oct 25, 2023
08225bd
fix test
ssd04 Oct 25, 2023
1278c3b
remove trie sync statistics
ssd04 Oct 25, 2023
55b9a70
remove unused code
ssd04 Oct 25, 2023
020f987
fix mutex usage
ssd04 Oct 25, 2023
d45437b
Merge branch 'rc/v1.6.0' into trie-storage-statistics-component
ssd04 Oct 26, 2023
9593d18
Merge branch 'rc/v1.7.0' into trie-storage-statistics-component
ssd04 Oct 26, 2023
ee9c8b6
revert changes from d45437b
ssd04 Oct 26, 2023
a444c20
fix unit test
ssd04 Oct 26, 2023
343e7c8
Merge branch 'rc/v1.7.0' into trie-storage-statistics-component
ssd04 Oct 30, 2023
2b19dd3
update stats return format
ssd04 Oct 30, 2023
97ad360
revert additional changes from 343e7c8
ssd04 Oct 30, 2023
63b2a82
Merge branch 'rc/v1.7.0' into trie-storage-statistics-component
ssd04 Nov 2, 2023
6a4be57
fix integration tests
ssd04 Nov 3, 2023
c192ba4
fix hardfork integration test
ssd04 Nov 3, 2023
d2f3f82
fix create prunning storer maps epoch
ssd04 Nov 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
21 changes: 21 additions & 0 deletions common/interface.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ type Trie interface {
VerifyProof(rootHash []byte, key []byte, proof [][]byte) (bool, error)
GetStorageManager() StorageManager
IsMigratedToLatestVersion() (bool, error)
GetStorageStats() string
Close() error
IsInterfaceNil() bool
}
Expand Down Expand Up @@ -120,6 +121,7 @@ type StorageManager interface {
type TrieStorageInteractor interface {
BaseStorer
GetIdentifier() string
GetStatsCollector() StateStatisticsHandler
}

// BaseStorer define the base methods needed for a storer
Expand All @@ -131,6 +133,12 @@ type BaseStorer interface {
IsInterfaceNil() bool
}

// StorerWithStats defines the behaviour of a base storer component with operations statistics
type StorerWithStats interface {
BaseStorer
GetWithStats(key []byte) ([]byte, bool, error)
}

// SnapshotDbHandler is used to keep track of how many references a snapshot db has
type SnapshotDbHandler interface {
BaseStorer
Expand Down Expand Up @@ -218,6 +226,19 @@ type TriesStatisticsCollector interface {
GetNumNodes() uint64
}

// StateStatisticsHandler defines the behaviour of a storage statistics handler
type StateStatisticsHandler interface {
Reset()
IncrCacheOp()
CacheOp() uint64
IncrPersisterOp()
PersisterOp() uint64
IncrTrieOp()
TrieOp() uint64
ToString() string
IsInterfaceNil() bool
}

// ProcessStatusHandler defines the behavior of a component able to hold the current status of the node and
// able to tell if the node is idle or processing/committing a block
type ProcessStatusHandler interface {
Expand Down
1 change: 1 addition & 0 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,7 @@ type StateTriesConfig struct {
PeerStatePruningEnabled bool
MaxStateTrieLevelInMemory uint
MaxPeerTrieLevelInMemory uint
StateStatisticsEnabled bool
}

// TrieStorageManagerConfig will hold config information about trie storage manager
Expand Down
21 changes: 11 additions & 10 deletions dataRetriever/factory/storageRequestersContainer/baseRequestersContainerFactory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
"github.com/multiversx/mx-chain-go/config"
"github.com/multiversx/mx-chain-go/dataRetriever"
disabledRequesters "github.com/multiversx/mx-chain-go/dataRetriever/requestHandlers/requesters/disabled"
"github.com/multiversx/mx-chain-go/dataRetriever/storageRequesters"
storagerequesters "github.com/multiversx/mx-chain-go/dataRetriever/storageRequesters"
"github.com/multiversx/mx-chain-go/errors"
"github.com/multiversx/mx-chain-go/p2p"
"github.com/multiversx/mx-chain-go/process/factory"
Expand Down Expand Up @@ -262,15 +262,16 @@ func (brcf *baseRequestersContainerFactory) newImportDBTrieStorage(
}

args := trieFactory.TrieCreateArgs{
MainStorer: mainStorer,
CheckpointsStorer: checkpointsStorer,
PruningEnabled: brcf.generalConfig.StateTriesConfig.AccountsStatePruningEnabled,
CheckpointsEnabled: brcf.generalConfig.StateTriesConfig.CheckpointsEnabled,
MaxTrieLevelInMem: brcf.generalConfig.StateTriesConfig.MaxStateTrieLevelInMemory,
SnapshotsEnabled: brcf.snapshotsEnabled,
IdleProvider: disabled.NewProcessStatusHandler(),
Identifier: storageIdentifier.String(),
EnableEpochsHandler: handler,
MainStorer: mainStorer,
CheckpointsStorer: checkpointsStorer,
PruningEnabled: brcf.generalConfig.StateTriesConfig.AccountsStatePruningEnabled,
CheckpointsEnabled: brcf.generalConfig.StateTriesConfig.CheckpointsEnabled,
MaxTrieLevelInMem: brcf.generalConfig.StateTriesConfig.MaxStateTrieLevelInMemory,
SnapshotsEnabled: brcf.snapshotsEnabled,
IdleProvider: disabled.NewProcessStatusHandler(),
Identifier: storageIdentifier.String(),
EnableEpochsHandler: handler,
StateStatisticsEnabled: brcf.generalConfig.StateTriesConfig.StateStatisticsEnabled,
}
return trieFactoryInstance.Create(args)
}
Expand Down
3 changes: 2 additions & 1 deletion epochStart/bootstrap/disabled/disabledChainStorer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (

"github.com/multiversx/mx-chain-go/dataRetriever"
"github.com/multiversx/mx-chain-go/storage"
"github.com/multiversx/mx-chain-go/testscommon"
logger "github.com/multiversx/mx-chain-logger-go"
)

Expand Down Expand Up @@ -59,7 +60,7 @@ func (c *chainStorer) GetStorer(unitType dataRetriever.UnitType) (storage.Storer
_, ok := c.mapStorages[unitType]
if !ok {
log.Debug("created new mem storer", "key", unitType)
c.mapStorages[unitType] = CreateMemUnit()
c.mapStorages[unitType] = testscommon.CreateMemStorerWithStats(CreateMemUnit())
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

don't we have the MemStorerWithStats somewhere else, not in the testscommon? please create it somewhere else, so that we do not import testscommon in the production code.

}

store := c.mapStorages[unitType]
Expand Down
1 change: 1 addition & 0 deletions epochStart/bootstrap/process_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ func createMockEpochStartBootstrapArgs(
StateTriesConfig: config.StateTriesConfig{
CheckpointRoundsModulus: 5,
AccountsStatePruningEnabled: true,
SnapshotsEnabled: true,
PeerStatePruningEnabled: true,
MaxStateTrieLevelInMemory: 5,
MaxPeerTrieLevelInMemory: 5,
Expand Down
18 changes: 2 additions & 16 deletions epochStart/metachain/epochStartData_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,6 @@ import (
"github.com/multiversx/mx-chain-go/process"
"github.com/multiversx/mx-chain-go/process/mock"
"github.com/multiversx/mx-chain-go/sharding"
"github.com/multiversx/mx-chain-go/storage"
"github.com/multiversx/mx-chain-go/storage/database"
"github.com/multiversx/mx-chain-go/storage/storageunit"
"github.com/multiversx/mx-chain-go/testscommon"
dataRetrieverMock "github.com/multiversx/mx-chain-go/testscommon/dataRetriever"
"github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock"
Expand Down Expand Up @@ -81,21 +78,10 @@ func createMockEpochStartCreatorArguments() ArgsNewEpochStartData {
return argsNewEpochStartData
}

func createMemUnit() storage.Storer {
capacity := uint32(10)
shards := uint32(1)
sizeInBytes := uint64(0)
cache, _ := storageunit.NewCache(storageunit.CacheConfig{Type: storageunit.LRUCache, Capacity: capacity, Shards: shards, SizeInBytes: sizeInBytes})
persist, _ := database.NewlruDB(100000)
unit, _ := storageunit.NewStorageUnit(cache, persist)

return unit
}

func createMetaStore() dataRetriever.StorageService {
store := dataRetriever.NewChainStorer()
store.AddStorer(dataRetriever.MetaBlockUnit, createMemUnit())
store.AddStorer(dataRetriever.BlockHeaderUnit, createMemUnit())
store.AddStorer(dataRetriever.MetaBlockUnit, testscommon.CreateMemUnit())
store.AddStorer(dataRetriever.BlockHeaderUnit, testscommon.CreateMemUnit())

return store
}
Expand Down
4 changes: 2 additions & 2 deletions epochStart/metachain/stakingDataProvider_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -426,7 +426,7 @@ func createStakingDataProviderWithMockArgs(
func createStakingDataProviderWithRealArgs(t *testing.T, owner []byte, blsKey []byte, topUpVal *big.Int) *stakingDataProvider {
args, _ := createFullArgumentsForSystemSCProcessing(config.EnableEpochs{
StakingV2EnableEpoch: 1000,
}, createMemUnit())
}, testscommon.CreateStorerWithStats())
args.EpochNotifier.CheckEpoch(&testscommon.HeaderHandlerStub{
EpochField: 1000000,
})
Expand Down Expand Up @@ -468,7 +468,7 @@ func createStakingDataProviderAndUpdateCache(t *testing.T, validatorsInfo map[ui

args, _ := createFullArgumentsForSystemSCProcessing(config.EnableEpochs{
StakingV2EnableEpoch: 1,
}, createMemUnit())
}, testscommon.CreateStorerWithStats())
args.EpochNotifier.CheckEpoch(&testscommon.HeaderHandlerStub{
EpochField: 1,
})
Expand Down