Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Optimize key creation in hashing #2899

Merged
merged 15 commits into from
Apr 2, 2024
Merged

Optimize key creation in hashing #2899

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
15e17bf
Cleanup calculateNodeIDs
StephenButtolph Mar 31, 2024
4ea7ef0
nit
StephenButtolph Mar 31, 2024
3c78610
typo
StephenButtolph Mar 31, 2024
eaa64a0
Add initial hashing benchmark
StephenButtolph Mar 31, 2024
4dec1ab
add tests
StephenButtolph Mar 31, 2024
dfeb3fb
nits
StephenButtolph Mar 31, 2024
2cf95e5
Remove allocations for keys
StephenButtolph Mar 31, 2024
7d6ac6c
typo
StephenButtolph Mar 31, 2024
a018121
comment
StephenButtolph Apr 1, 2024
256cef1
comment
StephenButtolph Apr 1, 2024
c4ffa4c
fix concurrency
StephenButtolph Apr 1, 2024
8a880e4
nit
StephenButtolph Apr 2, 2024
0209ded
Address nits
StephenButtolph Apr 2, 2024
46d52bc
merged
StephenButtolph Apr 2, 2024
97109b3
nit
StephenButtolph Apr 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
18 changes: 9 additions & 9 deletions x/merkledb/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,9 +216,9 @@ type merkleDB struct {
// Valid children of this trie.
childViews []*view

// calculateNodeIDsSema controls the number of goroutines inside
// [calculateNodeIDsHelper] at any given time.
calculateNodeIDsSema *semaphore.Weighted
// hashNodesSema controls the number of goroutines that are created inside
// [hashChangedNode] at any given time.
hashNodesSema *semaphore.Weighted

tokenSize int
}
Expand Down Expand Up @@ -270,12 +270,12 @@ func newDatabase(
bufferPool,
metrics,
int(config.ValueNodeCacheSize)),
history: newTrieHistory(int(config.HistoryLength)),
debugTracer: getTracerIfEnabled(config.TraceLevel, DebugTrace, config.Tracer),
infoTracer: getTracerIfEnabled(config.TraceLevel, InfoTrace, config.Tracer),
childViews: make([]*view, 0, defaultPreallocationSize),
calculateNodeIDsSema: semaphore.NewWeighted(int64(rootGenConcurrency)),
tokenSize: BranchFactorToTokenSize[config.BranchFactor],
history: newTrieHistory(int(config.HistoryLength)),
debugTracer: getTracerIfEnabled(config.TraceLevel, DebugTrace, config.Tracer),
infoTracer: getTracerIfEnabled(config.TraceLevel, InfoTrace, config.Tracer),
childViews: make([]*view, 0, defaultPreallocationSize),
hashNodesSema: semaphore.NewWeighted(int64(rootGenConcurrency)),
tokenSize: BranchFactorToTokenSize[config.BranchFactor],
}

if err := trieDB.initializeRoot(); err != nil {
Expand Down
2 changes: 1 addition & 1 deletion x/merkledb/history.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ type changeSummary struct {
// The ID of the trie after these changes.
rootID ids.ID
// The root before/after this change.
// Set in [calculateNodeIDs].
// Set in [applyValueChanges].
rootChange change[maybe.Maybe[*node]]
nodes map[Key]*change[*node]
values map[Key]*change[maybe.Maybe[[]byte]]
Expand Down
16 changes: 8 additions & 8 deletions x/merkledb/trie_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (
func getNodeValue(t Trie, key string) ([]byte, error) {
path := ToKey([]byte(key))
if asView, ok := t.(*view); ok {
if err := asView.calculateNodeIDs(context.Background()); err != nil {
if err := asView.applyValueChanges(context.Background()); err != nil {
return nil, err
}
}
Expand Down Expand Up @@ -131,7 +131,7 @@ func TestVisitPathToKey(t *testing.T) {
require.NoError(err)
require.IsType(&view{}, trieIntf)
trie = trieIntf.(*view)
require.NoError(trie.calculateNodeIDs(context.Background()))
require.NoError(trie.applyValueChanges(context.Background()))

nodePath = make([]*node, 0, 1)
require.NoError(visitPathToKey(trie, ToKey(key1), func(n *node) error {
Expand All @@ -156,7 +156,7 @@ func TestVisitPathToKey(t *testing.T) {
require.NoError(err)
require.IsType(&view{}, trieIntf)
trie = trieIntf.(*view)
require.NoError(trie.calculateNodeIDs(context.Background()))
require.NoError(trie.applyValueChanges(context.Background()))

nodePath = make([]*node, 0, 2)
require.NoError(visitPathToKey(trie, ToKey(key2), func(n *node) error {
Expand Down Expand Up @@ -185,7 +185,7 @@ func TestVisitPathToKey(t *testing.T) {
require.NoError(err)
require.IsType(&view{}, trieIntf)
trie = trieIntf.(*view)
require.NoError(trie.calculateNodeIDs(context.Background()))
require.NoError(trie.applyValueChanges(context.Background()))

// Trie is:
// []
Expand Down Expand Up @@ -775,7 +775,7 @@ func Test_Trie_ChainDeletion(t *testing.T) {
)
require.NoError(err)

require.NoError(newTrie.(*view).calculateNodeIDs(context.Background()))
require.NoError(newTrie.(*view).applyValueChanges(context.Background()))
maybeRoot := newTrie.getRoot()
require.NoError(err)
require.True(maybeRoot.HasValue())
Expand All @@ -794,7 +794,7 @@ func Test_Trie_ChainDeletion(t *testing.T) {
},
)
require.NoError(err)
require.NoError(newTrie.(*view).calculateNodeIDs(context.Background()))
require.NoError(newTrie.(*view).applyValueChanges(context.Background()))

// trie should be empty
root := newTrie.getRoot()
Expand Down Expand Up @@ -861,7 +861,7 @@ func Test_Trie_NodeCollapse(t *testing.T) {
)
require.NoError(err)

require.NoError(trie.(*view).calculateNodeIDs(context.Background()))
require.NoError(trie.(*view).applyValueChanges(context.Background()))

for _, kv := range kvs {
node, err := trie.getEditableNode(ToKey(kv.Key), true)
Expand All @@ -888,7 +888,7 @@ func Test_Trie_NodeCollapse(t *testing.T) {
)
require.NoError(err)

require.NoError(trie.(*view).calculateNodeIDs(context.Background()))
require.NoError(trie.(*view).applyValueChanges(context.Background()))

for _, kv := range deletedKVs {
_, err := trie.getEditableNode(ToKey(kv.Key), true)
Expand Down