From 4fa79accc40cde6100e8ac46f5b3f106518d8595 Mon Sep 17 00:00:00 2001 From: Akhil Repala Date: Sat, 14 Jun 2025 11:28:29 -0500 Subject: [PATCH 1/3] feat(config): Added changes to make BadgerDB cache size configurable via badgerCacheSize Signed-off-by: Akhil Repala --- chain/chain_test.go | 6 ++++-- config.go | 9 +++++++++ database/database.go | 3 ++- database/database_test.go | 3 ++- database/plugin/blob/badger/database.go | 7 +++++++ database/plugin/blob/store.go | 3 ++- dingo.yaml.example | 7 ++++++- internal/config/config.go | 2 ++ internal/config/config_test.go | 4 ++++ internal/node/load.go | 2 +- internal/node/node.go | 1 + node.go | 2 +- 12 files changed, 41 insertions(+), 8 deletions(-) diff --git a/chain/chain_test.go b/chain/chain_test.go index 31dda093..c17d1647 100644 --- a/chain/chain_test.go +++ b/chain/chain_test.go @@ -463,7 +463,8 @@ func TestChainFromIntersect(t *testing.T) { Slot: testBlocks[testForkPointIndex].MockSlot, }, } - db, err := database.New(nil, nil, "") + const testCacheSize int64 = 1 << 20 + db, err := database.New(nil, nil, "", testCacheSize) if err != nil { t.Fatalf("unexpected error creating database: %s", err) } @@ -521,7 +522,8 @@ func TestChainFork(t *testing.T) { MockPrevHash: testHashPrefix + "00a5", }, } - db, err := database.New(nil, nil, "") + const testCacheSize int64 = 1 << 20 + db, err := database.New(nil, nil, "", testCacheSize) if err != nil { t.Fatalf("unexpected error creating database: %s", err) } diff --git a/config.go b/config.go index 09d6b879..698fff8f 100644 --- a/config.go +++ b/config.go @@ -31,6 +31,7 @@ import ( type ListenerConfig = connmanager.ListenerConfig type Config struct { + badgerCacheSize int64 cardanoNodeConfig *cardano.CardanoNodeConfig dataDir string intersectPoints []ocommon.Point @@ -243,3 +244,11 @@ func WithTracingStdout(stdout bool) ConfigOptionFunc { c.tracingStdout = stdout } } + +// WithBadgerCacheSize sets the maximum cache size (in bytes).This controls memory usage by limiting the size of block and index caches. +// If not set, the default size defined in internal config will be used. +func WithBadgerCacheSize(cacheSize int64) ConfigOptionFunc { + return func(c *Config) { + c.badgerCacheSize = cacheSize + } +} diff --git a/database/database.go b/database/database.go index 70f3cf3b..bfc41787 100644 --- a/database/database.go +++ b/database/database.go @@ -97,12 +97,13 @@ func New( logger *slog.Logger, promRegistry prometheus.Registerer, dataDir string, + badgerCacheSize int64, ) (*Database, error) { metadataDb, err := metadata.New("sqlite", dataDir, logger, promRegistry) if err != nil { return nil, err } - blobDb, err := blob.New("badger", dataDir, logger, promRegistry) + blobDb, err := blob.New("badger", dataDir, logger, promRegistry, badgerCacheSize) if err != nil { return nil, err } diff --git a/database/database_test.go b/database/database_test.go index c09ec25f..d04869c9 100644 --- a/database/database_test.go +++ b/database/database_test.go @@ -42,7 +42,8 @@ func TestInMemorySqliteMultipleTransaction(t *testing.T) { } return nil } - db, err := database.New(nil, nil, "") // in-memory + const testCacheSize int64 = 1 << 20 + db, err := database.New(nil, nil, "", testCacheSize) // in-memory if err != nil { t.Fatalf("unexpected error: %s", err) } diff --git a/database/plugin/blob/badger/database.go b/database/plugin/blob/badger/database.go index e66533bd..6cb4b946 100644 --- a/database/plugin/blob/badger/database.go +++ b/database/plugin/blob/badger/database.go @@ -53,6 +53,7 @@ func New( dataDir string, logger *slog.Logger, promRegistry prometheus.Registerer, + badgerCacheSize int64, ) (*BlobStoreBadger, error) { var blobDb *badger.DB var err error @@ -68,6 +69,9 @@ func New( // The default INFO logging is a bit verbose WithLoggingLevel(badger.WARNING). WithInMemory(true) + badgerOpts = badgerOpts. + WithBlockCacheSize(badgerCacheSize). + WithIndexCacheSize(badgerCacheSize) blobDb, err = badger.Open(badgerOpts) if err != nil { return nil, err @@ -93,6 +97,9 @@ func New( WithLogger(NewBadgerLogger(logger)). // The default INFO logging is a bit verbose WithLoggingLevel(badger.WARNING) + badgerOpts = badgerOpts. + WithBlockCacheSize(badgerCacheSize). + WithIndexCacheSize(badgerCacheSize) blobDb, err = badger.Open(badgerOpts) if err != nil { return nil, err diff --git a/database/plugin/blob/store.go b/database/plugin/blob/store.go index c4bb611d..f2313eea 100644 --- a/database/plugin/blob/store.go +++ b/database/plugin/blob/store.go @@ -37,6 +37,7 @@ func New( pluginName, dataDir string, logger *slog.Logger, promRegistry prometheus.Registerer, + badgerCacheSize int64, ) (BlobStore, error) { - return badgerPlugin.New(dataDir, logger, promRegistry) + return badgerPlugin.New(dataDir, logger, promRegistry, badgerCacheSize) } diff --git a/dingo.yaml.example b/dingo.yaml.example index 7282933d..d0796dfe 100644 --- a/dingo.yaml.example +++ b/dingo.yaml.example @@ -50,4 +50,9 @@ utxorpcPort: 9090 # Ignore prior chain history and start from current tip (default: false) # This is experimental and may break — use with caution -intersectTip: false \ No newline at end of file +intersectTip: false + +# Maximum cache size in bytes used by BadgerDB for block/index cache +# Helps limit memory usage by capping Badger's internal cache +# Default: 67108864 (64 MB) +badgerCacheSize: 67108864 \ No newline at end of file diff --git a/internal/config/config.go b/internal/config/config.go index 0b2c7b05..09d4973f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -43,6 +43,7 @@ func FromContext(ctx context.Context) *Config { } type Config struct { + BadgerCacheSize int64 `split_words:"true" yaml:"badgerCacheSize"` BindAddr string `split_words:"true" yaml:"bindAddr"` CardanoConfig string ` yaml:"cardanoConfig" envconfig:"config"` DatabasePath string `split_words:"true" yaml:"databasePath"` @@ -60,6 +61,7 @@ type Config struct { } var globalConfig = &Config{ + BadgerCacheSize: 67108864, BindAddr: "0.0.0.0", CardanoConfig: "./config/cardano/preview/config.json", DatabasePath: ".dingo", diff --git a/internal/config/config_test.go b/internal/config/config_test.go index fb33ef2b..6e01e1e0 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -8,6 +8,7 @@ import ( func resetGlobalConfig() { globalConfig = &Config{ + BadgerCacheSize: 67108864, BindAddr: "0.0.0.0", CardanoConfig: "./config/cardano/preview/config.json", DatabasePath: ".dingo", @@ -28,6 +29,7 @@ func resetGlobalConfig() { func TestLoad_CompareFullStruct(t *testing.T) { resetGlobalConfig() yamlContent := ` +badgerCacheSize: 8388608 bindAddr: "127.0.0.1" cardanoConfig: "./cardano/preview/config.json" databasePath: ".dingo" @@ -52,6 +54,7 @@ tlsKeyFilePath: "key1.pem" defer os.Remove(tmpFile) expected := &Config{ + BadgerCacheSize: 8388608, BindAddr: "127.0.0.1", CardanoConfig: "./cardano/preview/config.json", DatabasePath: ".dingo", @@ -92,6 +95,7 @@ func TestLoad_WithoutConfigFile_UsesDefaults(t *testing.T) { // Expected is the original default values from globalConfig expected := &Config{ + BadgerCacheSize: 67108864, BindAddr: "0.0.0.0", CardanoConfig: "./config/cardano/preview/config.json", DatabasePath: ".dingo", diff --git a/internal/node/load.go b/internal/node/load.go index 3b93512d..6ceea1a5 100644 --- a/internal/node/load.go +++ b/internal/node/load.go @@ -47,7 +47,7 @@ func Load(cfg *config.Config, logger *slog.Logger, immutableDir string) error { ) } // Load database - db, err := database.New(logger, nil, cfg.DatabasePath) + db, err := database.New(logger, nil, cfg.DatabasePath, cfg.BadgerCacheSize) if err != nil { return err } diff --git a/internal/node/node.go b/internal/node/node.go index bf2cf2da..bb3961a4 100644 --- a/internal/node/node.go +++ b/internal/node/node.go @@ -104,6 +104,7 @@ func Run(cfg *config.Config, logger *slog.Logger) error { dingo.WithIntersectTip(cfg.IntersectTip), dingo.WithLogger(logger), dingo.WithDatabasePath(cfg.DatabasePath), + dingo.WithBadgerCacheSize(cfg.BadgerCacheSize), dingo.WithNetwork(cfg.Network), dingo.WithCardanoNodeConfig(nodeCfg), dingo.WithListeners(listeners...), diff --git a/node.go b/node.go index 971f5ae5..93cd149b 100644 --- a/node.go +++ b/node.go @@ -77,7 +77,7 @@ func (n *Node) Run() error { } // Load database dbNeedsRecovery := false - db, err := database.New(n.config.logger, n.config.promRegistry, n.config.dataDir) + db, err := database.New(n.config.logger, n.config.promRegistry, n.config.dataDir, n.config.badgerCacheSize) if db == nil { n.config.logger.Error( "failed to create database", From b8a235bb2abda2fccb42998f2b31c591178f1de6 Mon Sep 17 00:00:00 2001 From: Akhil Repala Date: Sat, 14 Jun 2025 11:35:28 -0500 Subject: [PATCH 2/3] feat(config): Modified the dingo.yaml.example file regarding the badger cache size Signed-off-by: Akhil Repala --- dingo.yaml.example | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dingo.yaml.example b/dingo.yaml.example index d0796dfe..2f8148d4 100644 --- a/dingo.yaml.example +++ b/dingo.yaml.example @@ -53,6 +53,5 @@ utxorpcPort: 9090 intersectTip: false # Maximum cache size in bytes used by BadgerDB for block/index cache -# Helps limit memory usage by capping Badger's internal cache # Default: 67108864 (64 MB) -badgerCacheSize: 67108864 \ No newline at end of file +badgerCacheSize: 67108864 From 0bde56161c03475a9e7f3f3ed31836185ca8b09e Mon Sep 17 00:00:00 2001 From: Akhil Repala Date: Sun, 15 Jun 2025 22:40:35 -0500 Subject: [PATCH 3/3] feat(config): Removed in-memory cache config, chain badgerOpts calls and update default cache size to 1GB Signed-off-by: Akhil Repala --- database/plugin/blob/badger/database.go | 11 +++-------- dingo.yaml.example | 4 ++-- internal/config/config.go | 2 +- internal/config/config_test.go | 4 ++-- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/database/plugin/blob/badger/database.go b/database/plugin/blob/badger/database.go index 6cb4b946..f623891d 100644 --- a/database/plugin/blob/badger/database.go +++ b/database/plugin/blob/badger/database.go @@ -69,9 +69,6 @@ func New( // The default INFO logging is a bit verbose WithLoggingLevel(badger.WARNING). WithInMemory(true) - badgerOpts = badgerOpts. - WithBlockCacheSize(badgerCacheSize). - WithIndexCacheSize(badgerCacheSize) blobDb, err = badger.Open(badgerOpts) if err != nil { return nil, err @@ -95,11 +92,9 @@ func New( db.gcEnabled = true badgerOpts := badger.DefaultOptions(blobDir). WithLogger(NewBadgerLogger(logger)). - // The default INFO logging is a bit verbose - WithLoggingLevel(badger.WARNING) - badgerOpts = badgerOpts. - WithBlockCacheSize(badgerCacheSize). - WithIndexCacheSize(badgerCacheSize) + WithLoggingLevel(badger.WARNING). + WithBlockCacheSize(int64(float64(badgerCacheSize) * 0.75)). // 75% for block cache + WithIndexCacheSize(int64(float64(badgerCacheSize) * 0.25)) // 25% for index cache blobDb, err = badger.Open(badgerOpts) if err != nil { return nil, err diff --git a/dingo.yaml.example b/dingo.yaml.example index 2f8148d4..2a48973c 100644 --- a/dingo.yaml.example +++ b/dingo.yaml.example @@ -53,5 +53,5 @@ utxorpcPort: 9090 intersectTip: false # Maximum cache size in bytes used by BadgerDB for block/index cache -# Default: 67108864 (64 MB) -badgerCacheSize: 67108864 +# Default: 1073741824 (1 GB) +badgerCacheSize: 1073741824 diff --git a/internal/config/config.go b/internal/config/config.go index 09d4973f..91911c45 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -61,7 +61,7 @@ type Config struct { } var globalConfig = &Config{ - BadgerCacheSize: 67108864, + BadgerCacheSize: 1073741824, BindAddr: "0.0.0.0", CardanoConfig: "./config/cardano/preview/config.json", DatabasePath: ".dingo", diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6e01e1e0..cba0541e 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -8,7 +8,7 @@ import ( func resetGlobalConfig() { globalConfig = &Config{ - BadgerCacheSize: 67108864, + BadgerCacheSize: 1073741824, BindAddr: "0.0.0.0", CardanoConfig: "./config/cardano/preview/config.json", DatabasePath: ".dingo", @@ -95,7 +95,7 @@ func TestLoad_WithoutConfigFile_UsesDefaults(t *testing.T) { // Expected is the original default values from globalConfig expected := &Config{ - BadgerCacheSize: 67108864, + BadgerCacheSize: 1073741824, BindAddr: "0.0.0.0", CardanoConfig: "./config/cardano/preview/config.json", DatabasePath: ".dingo",