diff --git a/database/plugin/metadata/sqlite/database.go b/database/plugin/metadata/sqlite/database.go index 6e015f89..a3cfef33 100644 --- a/database/plugin/metadata/sqlite/database.go +++ b/database/plugin/metadata/sqlite/database.go @@ -22,6 +22,7 @@ import ( "log/slog" "os" "path/filepath" + "time" "github.com/blinklabs-io/dingo/database/plugin" "github.com/blinklabs-io/dingo/database/plugin/metadata/sqlite/models" @@ -43,9 +44,10 @@ func init() { // MetadataStoreSqlite stores all data in sqlite. Data may not be persisted type MetadataStoreSqlite struct { - dataDir string - db *gorm.DB - logger *slog.Logger + dataDir string + db *gorm.DB + logger *slog.Logger + timerVacuum *time.Timer } // New creates a new database @@ -131,9 +133,56 @@ func (d *MetadataStoreSqlite) init() error { if err := d.db.Use(tracing.NewPlugin(tracing.WithoutMetrics())); err != nil { return err } + // Schedule daily database vacuum to free unused space + d.scheduleDailyVacuum() return nil } +func (d *MetadataStoreSqlite) runVacuum() error { + if d.dataDir == "" { + return nil + } + // Open a new connection to the DB only for vacuum + metadataDbPath := filepath.Join( + d.dataDir, + "metadata.sqlite", + ) + db, err := gorm.Open(sqlite.Open("file:" + metadataDbPath)) + if err != nil { + return err + } + sqlDB, err := db.DB() + if err != nil { + return err + } + defer sqlDB.Close() + if result := db.Exec("VACUUM"); result.Error != nil { + return result.Error + } + return nil +} + +func (d *MetadataStoreSqlite) scheduleDailyVacuum() { + if d.timerVacuum != nil { + d.timerVacuum.Stop() + } + daily := time.Duration(24) * time.Hour + f := func() { + d.logger.Debug( + "running vacuum on sqlite metadata database", + ) + // schedule next run + defer d.scheduleDailyVacuum() + if err := d.runVacuum(); err != nil { + d.logger.Error( + "failed to free unused space in metadata store", + "component", "database", + ) + } + } + d.timerVacuum = time.AfterFunc(daily, f) +} + // AutoMigrate wraps the gorm AutoMigrate func (d *MetadataStoreSqlite) AutoMigrate(dst ...interface{}) error { return d.DB().AutoMigrate(dst...) diff --git a/database/plugin/metadata/sqlite/epoch.go b/database/plugin/metadata/sqlite/epoch.go index 0b53e411..adacbb83 100644 --- a/database/plugin/metadata/sqlite/epoch.go +++ b/database/plugin/metadata/sqlite/epoch.go @@ -16,6 +16,7 @@ package sqlite import ( "errors" + "strconv" "github.com/blinklabs-io/dingo/database/plugin/metadata/sqlite/models" "gorm.io/gorm" @@ -88,5 +89,13 @@ func (d *MetadataStoreSqlite) SetEpoch( return result.Error } } + // Run a vacuum, on error only log + if err := d.runVacuum(); err != nil { + d.logger.Error( + "failed to free space in metdata store", + "epoch", strconv.FormatUint(epoch, 10), + "error", err, + ) + } return nil } diff --git a/state/chainsync.go b/state/chainsync.go index 45990636..ac6fc912 100644 --- a/state/chainsync.go +++ b/state/chainsync.go @@ -215,20 +215,6 @@ func (ls *LedgerState) processBlockEvents() error { return nil } -func (ls *LedgerState) vacuumMetadata() error { - ls.config.Logger.Debug( - "vacuuming metadata database", - "epoch", fmt.Sprintf("%+v", ls.currentEpoch), - "component", "ledger", - ) - // Start a transaction - txn := ls.db.Transaction(true) - if result := txn.Metadata().Raw("VACUUM"); result.Error != nil { - return result.Error - } - return nil -} - func (ls *LedgerState) processGenesisBlock( txn *database.Txn, point ocommon.Point, @@ -427,13 +413,6 @@ func (ls *LedgerState) processEpochRollover( "epoch", fmt.Sprintf("%+v", newEpoch), "component", "ledger", ) - if err := ls.vacuumMetadata(); err != nil { - ls.config.Logger.Error( - "failed to vacuum database", - "epoch", fmt.Sprintf("%+v", newEpoch), - "component", "ledger", - ) - } } return nil }