refactor(database): move sqlite vacuum to plugin, schedule daily

database/plugin/metadata/sqlite/database.go

@@ -22,6 +22,7 @@ import (
 	"log/slog"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/blinklabs-io/dingo/database/plugin"
 	"github.com/blinklabs-io/dingo/database/plugin/metadata/sqlite/models"
@@ -43,9 +44,10 @@ func init() {
 
 // MetadataStoreSqlite stores all data in sqlite. Data may not be persisted
 type MetadataStoreSqlite struct {
-	dataDir string
-	db      *gorm.DB
-	logger  *slog.Logger
+	dataDir     string
+	db          *gorm.DB
+	logger      *slog.Logger
+	timerVacuum *time.Timer
 }
 
 // New creates a new database
@@ -131,9 +133,56 @@ func (d *MetadataStoreSqlite) init() error {
 	if err := d.db.Use(tracing.NewPlugin(tracing.WithoutMetrics())); err != nil {
 		return err
 	}
+	// Schedule daily database vacuum to free unused space
+	d.scheduleDailyVacuum()
 	return nil
 }
 
+func (d *MetadataStoreSqlite) runVacuum() error {
+	if d.dataDir == "" {
+		return nil
+	}
+	// Open a new connection to the DB only for vacuum
+	metadataDbPath := filepath.Join(
+		d.dataDir,
+		"metadata.sqlite",
+	)
+	db, err := gorm.Open(sqlite.Open("file:" + metadataDbPath))
+	if err != nil {
+		return err
+	}
+	sqlDB, err := db.DB()
+	if err != nil {
+		return err
+	}
+	defer sqlDB.Close()
+	if result := db.Exec("VACUUM"); result.Error != nil {
+		return result.Error
+	}
+	return nil
+}
+
+func (d *MetadataStoreSqlite) scheduleDailyVacuum() {
+	if d.timerVacuum != nil {
+		d.timerVacuum.Stop()
+	}
+	daily := time.Duration(24) * time.Hour
+	f := func() {
+		d.logger.Debug(
+			"running vacuum on sqlite metadata database",
+		)
+		// schedule next run
+		defer d.scheduleDailyVacuum()
+		if err := d.runVacuum(); err != nil {
+			d.logger.Error(
+				"failed to free unused space in metadata store",
+				"component", "database",
+			)
+		}
+	}
+	d.timerVacuum = time.AfterFunc(daily, f)
+}
+
 // AutoMigrate wraps the gorm AutoMigrate
 func (d *MetadataStoreSqlite) AutoMigrate(dst ...interface{}) error {
 	return d.DB().AutoMigrate(dst...)

database/plugin/metadata/sqlite/epoch.go

@@ -16,6 +16,7 @@ package sqlite
 
 import (
 	"errors"
+	"strconv"
 
 	"github.com/blinklabs-io/dingo/database/plugin/metadata/sqlite/models"
 	"gorm.io/gorm"
@@ -88,5 +89,13 @@ func (d *MetadataStoreSqlite) SetEpoch(
 			return result.Error
 		}
 	}
+	// Run a vacuum, on error only log
+	if err := d.runVacuum(); err != nil {
+		d.logger.Error(
+			"failed to free space in metdata store",
+			"epoch", strconv.FormatUint(epoch, 10),
+			"error", err,
+		)
+	}
 	return nil
 }

state/chainsync.go

@@ -215,20 +215,6 @@ func (ls *LedgerState) processBlockEvents() error {
 	return nil
 }
 
-func (ls *LedgerState) vacuumMetadata() error {
-	ls.config.Logger.Debug(
-		"vacuuming metadata database",
-		"epoch", fmt.Sprintf("%+v", ls.currentEpoch),
-		"component", "ledger",
-	)
-	// Start a transaction
-	txn := ls.db.Transaction(true)
-	if result := txn.Metadata().Raw("VACUUM"); result.Error != nil {
-		return result.Error
-	}
-	return nil
-}
-
 func (ls *LedgerState) processGenesisBlock(
 	txn *database.Txn,
 	point ocommon.Point,
@@ -427,13 +413,6 @@ func (ls *LedgerState) processEpochRollover(
 			"epoch", fmt.Sprintf("%+v", newEpoch),
 			"component", "ledger",
 		)
-		if err := ls.vacuumMetadata(); err != nil {
-			ls.config.Logger.Error(
-				"failed to vacuum database",
-				"epoch", fmt.Sprintf("%+v", newEpoch),
-				"component", "ledger",
-			)
-		}
 	}
 	return nil
 }