Feat/db auto migrate

code/go/0chain.net/blobber/config.go

@@ -11,7 +11,7 @@ import (
 )
 
 func setupConfig() {
-	fmt.Print("[2/11] load config")
+	fmt.Print("[2/12] load config")
 	// setup default
 	config.SetupDefaultConfig()
 

code/go/0chain.net/blobber/datastore.go

@@ -9,11 +9,11 @@ import (
 )
 
 func setupDatabase() error {
-	fmt.Print("\r[7/11] connect data store")
+	fmt.Print("\r[7/12] connect data store")
 	// check for database connection
 	for i := 0; i < 600; i++ {
 		if i > 0 {
-			fmt.Printf("\r[7/10] connect(%v) data store", i)
+			fmt.Printf("\r[7/12] connect(%v) data store", i)
 		}
 
 		if err := datastore.GetStore().Open(); err == nil {
@@ -22,11 +22,19 @@ func setupDatabase() error {
 				return err
 			}
 			fmt.Print("	[OK]\n")
-			return nil
+			break
 		}
 
 		time.Sleep(1 * time.Second)
 	}
 
+	fmt.Println("\r[8/12] auto migrate datastore")
+	err := datastore.GetStore().AutoMigrate()
+	if err != nil {
+		logging.Logger.Error("Failed to migrate to the database.")
+		return err
+	}
+	//fmt.Print("	[OK]\n")
+
 	return nil
 }

code/go/0chain.net/blobber/filestore.go

@@ -9,7 +9,7 @@ import (
 var fsStore filestore.FileStore //nolint:unused // global which might be needed somewhere
 
 func setupFileStore() (err error) {
-	fmt.Print("[8/11] setup file store")
+	fmt.Print("[9/12] setup file store")
 
 	fsStore, err = filestore.SetupFSStore(filesDir + "/files")
 

code/go/0chain.net/blobber/flags.go

@@ -44,7 +44,7 @@ func init() {
 }
 
 func parseFlags() {
-	fmt.Print("[1/11] load flags")
+	fmt.Print("[1/12] load flags")
 	flag.Parse()
 
 	if filesDir == "" {

code/go/0chain.net/blobber/grpc.go

@@ -1,3 +1,4 @@
+//go:build !integration_tests
 // +build !integration_tests
 
 package main
@@ -38,7 +39,7 @@ func startGRPCServer() {
 		panic(err)
 	}
 
-	fmt.Println("[10/11] starting grpc server	[OK]")
+	fmt.Println("[11/12] starting grpc server	[OK]")
 	go func() {
 		log.Fatal(grpcServer.Serve(lis))
 	}()

code/go/0chain.net/blobber/http.go

@@ -38,7 +38,7 @@ func startHttpServer() {
 	go startServer(&wg, r, mode, httpsPort, true)
 
 	logging.Logger.Info("Ready to listen to the requests")
-	fmt.Println("[11/11] start http server	[OK]")
+	fmt.Println("[12/12] start http server	[OK]")
 
 	wg.Wait()
 }

code/go/0chain.net/blobber/logging.go

@@ -9,7 +9,7 @@ import (
 )
 
 func setupLogging() {
-	fmt.Print("[3/11] init logging")
+	fmt.Print("[3/12] init logging")
 
 	if config.Development() {
 		logging.InitLogging("development", logDir, "0chainBlobber.log")

code/go/0chain.net/blobber/minio.go

@@ -10,7 +10,7 @@ import (
 )
 
 func setupMinio() error {
-	fmt.Print("[4/11] setup minio")
+	fmt.Print("[4/12] setup minio")
 	reader, err := os.Open(minioFile)
 	if err != nil {
 		return err

code/go/0chain.net/blobber/node.go

@@ -12,7 +12,7 @@ import (
 )
 
 func setupNode() error {
-	fmt.Print("[5/11] setup blobber")
+	fmt.Print("[5/12] setup blobber")
 
 	reader, err := os.Open(keysFile)
 	if err != nil {

code/go/0chain.net/blobber/zcn.go

@@ -17,7 +17,7 @@ import (
 func setupOnChain() {
 	//wait http & grpc startup, and go to setup on chain
 	time.Sleep(1 * time.Second)
-	fmt.Println("[9/11] connecting to chain	")
+	fmt.Println("[10/12] connecting to chain	")
 
 	const ATTEMPT_DELAY = 60 * 1
 

code/go/0chain.net/blobbercore/datastore/README.md

@@ -0,0 +1,43 @@
+# How to auto migrate database schema 
+
+## what is `version`. how it works
+Given a version number TABLE.INDEX.COLUMN, increment the:
+
+- TABLE version when you add/drop any table,
+- INDEX version when you add/drop/update and index
+- COLUMN version when you add/drop/update any column
+
+NB: current schema that is created by sql scripts is versioned as `0.0.0`. 
+
+## How to add a new version
+
+### Migrate table/column in gorm.AutoMigrate
+ if migration works with gorm.AutoMigrate, please use it to migrate.
+ - update your model
+ - added your model in [AutoMigrate](postgres.go#L76)if it doesn't exists
+  ```
+   db.AutoMigrate(&Migration{},&YourModel{})
+  ```
+
+### Migrate index/constraints manually if it is not easy to do in `AutoMigrate`
+- create a new `Migration` with scripts
+- append it in [releases](postgres_releases.go#L4) 
+```
+var releases = []Migration{
+	{
+		Version:   "0.1.0",
+		CreatedAt: time.Date(2021, 10, 15, 0, 0, 0, 0, time.UTC),
+		Scripts: []string{
+			"CREATE INDEX idx_allocation_path ON reference_objects (allocation_id,path);",
+		},
+	},
+    {
+		Version:   "0.1.1",
+		CreatedAt: time.Date(2021, 10, 16, 0, 0, 0, 0, time.UTC),
+		Scripts: []string{
+			"sql1",
+            "sql2",
+		},
+	},
+}
+```

code/go/0chain.net/blobbercore/datastore/mocket.go

@@ -84,3 +84,7 @@ func (store *Mocket) GetTransaction(ctx context.Context) *gorm.DB {
 func (store *Mocket) GetDB() *gorm.DB {
 	return store.db
 }
+
+func (store *Mocket) AutoMigrate() error {
+	return nil
+}

code/go/0chain.net/blobbercore/datastore/postgres.go

@@ -7,7 +7,8 @@ import (
 
 	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
 	"github.com/0chain/blobber/code/go/0chain.net/core/common"
-	. "github.com/0chain/blobber/code/go/0chain.net/core/logging"
+	"github.com/0chain/blobber/code/go/0chain.net/core/logging"
+	"go.uber.org/zap"
 	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
 )
@@ -22,7 +23,10 @@ func (store *postgresStore) Open() error {
 		"host=%v port=%v user=%v dbname=%v password=%v sslmode=disable",
 		config.Configuration.DBHost, config.Configuration.DBPort,
 		config.Configuration.DBUserName, config.Configuration.DBName,
-		config.Configuration.DBPassword)), &gorm.Config{})
+		config.Configuration.DBPassword)), &gorm.Config{
+		SkipDefaultTransaction: true, // https://gorm.io/docs/performance.html#Disable-Default-Transaction
+		PrepareStmt:            true, //https://gorm.io/docs/performance.html#Caches-Prepared-Statement
+	})
 	if err != nil {
 		return common.NewErrorf("db_open_error", "Error opening the DB connection: %v", err)
 	}
@@ -59,10 +63,65 @@ func (store *postgresStore) GetTransaction(ctx context.Context) *gorm.DB {
 	if conn != nil {
 		return conn.(*gorm.DB)
 	}
-	Logger.Error("No connection in the context.")
+	logging.Logger.Error("No connection in the context.")
 	return nil
 }
 
 func (store *postgresStore) GetDB() *gorm.DB {
 	return store.db
 }
+
+func (store *postgresStore) AutoMigrate() error {
+
+	err := store.db.AutoMigrate(&Migration{})
+	if err != nil {
+		logging.Logger.Error("[db]", zap.Error(err))
+	}
+
+	if len(releases) == 0 {
+		fmt.Print("	+ No releases 	[SKIP]\n")
+		return nil
+	}
+
+	for i := 0; i < len(releases); i++ {
+		v := releases[i]
+		fmt.Print("\r	+ ", v.Version, "	")
+		isMigrated, err := store.IsMigrated(v)
+		if err != nil {
+			return err
+		}
+		if isMigrated {
+			fmt.Print("	[SKIP]\n")
+			continue
+		}
+
+		err = v.Migrate(store.db)
+		if err != nil {
+			logging.Logger.Error("[db]"+v.Version, zap.Error(err))
+			return err
+		}
+
+		logging.Logger.Info("[db]" + v.Version + " migrated")
+		fmt.Print("	[OK]\n")
+
+	}
+	return nil
+}
+
+func (store *postgresStore) IsMigrated(m Migration) (bool, error) {
+	var version string
+	err := store.db.
+		Raw(`select version from "migrations" where version=?`, m.Version).
+		Pluck("version", &version).
+		Error
+
+	if err == nil {
+		return false, nil
+	}
+
+	if err == gorm.ErrRecordNotFound {
+		return false, nil
+	}
+
+	return false, err
+}

code/go/0chain.net/blobbercore/datastore/postgres_migration.go

@@ -0,0 +1,45 @@
+package datastore
+
+import (
+	"time"
+
+	"gorm.io/gorm"
+)
+
+const (
+	TableNameMigration = "migrations"
+)
+
+// Migration migration history
+type Migration struct {
+	// Version format is "[table].[index].[column]"
+	//  + increase table version if any table is added or deleted
+	//  + increase index version if any index is changed
+	//  + increase column version if any column/constraint is changed
+	Version   string    `gorm:"column:version;primary_key"`
+	CreatedAt time.Time `gorm:"column:created_at"`
+	Scripts   []string  `gorm:"-"`
+}
+
+// Migrate migrate database
+func (m *Migration) Migrate(db *gorm.DB) error {
+	return db.Transaction(func(tx *gorm.DB) error {
+
+		for _, s := range m.Scripts {
+			if err := tx.Exec(s).Error; err != nil {
+				return err
+			}
+		}
+
+		if err := tx.Create(m).Error; err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+// TableName get table name of migrate
+func (Migration) TableName() string {
+	return TableNameMigration
+}

code/go/0chain.net/blobbercore/datastore/postgres_releases.go

@@ -0,0 +1,12 @@
+package datastore
+
+// releases migration histories
+var releases = []Migration{
+	// {
+	// 	Version:   "0.1.0",
+	// 	CreatedAt: time.Date(2021, 10, 15, 0, 0, 0, 0, time.UTC),
+	// 	Scripts: []string{
+	// 		"CREATE INDEX idx_allocation_path ON reference_objects (allocation_id,path,deleted_at);",
+	// 	},
+	// },
+}

code/go/0chain.net/blobbercore/datastore/sqlmock.go

@@ -79,3 +79,7 @@ func (store *Sqlmock) GetTransaction(ctx context.Context) *gorm.DB {
 func (store *Sqlmock) GetDB() *gorm.DB {
 	return store.db
 }
+
+func (store *Sqlmock) AutoMigrate() error {
+	return nil
+}

code/go/0chain.net/blobbercore/datastore/store.go

@@ -24,6 +24,7 @@ type Store interface {
 
 	Open() error
 	Close()
+	AutoMigrate() error
 }
 
 var instance Store

sql/00-create-user.sql

@@ -3,4 +3,5 @@ CREATE DATABASE blobber_meta;
 
 \connect blobber_meta;
 CREATE USER blobber_user WITH ENCRYPTED PASSWORD 'blobber';
-GRANT ALL PRIVILEGES ON DATABASE blobber_meta TO blobber_user;
+GRANT ALL PRIVILEGES ON DATABASE blobber_meta TO blobber_user;
+grant postgres to blobber_user;