diff --git a/code/go/0chain.net/blobber/config.go b/code/go/0chain.net/blobber/config.go index 0b4665494..2ac2c85ef 100644 --- a/code/go/0chain.net/blobber/config.go +++ b/code/go/0chain.net/blobber/config.go @@ -11,7 +11,7 @@ import ( ) func setupConfig() { - fmt.Print("[2/11] load config") + fmt.Print("[2/12] load config") // setup default config.SetupDefaultConfig() diff --git a/code/go/0chain.net/blobber/datastore.go b/code/go/0chain.net/blobber/datastore.go index acb0e0506..d0abc4843 100644 --- a/code/go/0chain.net/blobber/datastore.go +++ b/code/go/0chain.net/blobber/datastore.go @@ -9,11 +9,11 @@ import ( ) func setupDatabase() error { - fmt.Print("\r[7/11] connect data store") + fmt.Print("\r[7/12] connect data store") // check for database connection for i := 0; i < 600; i++ { if i > 0 { - fmt.Printf("\r[7/10] connect(%v) data store", i) + fmt.Printf("\r[7/12] connect(%v) data store", i) } if err := datastore.GetStore().Open(); err == nil { @@ -22,11 +22,19 @@ func setupDatabase() error { return err } fmt.Print(" [OK]\n") - return nil + break } time.Sleep(1 * time.Second) } + fmt.Println("\r[8/12] auto migrate datastore") + err := datastore.GetStore().AutoMigrate() + if err != nil { + logging.Logger.Error("Failed to migrate to the database.") + return err + } + //fmt.Print(" [OK]\n") + return nil } diff --git a/code/go/0chain.net/blobber/filestore.go b/code/go/0chain.net/blobber/filestore.go index 7c111af93..06b4a4eba 100644 --- a/code/go/0chain.net/blobber/filestore.go +++ b/code/go/0chain.net/blobber/filestore.go @@ -9,7 +9,7 @@ import ( var fsStore filestore.FileStore //nolint:unused // global which might be needed somewhere func setupFileStore() (err error) { - fmt.Print("[8/11] setup file store") + fmt.Print("[9/12] setup file store") fsStore, err = filestore.SetupFSStore(filesDir + "/files") diff --git a/code/go/0chain.net/blobber/flags.go b/code/go/0chain.net/blobber/flags.go index f41af64e9..838a6c5fa 100644 --- a/code/go/0chain.net/blobber/flags.go +++ b/code/go/0chain.net/blobber/flags.go @@ -44,7 +44,7 @@ func init() { } func parseFlags() { - fmt.Print("[1/11] load flags") + fmt.Print("[1/12] load flags") flag.Parse() if filesDir == "" { diff --git a/code/go/0chain.net/blobber/grpc.go b/code/go/0chain.net/blobber/grpc.go index cdb6ea692..d2aa9a1de 100644 --- a/code/go/0chain.net/blobber/grpc.go +++ b/code/go/0chain.net/blobber/grpc.go @@ -1,3 +1,4 @@ +//go:build !integration_tests // +build !integration_tests package main @@ -38,7 +39,7 @@ func startGRPCServer() { panic(err) } - fmt.Println("[10/11] starting grpc server [OK]") + fmt.Println("[11/12] starting grpc server [OK]") go func() { log.Fatal(grpcServer.Serve(lis)) }() diff --git a/code/go/0chain.net/blobber/http.go b/code/go/0chain.net/blobber/http.go index a946ebff9..398e832a9 100644 --- a/code/go/0chain.net/blobber/http.go +++ b/code/go/0chain.net/blobber/http.go @@ -38,7 +38,7 @@ func startHttpServer() { go startServer(&wg, r, mode, httpsPort, true) logging.Logger.Info("Ready to listen to the requests") - fmt.Println("[11/11] start http server [OK]") + fmt.Println("[12/12] start http server [OK]") wg.Wait() } diff --git a/code/go/0chain.net/blobber/logging.go b/code/go/0chain.net/blobber/logging.go index 33c449213..ccfa8a3cb 100644 --- a/code/go/0chain.net/blobber/logging.go +++ b/code/go/0chain.net/blobber/logging.go @@ -9,7 +9,7 @@ import ( ) func setupLogging() { - fmt.Print("[3/11] init logging") + fmt.Print("[3/12] init logging") if config.Development() { logging.InitLogging("development", logDir, "0chainBlobber.log") diff --git a/code/go/0chain.net/blobber/minio.go b/code/go/0chain.net/blobber/minio.go index 61d65b482..1d5140fa1 100644 --- a/code/go/0chain.net/blobber/minio.go +++ b/code/go/0chain.net/blobber/minio.go @@ -10,7 +10,7 @@ import ( ) func setupMinio() error { - fmt.Print("[4/11] setup minio") + fmt.Print("[4/12] setup minio") reader, err := os.Open(minioFile) if err != nil { return err diff --git a/code/go/0chain.net/blobber/node.go b/code/go/0chain.net/blobber/node.go index 072e47241..8fd1ee212 100644 --- a/code/go/0chain.net/blobber/node.go +++ b/code/go/0chain.net/blobber/node.go @@ -12,7 +12,7 @@ import ( ) func setupNode() error { - fmt.Print("[5/11] setup blobber") + fmt.Print("[5/12] setup blobber") reader, err := os.Open(keysFile) if err != nil { diff --git a/code/go/0chain.net/blobber/zcn.go b/code/go/0chain.net/blobber/zcn.go index 896635ee0..ab07abf98 100644 --- a/code/go/0chain.net/blobber/zcn.go +++ b/code/go/0chain.net/blobber/zcn.go @@ -17,7 +17,7 @@ import ( func setupOnChain() { //wait http & grpc startup, and go to setup on chain time.Sleep(1 * time.Second) - fmt.Println("[9/11] connecting to chain ") + fmt.Println("[10/12] connecting to chain ") const ATTEMPT_DELAY = 60 * 1 diff --git a/code/go/0chain.net/blobbercore/datastore/README.md b/code/go/0chain.net/blobbercore/datastore/README.md new file mode 100644 index 000000000..7b415c7b9 --- /dev/null +++ b/code/go/0chain.net/blobbercore/datastore/README.md @@ -0,0 +1,43 @@ +# How to auto migrate database schema + +## what is `version`. how it works +Given a version number TABLE.INDEX.COLUMN, increment the: + +- TABLE version when you add/drop any table, +- INDEX version when you add/drop/update and index +- COLUMN version when you add/drop/update any column + +NB: current schema that is created by sql scripts is versioned as `0.0.0`. + +## How to add a new version + +### Migrate table/column in gorm.AutoMigrate + if migration works with gorm.AutoMigrate, please use it to migrate. + - update your model + - added your model in [AutoMigrate](postgres.go#L76)if it doesn't exists + ``` + db.AutoMigrate(&Migration{},&YourModel{}) + ``` + +### Migrate index/constraints manually if it is not easy to do in `AutoMigrate` +- create a new `Migration` with scripts +- append it in [releases](postgres_releases.go#L4) +``` +var releases = []Migration{ + { + Version: "0.1.0", + CreatedAt: time.Date(2021, 10, 15, 0, 0, 0, 0, time.UTC), + Scripts: []string{ + "CREATE INDEX idx_allocation_path ON reference_objects (allocation_id,path);", + }, + }, + { + Version: "0.1.1", + CreatedAt: time.Date(2021, 10, 16, 0, 0, 0, 0, time.UTC), + Scripts: []string{ + "sql1", + "sql2", + }, + }, +} +``` \ No newline at end of file diff --git a/code/go/0chain.net/blobbercore/datastore/mocket.go b/code/go/0chain.net/blobbercore/datastore/mocket.go index 3e8f9be7a..be6c115b1 100644 --- a/code/go/0chain.net/blobbercore/datastore/mocket.go +++ b/code/go/0chain.net/blobbercore/datastore/mocket.go @@ -84,3 +84,7 @@ func (store *Mocket) GetTransaction(ctx context.Context) *gorm.DB { func (store *Mocket) GetDB() *gorm.DB { return store.db } + +func (store *Mocket) AutoMigrate() error { + return nil +} diff --git a/code/go/0chain.net/blobbercore/datastore/postgres.go b/code/go/0chain.net/blobbercore/datastore/postgres.go index 98a0db1f0..71ec3b440 100644 --- a/code/go/0chain.net/blobbercore/datastore/postgres.go +++ b/code/go/0chain.net/blobbercore/datastore/postgres.go @@ -7,7 +7,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/core/common" - . "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" "gorm.io/driver/postgres" "gorm.io/gorm" ) @@ -22,7 +23,10 @@ func (store *postgresStore) Open() error { "host=%v port=%v user=%v dbname=%v password=%v sslmode=disable", config.Configuration.DBHost, config.Configuration.DBPort, config.Configuration.DBUserName, config.Configuration.DBName, - config.Configuration.DBPassword)), &gorm.Config{}) + config.Configuration.DBPassword)), &gorm.Config{ + SkipDefaultTransaction: true, // https://gorm.io/docs/performance.html#Disable-Default-Transaction + PrepareStmt: true, //https://gorm.io/docs/performance.html#Caches-Prepared-Statement + }) if err != nil { return common.NewErrorf("db_open_error", "Error opening the DB connection: %v", err) } @@ -59,10 +63,65 @@ func (store *postgresStore) GetTransaction(ctx context.Context) *gorm.DB { if conn != nil { return conn.(*gorm.DB) } - Logger.Error("No connection in the context.") + logging.Logger.Error("No connection in the context.") return nil } func (store *postgresStore) GetDB() *gorm.DB { return store.db } + +func (store *postgresStore) AutoMigrate() error { + + err := store.db.AutoMigrate(&Migration{}) + if err != nil { + logging.Logger.Error("[db]", zap.Error(err)) + } + + if len(releases) == 0 { + fmt.Print(" + No releases [SKIP]\n") + return nil + } + + for i := 0; i < len(releases); i++ { + v := releases[i] + fmt.Print("\r + ", v.Version, " ") + isMigrated, err := store.IsMigrated(v) + if err != nil { + return err + } + if isMigrated { + fmt.Print(" [SKIP]\n") + continue + } + + err = v.Migrate(store.db) + if err != nil { + logging.Logger.Error("[db]"+v.Version, zap.Error(err)) + return err + } + + logging.Logger.Info("[db]" + v.Version + " migrated") + fmt.Print(" [OK]\n") + + } + return nil +} + +func (store *postgresStore) IsMigrated(m Migration) (bool, error) { + var version string + err := store.db. + Raw(`select version from "migrations" where version=?`, m.Version). + Pluck("version", &version). + Error + + if err == nil { + return false, nil + } + + if err == gorm.ErrRecordNotFound { + return false, nil + } + + return false, err +} diff --git a/code/go/0chain.net/blobbercore/datastore/postgres_migration.go b/code/go/0chain.net/blobbercore/datastore/postgres_migration.go new file mode 100644 index 000000000..153fd401c --- /dev/null +++ b/code/go/0chain.net/blobbercore/datastore/postgres_migration.go @@ -0,0 +1,45 @@ +package datastore + +import ( + "time" + + "gorm.io/gorm" +) + +const ( + TableNameMigration = "migrations" +) + +// Migration migration history +type Migration struct { + // Version format is "[table].[index].[column]" + // + increase table version if any table is added or deleted + // + increase index version if any index is changed + // + increase column version if any column/constraint is changed + Version string `gorm:"column:version;primary_key"` + CreatedAt time.Time `gorm:"column:created_at"` + Scripts []string `gorm:"-"` +} + +// Migrate migrate database +func (m *Migration) Migrate(db *gorm.DB) error { + return db.Transaction(func(tx *gorm.DB) error { + + for _, s := range m.Scripts { + if err := tx.Exec(s).Error; err != nil { + return err + } + } + + if err := tx.Create(m).Error; err != nil { + return err + } + + return nil + }) +} + +// TableName get table name of migrate +func (Migration) TableName() string { + return TableNameMigration +} diff --git a/code/go/0chain.net/blobbercore/datastore/postgres_releases.go b/code/go/0chain.net/blobbercore/datastore/postgres_releases.go new file mode 100644 index 000000000..e6b83695d --- /dev/null +++ b/code/go/0chain.net/blobbercore/datastore/postgres_releases.go @@ -0,0 +1,12 @@ +package datastore + +// releases migration histories +var releases = []Migration{ + // { + // Version: "0.1.0", + // CreatedAt: time.Date(2021, 10, 15, 0, 0, 0, 0, time.UTC), + // Scripts: []string{ + // "CREATE INDEX idx_allocation_path ON reference_objects (allocation_id,path,deleted_at);", + // }, + // }, +} diff --git a/code/go/0chain.net/blobbercore/datastore/sqlmock.go b/code/go/0chain.net/blobbercore/datastore/sqlmock.go index 1dc998035..4a0f8d446 100644 --- a/code/go/0chain.net/blobbercore/datastore/sqlmock.go +++ b/code/go/0chain.net/blobbercore/datastore/sqlmock.go @@ -79,3 +79,7 @@ func (store *Sqlmock) GetTransaction(ctx context.Context) *gorm.DB { func (store *Sqlmock) GetDB() *gorm.DB { return store.db } + +func (store *Sqlmock) AutoMigrate() error { + return nil +} diff --git a/code/go/0chain.net/blobbercore/datastore/store.go b/code/go/0chain.net/blobbercore/datastore/store.go index bfaae071d..2cf8189e5 100644 --- a/code/go/0chain.net/blobbercore/datastore/store.go +++ b/code/go/0chain.net/blobbercore/datastore/store.go @@ -24,6 +24,7 @@ type Store interface { Open() error Close() + AutoMigrate() error } var instance Store diff --git a/sql/00-create-user.sql b/sql/00-create-user.sql index 50d7988a8..a6696c1a4 100644 --- a/sql/00-create-user.sql +++ b/sql/00-create-user.sql @@ -3,4 +3,5 @@ CREATE DATABASE blobber_meta; \connect blobber_meta; CREATE USER blobber_user WITH ENCRYPTED PASSWORD 'blobber'; -GRANT ALL PRIVILEGES ON DATABASE blobber_meta TO blobber_user; \ No newline at end of file +GRANT ALL PRIVILEGES ON DATABASE blobber_meta TO blobber_user; +grant postgres to blobber_user; \ No newline at end of file