diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 04953f51..1f718f9a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,10 +32,40 @@ # use the pull.rebase config option to change the behavior for every git pull (instead of only newly-created branches) git config pull.rebase true ``` +3. 如果您提交的代码涉及到数据迁移，请在`internal/migrations`目录下添加迁移脚本，并在`migration.go`中注册该脚本 + ```go + type MigrationXXXXX struct { + } + + func (m MigrationXXXXX) GetCreateAt() time.Time{ + //时间请设定为提交代码的当前时间，以便于系统自动记录迁移历史 + return time.Date(2022, 1, 1, 0, 0, 0, 0, time.Local) + } + func (m Migration20220101) Upgrade(ormer orm.Ormer) error{ + //样例代码如下，err不为nil时，会自动回滚所有操作 + /* + err := ormer.Raw(xxxx).Exec() + return err + */ + + } -3. Create a pull request to the main repository on GitHub. -4. When the reviewer makes some comments, address any feedback that comes and update the pull request. -5. When your contribution is accepted, your pull request will be approved and merged to the main branch. + ``` + + ```go + //在migrations.go中 + func InitMigration() { + migrationTypes := MigrationTypes{ + //注册 + new(MigrationXXXXX), + //... + } + + } + ``` +4. Create a pull request to the main repository on GitHub. +5. When the reviewer makes some comments, address any feedback that comes and update the pull request. +6. When your contribution is accepted, your pull request will be approved and merged to the main branch. ### 2. 文档贡献 diff --git a/deploy/docker-compose/init.sh b/deploy/docker-compose/init.sh index 8a0ca21d..db8d3a77 100644 --- a/deploy/docker-compose/init.sh +++ b/deploy/docker-compose/init.sh @@ -30,16 +30,5 @@ then echo "$NOT_UP restart failed, please check docker log use 'docker-compose logs [docker-name]'" fi -MYSQL_DB=$(grep "MYSQL_DATABASE" docker-compose.yml | awk -F':' '{print $2}') -MYSQL_DB_STRIP=`echo ${MYSQL_DB} | sed 's/ //g'` - -MYSQL_PASSWORD=$(grep "MYSQL_ROOT_PASSWORD" docker-compose.yml | awk -F':' '{print $2}') -MYSQL_PASSWORD_STRIP=$(echo $MYSQL_PASSWORD | sed 's/ //g') - -echo "mysql database: $MYSQL_DB" -echo "root password: $MYSQL_PASSWORD_STRIP" - -docker exec mysql mysql -uroot -p$MYSQL_PASSWORD_STRIP $MYSQL_DB_STRIP < mysql/sql/v1.3.2_00.sql - # init result verify [ $? -eq 0 ] && echo -e "AtomCI 初始化成功(:\n\n访问atomci: http://localhost:8090 \n" || echo -e "AtomCI 初始化失败, 请确认atomci 容器日志，\n或是 https://github.com/go-atomci/atomci-press/issues/new 反馈你的问题(:" diff --git a/deploy/release/sql/v1.3.2_00.sql b/deploy/release/sql/v1.3.2_00.sql deleted file mode 100644 index 91b73f0e..00000000 --- a/deploy/release/sql/v1.3.2_00.sql +++ /dev/null @@ -1,26 +0,0 @@ -# support normal docker registry -UPDATE `sys_integrate_setting` SET `type`='registry' WHERE `type`='harbor'; - - -# modify table project_env column harbor to registry -DROP PROCEDURE IF EXISTS `ModifyHarborToRegistry`; -delimiter $$ -CREATE PROCEDURE `ModifyHarborToRegistry`() -BEGIN - DECLARE HARBOREXISTS int DEFAULT 0; - DECLARE REGISTRYEXISTS int DEFAULT 0; - SELECT count(1) INTO @HARBOREXISTS FROM information_schema.COLUMNS WHERE TABLE_NAME='project_env' AND COLUMN_NAME='harbor'; - SELECT count(1) INTO @REGISTRYEXISTS FROM information_schema.COLUMNS WHERE TABLE_NAME='project_env' AND COLUMN_NAME='registry'; - IF @HARBOREXISTS>0 AND @REGISTRYEXISTS=0 #存在harbor列 不存在registry列时 直接修改列名 - THEN - ALTER TABLE `project_env` CHANGE COLUMN `harbor` `registry` bigint(20) NOT NULL DEFAULT 0; - ELSEIF @HARBOREXISTS>0 AND @REGISTRYEXISTS>0 #harbor列和registry都存在时迁移数据并删除harbor列 - THEN - UPDATE `project_env` SET `registry`=`harbor`; - ALTER TABLE `project_env` DROP COLUMN `harbor`; - END IF; -END; -$$ -delimiter ; -CALL `ModifyHarborToRegistry`; -DROP PROCEDURE IF EXISTS `ModifyHarborToRegistry`; \ No newline at end of file diff --git a/internal/migrations/migration20220101.go b/internal/migrations/migration20220101.go new file mode 100644 index 00000000..aefa4e1f --- /dev/null +++ b/internal/migrations/migration20220101.go @@ -0,0 +1,84 @@ +package migrations + +import ( + "fmt" + "github.com/astaxie/beego/orm" + "github.com/go-atomci/atomci/internal/middleware/log" + "time" +) + +type Migration20220101 struct { +} + +func (m Migration20220101) GetCreateAt() time.Time { + return time.Date(2022, 1, 1, 0, 0, 0, 0, time.Local) +} + +func (m Migration20220101) Upgrade(ormer orm.Ormer) error { + tables := []string{ + "sys_resource_type", + "sys_resource_operation", + "sys_resource_constraint", + "sys_user", + "sys_group", + "sys_group_user_rel", + "sys_group_user_constraint", + "sys_group_role", + "sys_group_role_user", + "sys_group_role_operation", + "sys_audit", + "sys_resource_router", + } + + if err := setCreateAt(ormer, tables); err != nil { + log.Log.Error(err.Error()) + return err + } + if err := setUpdateAt(ormer, tables); err != nil { + log.Log.Error(err.Error()) + return err + } + return nil +} + +func setCreateAt(ormer orm.Ormer, tables []string) error { + for _, table := range tables { + var count int + sql := `SELECT count(1) FROM INFORMATION_SCHEMA.Columns WHERE table_schema=DATABASE() AND table_name=? + AND column_name='create_at' AND COLUMN_DEFAULT='CURRENT_TIMESTAMP'` + if err := ormer.Raw(sql, table).QueryRow(&count); err != nil { + return err + } + if count == 0 { + sql = `alter table ` + table + ` modify column create_at datetime not null DEFAULT CURRENT_TIMESTAMP` + if _, err := ormer.Raw(sql).Exec(); err != nil { + return err + } + log.Log.Info(sql) + } else { + log.Log.Debug(fmt.Sprintf("table `%v` already alter create_at, skip", table)) + } + } + return nil +} + +func setUpdateAt(ormer orm.Ormer, tables []string) error { + for _, table := range tables { + var count int + sql := `SELECT count(1) FROM INFORMATION_SCHEMA.Columns WHERE table_schema=DATABASE() AND table_name=? + AND column_name='update_at' AND COLUMN_DEFAULT='CURRENT_TIMESTAMP' AND EXTRA='on update CURRENT_TIMESTAMP'` + if err := ormer.Raw(sql, table).QueryRow(&count); err != nil { + return err + } + if count == 0 { + sql = `alter table ` + table + ` modify column update_at datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP` + if _, err := ormer.Raw(sql).Exec(); err != nil { + return err + } + log.Log.Info(sql) + } else { + log.Log.Debug(fmt.Sprintf("table `%v` already alter update_at, skip", table)) + } + } + return nil +} diff --git a/internal/migrations/migration20220309.go b/internal/migrations/migration20220309.go new file mode 100644 index 00000000..22ea2490 --- /dev/null +++ b/internal/migrations/migration20220309.go @@ -0,0 +1,46 @@ +package migrations + +import ( + "github.com/astaxie/beego/orm" + "strings" + "time" +) + +type Migration20220309 struct { +} + +func (m Migration20220309) GetCreateAt() time.Time { + return time.Date(2022, 3, 9, 0, 0, 0, 0, time.Local) +} + +func (m Migration20220309) Upgrade(ormer orm.Ormer) error { + _, err := ormer.Raw("UPDATE `sys_integrate_setting` SET `type`='registry' WHERE `type`='harbor';").Exec() + if err != nil { + return err + } + + _, err = ormer.Raw("DROP PROCEDURE IF EXISTS `ModifyHarborToRegistry`;").Exec() + _, err = ormer.Raw(strings.ReplaceAll(`CREATE PROCEDURE <|SPIT|>ModifyHarborToRegistry<|SPIT|>() +BEGIN + DECLARE HARBOREXISTS int DEFAULT 0; + DECLARE REGISTRYEXISTS int DEFAULT 0; + SELECT count(1) INTO @HARBOREXISTS FROM information_schema.COLUMNS WHERE TABLE_NAME='project_env' AND COLUMN_NAME='harbor'; + SELECT count(1) INTO @REGISTRYEXISTS FROM information_schema.COLUMNS WHERE TABLE_NAME='project_env' AND COLUMN_NAME='registry'; + IF @HARBOREXISTS>0 AND @REGISTRYEXISTS=0 + THEN + ALTER TABLE <|SPIT|>project_env<|SPIT|> CHANGE COLUMN <|SPIT|>harbor<|SPIT|> <|SPIT|>registry<|SPIT|> bigint(20) NOT NULL DEFAULT 0; + ELSEIF @HARBOREXISTS>0 AND @REGISTRYEXISTS>0 + THEN + UPDATE <|SPIT|>project_env<|SPIT|> SET <|SPIT|>registry<|SPIT|>=<|SPIT|>harbor<|SPIT|>; + ALTER TABLE <|SPIT|>project_env<|SPIT|> DROP COLUMN <|SPIT|>harbor<|SPIT|>; + END IF; +END;`, "<|SPIT|>", "`")).Exec() + + _, err = ormer.Raw("CALL `ModifyHarborToRegistry`;").Exec() + _, err = ormer.Raw("DROP PROCEDURE IF EXISTS `ModifyHarborToRegistry`;").Exec() + + if err != nil { + return err + } + return nil +} diff --git a/internal/migrations/migrations.go b/internal/migrations/migrations.go new file mode 100644 index 00000000..85ca69f9 --- /dev/null +++ b/internal/migrations/migrations.go @@ -0,0 +1,97 @@ +package migrations + +import ( + "github.com/astaxie/beego/orm" + "sort" + "time" +) + +type MigrationTypes []Migration + +// Migration db migration base interface +type Migration interface { + GetCreateAt() time.Time + Upgrade(ormer orm.Ormer) error +} + +// Len 排序三人组 +func (t MigrationTypes) Len() int { + return len(t) +} + +// Less 排序三人组 +func (t MigrationTypes) Less(i, j int) bool { + return t[i].GetCreateAt().Before(t[j].GetCreateAt()) +} + +// Swap 排序三人组 +func (t MigrationTypes) Swap(i, j int) { + t[i], t[j] = t[j], t[i] +} + +// InitMigration db migration register +func InitMigration() { + migrationTypes := MigrationTypes{ + new(Migration20220101), + new(Migration20220309), + } + + migrateInTx(migrationTypes) +} + +func migrateInTx(migrationTypes MigrationTypes) { + //升序 + sort.Sort(migrationTypes) + + //数据迁移(事务） + ormClient := orm.NewOrm() + last := getNewestData(ormClient) + tempLast := last + errRet := ormClient.Begin() + for _, m := range migrationTypes { + if m.GetCreateAt().After(last) { + errRet = m.Upgrade(ormClient) + if errRet != nil { + break + } + } + tempLast = m.GetCreateAt() + } + errRet = updateNewestData(ormClient, tempLast) + if errRet != nil { + ormClient.Rollback() + } else { + ormClient.Commit() + } +} + +func getNewestData(ormer orm.Ormer) time.Time { + sureCreateTable(ormer) + sql := `Select * From __dbmigration Limit 1` + var lastMigrationDate time.Time + ormer.Raw(sql).QueryRow(&lastMigrationDate) + if lastMigrationDate.IsZero() { + lastMigrationDate = time.Now() + } + return lastMigrationDate +} + +func updateNewestData(ormer orm.Ormer, lastTime time.Time) error { + countSql := "Select count(*) from __dbmigration" + var count int + ormer.Raw(countSql).QueryRow(&count) + + sql := "Update __dbmigration set last_migration_date=?" + if count == 0 { + sql = "Insert into __dbmigration(last_migration_date) values (?)" + } + _, err := ormer.Raw(sql, lastTime).Exec() + return err +} + +func sureCreateTable(ormer orm.Ormer) { + ddl := `CREATE TABLE IF NOT EXISTS __dbmigration ( + last_migration_date datetime DEFAULT CURRENT_TIMESTAMP + )` + ormer.Raw(ddl).Exec() +} diff --git a/internal/models/models.go b/internal/models/models.go index fe6f4cd1..64561e50 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -18,11 +18,10 @@ package models import ( "fmt" + "github.com/go-atomci/atomci/internal/migrations" "os" "time" - "github.com/go-atomci/atomci/internal/middleware/log" - "github.com/astaxie/beego" "github.com/astaxie/beego/orm" "github.com/go-sql-driver/mysql" @@ -66,50 +65,6 @@ var ( tableNames []string ) -func setCreateAt(tables []string) error { - for _, table := range tables { - var count int - sql := `SELECT count(1) FROM INFORMATION_SCHEMA.Columns WHERE table_schema=DATABASE() AND table_name=? - AND column_name='create_at' AND COLUMN_DEFAULT='CURRENT_TIMESTAMP'` - ormer := orm.NewOrm() - if err := ormer.Raw(sql, table).QueryRow(&count); err != nil { - return err - } - if count == 0 { - sql = `alter table ` + table + ` modify column create_at datetime not null DEFAULT CURRENT_TIMESTAMP` - if _, err := ormer.Raw(sql).Exec(); err != nil { - return err - } - log.Log.Info(sql) - } else { - log.Log.Debug(fmt.Sprintf("table `%v` already alter create_at, skip", table)) - } - } - return nil -} - -func setUpdateAt(tables []string) error { - for _, table := range tables { - var count int - sql := `SELECT count(1) FROM INFORMATION_SCHEMA.Columns WHERE table_schema=DATABASE() AND table_name=? - AND column_name='update_at' AND COLUMN_DEFAULT='CURRENT_TIMESTAMP' AND EXTRA='on update CURRENT_TIMESTAMP'` - ormer := orm.NewOrm() - if err := ormer.Raw(sql, table).QueryRow(&count); err != nil { - return err - } - if count == 0 { - sql = `alter table ` + table + ` modify column update_at datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP` - if _, err := ormer.Raw(sql).Exec(); err != nil { - return err - } - log.Log.Info(sql) - } else { - log.Log.Debug(fmt.Sprintf("table `%v` already alter update_at, skip", table)) - } - } - return nil -} - func initOrm() { DatabaseURL := beego.AppConfig.String("DB::url") DatabaseDebug, _ := beego.AppConfig.Bool("DB::debug") @@ -193,28 +148,6 @@ func initOrm() { orm.RunSyncdb("default", false, true) - tables := []string{ - "sys_resource_type", - "sys_resource_operation", - "sys_resource_constraint", - "sys_user", - "sys_group", - "sys_group_user_rel", - "sys_group_user_constraint", - "sys_group_role", - "sys_group_role_user", - "sys_group_role_operation", - "sys_audit", - "sys_resource_router", - } - if err := setCreateAt(tables); err != nil { - log.Log.Error(err.Error()) - os.Exit(2) - } - if err := setUpdateAt(tables); err != nil { - log.Log.Error(err.Error()) - os.Exit(2) - } } // Init ... @@ -223,5 +156,6 @@ func init() { return } initOrm() + migrations.InitMigration() // orm.RunSyncdb("default", false, true) }