Feature db migration

CONTRIBUTING.md

@@ -32,10 +32,40 @@
    # use the pull.rebase config option to change the behavior for every git pull (instead of only newly-created branches)
    git config pull.rebase true
    ```
+3. 如果您提交的代码涉及到数据迁移，请在`internal/migrations`目录下添加迁移脚本，并在`migration.go`中注册该脚本
+   ```go
+   type MigrationXXXXX struct {
+   }
+
+   func (m MigrationXXXXX) GetCreateAt() time.Time{
+      //时间请设定为提交代码的当前时间，以便于系统自动记录迁移历史
+      return time.Date(2022, 1, 1, 0, 0, 0, 0, time.Local)
+   }
+   func (m Migration20220101) Upgrade(ormer orm.Ormer) error{ 
+      //样例代码如下，err不为nil时，会自动回滚所有操作
+      /*
+      err := ormer.Raw(xxxx).Exec()
+      return err
+      */
+
+   }
 
-3. Create a pull request to the main repository on GitHub.
-4. When the reviewer makes some comments, address any feedback that comes and update the pull request.
-5. When your contribution is accepted, your pull request will be approved and merged to the main branch.
+   ```
+
+   ```go
+   //在migrations.go中
+   func InitMigration() {
+      migrationTypes := MigrationTypes{
+         //注册
+         new(MigrationXXXXX),
+         //...
+      }
+
+   }
+   ```
+4. Create a pull request to the main repository on GitHub.
+5. When the reviewer makes some comments, address any feedback that comes and update the pull request.
+6. When your contribution is accepted, your pull request will be approved and merged to the main branch.
 
 ### 2. 文档贡献
 

deploy/docker-compose/init.sh

@@ -30,16 +30,5 @@ then
     echo "$NOT_UP restart failed, please check docker log use 'docker-compose logs [docker-name]'"
 fi
 
-MYSQL_DB=$(grep "MYSQL_DATABASE"  docker-compose.yml  | awk -F':' '{print $2}')
-MYSQL_DB_STRIP=`echo ${MYSQL_DB} | sed 's/ //g'`
-
-MYSQL_PASSWORD=$(grep "MYSQL_ROOT_PASSWORD"  docker-compose.yml  | awk -F':' '{print $2}')
-MYSQL_PASSWORD_STRIP=$(echo $MYSQL_PASSWORD | sed 's/ //g')
-
-echo "mysql database: $MYSQL_DB"
-echo "root password: $MYSQL_PASSWORD_STRIP"
-
-docker exec mysql mysql -uroot -p$MYSQL_PASSWORD_STRIP  $MYSQL_DB_STRIP < mysql/sql/v1.3.2_00.sql
-
 # init result verify
 [ $? -eq 0 ] && echo -e "AtomCI 初始化成功(:\n\n访问atomci: http://localhost:8090 \n" || echo -e "AtomCI 初始化失败, 请确认atomci 容器日志，\n或是 https://github.com/go-atomci/atomci-press/issues/new 反馈你的问题(:"

internal/migrations/migration20220101.go

@@ -0,0 +1,84 @@
+package migrations
+
+import (
+	"fmt"
+	"github.com/astaxie/beego/orm"
+	"github.com/go-atomci/atomci/internal/middleware/log"
+	"time"
+)
+
+type Migration20220101 struct {
+}
+
+func (m Migration20220101) GetCreateAt() time.Time {
+	return time.Date(2022, 1, 1, 0, 0, 0, 0, time.Local)
+}
+
+func (m Migration20220101) Upgrade(ormer orm.Ormer) error {
+	tables := []string{
+		"sys_resource_type",
+		"sys_resource_operation",
+		"sys_resource_constraint",
+		"sys_user",
+		"sys_group",
+		"sys_group_user_rel",
+		"sys_group_user_constraint",
+		"sys_group_role",
+		"sys_group_role_user",
+		"sys_group_role_operation",
+		"sys_audit",
+		"sys_resource_router",
+	}
+
+	if err := setCreateAt(ormer, tables); err != nil {
+		log.Log.Error(err.Error())
+		return err
+	}
+	if err := setUpdateAt(ormer, tables); err != nil {
+		log.Log.Error(err.Error())
+		return err
+	}
+	return nil
+}
+
+func setCreateAt(ormer orm.Ormer, tables []string) error {
+	for _, table := range tables {
+		var count int
+		sql := `SELECT count(1) FROM INFORMATION_SCHEMA.Columns WHERE table_schema=DATABASE() AND table_name=? 
+				AND column_name='create_at' AND  COLUMN_DEFAULT='CURRENT_TIMESTAMP'`
+		if err := ormer.Raw(sql, table).QueryRow(&count); err != nil {
+			return err
+		}
+		if count == 0 {
+			sql = `alter table ` + table + ` modify column create_at datetime not null DEFAULT CURRENT_TIMESTAMP`
+			if _, err := ormer.Raw(sql).Exec(); err != nil {
+				return err
+			}
+			log.Log.Info(sql)
+		} else {
+			log.Log.Debug(fmt.Sprintf("table `%v` already alter create_at, skip", table))
+		}
+	}
+	return nil
+}
+
+func setUpdateAt(ormer orm.Ormer, tables []string) error {
+	for _, table := range tables {
+		var count int
+		sql := `SELECT count(1) FROM INFORMATION_SCHEMA.Columns WHERE table_schema=DATABASE() AND table_name=? 
+				AND column_name='update_at' AND COLUMN_DEFAULT='CURRENT_TIMESTAMP' AND EXTRA='on update CURRENT_TIMESTAMP'`
+		if err := ormer.Raw(sql, table).QueryRow(&count); err != nil {
+			return err
+		}
+		if count == 0 {
+			sql = `alter table ` + table + ` modify column update_at datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP`
+			if _, err := ormer.Raw(sql).Exec(); err != nil {
+				return err
+			}
+			log.Log.Info(sql)
+		} else {
+			log.Log.Debug(fmt.Sprintf("table `%v` already alter update_at, skip", table))
+		}
+	}
+	return nil
+}

internal/migrations/migration20220309.go

@@ -0,0 +1,46 @@
+package migrations
+
+import (
+	"github.com/astaxie/beego/orm"
+	"strings"
+	"time"
+)
+
+type Migration20220309 struct {
+}
+
+func (m Migration20220309) GetCreateAt() time.Time {
+	return time.Date(2022, 3, 9, 0, 0, 0, 0, time.Local)
+}
+
+func (m Migration20220309) Upgrade(ormer orm.Ormer) error {
+	_, err := ormer.Raw("UPDATE `sys_integrate_setting` SET `type`='registry' WHERE `type`='harbor';").Exec()
+	if err != nil {
+		return err
+	}
+
+	_, err = ormer.Raw("DROP PROCEDURE IF EXISTS `ModifyHarborToRegistry`;").Exec()
+	_, err = ormer.Raw(strings.ReplaceAll(`CREATE PROCEDURE <|SPIT|>ModifyHarborToRegistry<|SPIT|>()
+BEGIN
+    DECLARE HARBOREXISTS int DEFAULT 0;
+    DECLARE REGISTRYEXISTS int DEFAULT 0;
+    SELECT count(1) INTO @HARBOREXISTS FROM information_schema.COLUMNS WHERE TABLE_NAME='project_env' AND COLUMN_NAME='harbor';
+    SELECT count(1) INTO @REGISTRYEXISTS FROM information_schema.COLUMNS WHERE TABLE_NAME='project_env' AND COLUMN_NAME='registry';
+    IF @HARBOREXISTS>0 AND @REGISTRYEXISTS=0
+    THEN
+        ALTER TABLE <|SPIT|>project_env<|SPIT|> CHANGE COLUMN <|SPIT|>harbor<|SPIT|> <|SPIT|>registry<|SPIT|> bigint(20) NOT NULL DEFAULT 0;
+    ELSEIF  @HARBOREXISTS>0 AND @REGISTRYEXISTS>0
+    THEN
+        UPDATE <|SPIT|>project_env<|SPIT|> SET <|SPIT|>registry<|SPIT|>=<|SPIT|>harbor<|SPIT|>;
+        ALTER TABLE <|SPIT|>project_env<|SPIT|> DROP COLUMN <|SPIT|>harbor<|SPIT|>;
+    END IF;
+END;`, "<|SPIT|>", "`")).Exec()
+
+	_, err = ormer.Raw("CALL `ModifyHarborToRegistry`;").Exec()
+	_, err = ormer.Raw("DROP PROCEDURE IF EXISTS `ModifyHarborToRegistry`;").Exec()
+
+	if err != nil {
+		return err
+	}
+	return nil
+}

internal/migrations/migrations.go

@@ -0,0 +1,97 @@
+package migrations
+
+import (
+	"github.com/astaxie/beego/orm"
+	"sort"
+	"time"
+)
+
+type MigrationTypes []Migration
+
+// Migration db migration base interface
+type Migration interface {
+	GetCreateAt() time.Time
+	Upgrade(ormer orm.Ormer) error
+}
+
+// Len 排序三人组
+func (t MigrationTypes) Len() int {
+	return len(t)
+}
+
+// Less 排序三人组
+func (t MigrationTypes) Less(i, j int) bool {
+	return t[i].GetCreateAt().Before(t[j].GetCreateAt())
+}
+
+// Swap 排序三人组
+func (t MigrationTypes) Swap(i, j int) {
+	t[i], t[j] = t[j], t[i]
+}
+
+// InitMigration db migration register
+func InitMigration() {
+	migrationTypes := MigrationTypes{
+		new(Migration20220101),
+		new(Migration20220309),
+	}
+
+	migrateInTx(migrationTypes)
+}
+
+func migrateInTx(migrationTypes MigrationTypes) {
+	//升序
+	sort.Sort(migrationTypes)
+
+	//数据迁移(事务）
+	ormClient := orm.NewOrm()
+	last := getNewestData(ormClient)
+	tempLast := last
+	errRet := ormClient.Begin()
+	for _, m := range migrationTypes {
+		if m.GetCreateAt().After(last) {
+			errRet = m.Upgrade(ormClient)
+			if errRet != nil {
+				break
+			}
+		}
+		tempLast = m.GetCreateAt()
+	}
+	errRet = updateNewestData(ormClient, tempLast)
+	if errRet != nil {
+		ormClient.Rollback()
+	} else {
+		ormClient.Commit()
+	}
+}
+
+func getNewestData(ormer orm.Ormer) time.Time {
+	sureCreateTable(ormer)
+	sql := `Select * From __dbmigration Limit 1`
+	var lastMigrationDate time.Time
+	ormer.Raw(sql).QueryRow(&lastMigrationDate)
+	if lastMigrationDate.IsZero() {
+		lastMigrationDate = time.Now()
+	}
+	return lastMigrationDate
+}
+
+func updateNewestData(ormer orm.Ormer, lastTime time.Time) error {
+	countSql := "Select count(*) from __dbmigration"
+	var count int
+	ormer.Raw(countSql).QueryRow(&count)
+
+	sql := "Update __dbmigration set last_migration_date=?"
+	if count == 0 {
+		sql = "Insert into __dbmigration(last_migration_date) values (?)"
+	}
+	_, err := ormer.Raw(sql, lastTime).Exec()
+	return err
+}
+
+func sureCreateTable(ormer orm.Ormer) {
+	ddl := `CREATE TABLE IF NOT EXISTS __dbmigration (
+	  last_migration_date datetime DEFAULT CURRENT_TIMESTAMP
+	)`
+	ormer.Raw(ddl).Exec()
+}