feat(be): implement migrations for BoltDB

api/tasks/runner.go

@@ -828,7 +828,7 @@ func (t *task) setCmdEnvironment(cmd *exec.Cmd, gitSSHCommand string) {
 	//		}
 	//		env = append(env, "SEMAPHORE_INCOMING_VERSION="+incomingVersion)
 	//		if t.template.Type == db.TemplateBuild {
-	//			env = append(env, "SEMAPHORE_TARGET_VERSION="+*t.task.Version)
+	//			env = append(env, "SEMAPHORE_TARGET_VERSION="+*t.task.Migration)
 	//		}
 	//	}
 	//}

cli/cmd/migrate.go

@@ -15,6 +15,6 @@ var migrateCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		store := createStore()
 		defer store.Close()
-		fmt.Println("\n DB migrations run on startup automatically")
+		fmt.Println("\n db migrations run on startup automatically")
 	},
-}
+}

cli/cmd/setup.go

@@ -50,7 +50,7 @@ func doSetup() int {
 		os.Exit(1)
 	}
 
-	fmt.Println("Running DB Migrations..")
+	fmt.Println("Running db Migrations..")
 	if err := store.Migrate(); err != nil {
 		fmt.Printf("Database migrations failed!\n %v\n", err.Error())
 		os.Exit(1)

cli/setup/setup.go

@@ -86,21 +86,21 @@ func scanBoltDb(conf *util.ConfigType) {
 		workingDirectory = os.TempDir()
 	}
 	defaultBoltDBPath := filepath.Join(workingDirectory, "database.boltdb")
-	askValue("DB filename", defaultBoltDBPath, &conf.BoltDb.Hostname)
+	askValue("db filename", defaultBoltDBPath, &conf.BoltDb.Hostname)
 }
 
 func scanMySQL(conf *util.ConfigType) {
-	askValue("DB Hostname", "127.0.0.1:3306", &conf.MySQL.Hostname)
-	askValue("DB User", "root", &conf.MySQL.Username)
-	askValue("DB Password", "", &conf.MySQL.Password)
-	askValue("DB Name", "semaphore", &conf.MySQL.DbName)
+	askValue("db Hostname", "127.0.0.1:3306", &conf.MySQL.Hostname)
+	askValue("db User", "root", &conf.MySQL.Username)
+	askValue("db Password", "", &conf.MySQL.Password)
+	askValue("db Name", "semaphore", &conf.MySQL.DbName)
 }
 
 func scanPostgres(conf *util.ConfigType) {
-	askValue("DB Hostname", "127.0.0.1:5432", &conf.Postgres.Hostname)
-	askValue("DB User", "root", &conf.Postgres.Username)
-	askValue("DB Password", "", &conf.Postgres.Password)
-	askValue("DB Name", "semaphore", &conf.Postgres.DbName)
+	askValue("db Hostname", "127.0.0.1:5432", &conf.Postgres.Hostname)
+	askValue("db User", "root", &conf.Postgres.Username)
+	askValue("db Password", "", &conf.Postgres.Password)
+	askValue("db Name", "semaphore", &conf.Postgres.DbName)
 }
 
 func scanErrorChecker(n int, err error) {

db/Migration.go

@@ -0,0 +1,84 @@
+package db
+
+import (
+	"fmt"
+	"time"
+)
+
+// Migration represents sql schema version
+type Migration struct {
+	Version      string     `db:"version" json:"version"`
+	UpgradedDate *time.Time `db:"upgraded_date" json:"upgraded_date"`
+	Notes        *string    `db:"notes" json:"notes"`
+}
+
+// HumanoidVersion adds a v to the VersionString
+func (m Migration) HumanoidVersion() string {
+	return "v" + m.Version
+}
+
+func GetMigrations() []Migration {
+	return []Migration{
+		{},
+		{Version: "1.0.0"},
+		{Version: "1.2.0"},
+		{Version: "1.3.0"},
+		{Version: "1.4.0"},
+		{Version: "1.5.0"},
+		{Version: "1.6.0"},
+		{Version: "1.7.0"},
+		{Version: "1.8.0"},
+		{Version: "1.9.0"},
+		{Version: "2.2.1"},
+		{Version: "2.3.0"},
+		{Version: "2.3.1"},
+		{Version: "2.3.2"},
+		{Version: "2.4.0"},
+		{Version: "2.5.0"},
+		{Version: "2.5.2"},
+		{Version: "2.7.1"},
+		{Version: "2.7.4"},
+		{Version: "2.7.6"},
+		{Version: "2.7.8"},
+		{Version: "2.7.9"},
+		{Version: "2.7.10"},
+		{Version: "2.7.12"},
+		{Version: "2.7.13"},
+		{Version: "2.8.0"},
+		{Version: "2.8.1"},
+		{Version: "2.8.7"},
+		{Version: "2.8.8"},
+		{Version: "2.8.20"},
+		{Version: "2.8.25"},
+		{Version: "2.8.26"},
+	}
+}
+
+func Migrate(d Store) error {
+	fmt.Println("Checking db migrations")
+	didRun := false
+
+	for _, version := range GetMigrations() {
+		if exists, err := d.IsMigrationApplied(version); err != nil || exists {
+			if exists {
+				continue
+			}
+
+			return err
+		}
+
+		didRun = true
+		fmt.Printf("Executing migration %s (at %v)...\n", version.HumanoidVersion(), time.Now())
+		if err := d.ApplyMigration(version); err != nil {
+			fmt.Printf("Rolling back %s (time: %v)...\n", version.HumanoidVersion(), time.Now())
+			d.TryRollbackMigration(version)
+			return err
+		}
+	}
+
+	if didRun {
+		fmt.Println("Migrations Finished")
+	}
+
+	return nil
+}

db/Store.go

@@ -67,10 +67,18 @@ func (e *ValidationError) Error() string {
 type Store interface {
 	Connect() error
 	Close() error
+
 	// IsInitialized indicates is database already initialized, or it is empty.
 	// The method is useful for creating required entities in database during first run.
 	IsInitialized() (bool, error)
-	Migrate() error
+	// IsMigrationApplied queries the database to see if a migration table with
+	// this version id exists already
+	IsMigrationApplied(version Migration) (bool, error)
+	// ApplyMigration runs executes a database migration
+	ApplyMigration(version Migration) error
+	// TryRollbackMigration attempts to rollback the database to an earlier version
+	// if a rollback exists
+	TryRollbackMigration(version Migration)
 
 	GetEnvironment(projectID int, environmentID int) (Environment, error)
 	GetEnvironments(projectID int, params RetrieveQueryParams) ([]Environment, error)

db/bolt/migration.go

@@ -0,0 +1,64 @@
+package bolt
+
+import (
+	"encoding/json"
+	"github.com/ansible-semaphore/semaphore/db"
+	"github.com/ansible-semaphore/semaphore/db/bolt/migrations"
+	"go.etcd.io/bbolt"
+)
+
+func (d *BoltDb) IsMigrationApplied(migration db.Migration) (bool, error) {
+	err := d.db.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte("versions"))
+		if b == nil {
+			return db.ErrNotFound
+		}
+
+		d := b.Get([]byte(migration.Version))
+
+		if d == nil {
+			return db.ErrNotFound
+		}
+
+		return nil
+	})
+
+	if err == nil {
+		return true, nil
+	}
+
+	if err == db.ErrNotFound {
+		return false, nil
+	}
+
+	return false, err
+}
+
+func (d *BoltDb) ApplyMigration(migration db.Migration) (err error) {
+	switch migration.Version {
+	case "2.8.26":
+		err = migrations.Migration_2_8_28{DB: d.db}.Apply()
+	}
+
+	if err != nil {
+		return
+	}
+
+	return d.db.Update(func(tx *bbolt.Tx) error {
+		b, err := tx.CreateBucketIfNotExists([]byte("versions"))
+
+		if err != nil {
+			return err
+		}
+
+		j, err := json.Marshal(migration)
+
+		return b.Put([]byte(migration.Version), j)
+	})
+}
+
+func (d *BoltDb) TryRollbackMigration(migration db.Migration) {
+	switch migration.Version {
+	case "2.8.26":
+	}
+}

db/bolt/migrations/Migration_2_8_28.go

@@ -0,0 +1,78 @@
+package migrations
+
+import (
+	"encoding/json"
+	"go.etcd.io/bbolt"
+	"strings"
+)
+
+type Migration_2_8_28 struct {
+	DB *bbolt.DB
+}
+
+func (d Migration_2_8_28) getProjectRepositories_2_8_26(projectID string) (repos map[string]map[string]interface{}, err error) {
+	err = d.DB.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte("project__repository_" + projectID))
+		return b.ForEach(func(id, body []byte) error {
+			r := make(map[string]interface{})
+			repos[string(id)] = r
+			return json.Unmarshal(body, &r)
+		})
+	})
+	return
+}
+
+func (d Migration_2_8_28) setProjectRepository_2_8_26(projectID string, repoID string, repo map[string]interface{}) error {
+	return d.DB.Update(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte("project__repository_" + projectID))
+		j, err := json.Marshal(repo)
+		if err != nil {
+			return err
+		}
+		return b.Put([]byte(repoID), j)
+	})
+}
+
+func (d Migration_2_8_28) Apply() (err error) {
+	var projectIDs []string
+
+	err = d.DB.View(func(tx *bbolt.Tx) error {
+		return tx.Bucket([]byte("projects")).ForEach(func(id, _ []byte) error {
+			projectIDs = append(projectIDs, string(id))
+			return nil
+		})
+	})
+
+	if err != nil {
+		return
+	}
+
+	projectsRepositories := make(map[string]map[string]map[string]interface{})
+
+	for _, projectID := range projectIDs {
+		var err2 error
+		projectsRepositories[projectID], err2 = d.getProjectRepositories_2_8_26(projectID)
+		if err2 != nil {
+			return err2
+		}
+	}
+
+	for projectID, repositories := range projectsRepositories {
+		for repoID, repo := range repositories {
+			branch := "master"
+			url := repo["git_url"].(string)
+			parts := strings.Split(url, "#")
+			if len(parts) > 1 {
+				url, branch = parts[0], parts[1]
+			}
+			repo["git_url"] = url
+			repo["git_branch"] = branch
+			err = d.setProjectRepository_2_8_26(projectID, repoID, repo)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}