Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(engine): engine database upgrade improvement #3811

Merged
merged 4 commits into from
Jan 7, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion engine/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ BUILDTIME := `date "+%m/%d/%y-%H:%M:%S"`

TARGET_DIR = ./dist
TARGET_ENGINE = cds-engine
TARGET_LDFLAGS = -ldflags "-X github.com/ovh/cds/sdk.VERSION=$(VERSION) -X github.com/ovh/cds/sdk.GOOS=$$GOOS -X github.com/ovh/cds/sdk.GOARCH=$$GOARCH -X github.com/ovh/cds/sdk.GITHASH=$(GITHASH) -X github.com/ovh/cds/sdk.BUILDTIME=$(BUILDTIME) -X github.com/ovh/cds/sdk.BINARY=$(TARGET_ENGINE)"
DBMIGRATE = $(words $(wildcard sql/*.sql))
TARGET_LDFLAGS = -ldflags "-X github.com/ovh/cds/sdk.VERSION=$(VERSION) -X github.com/ovh/cds/sdk.GOOS=$$GOOS -X github.com/ovh/cds/sdk.GOARCH=$$GOARCH -X github.com/ovh/cds/sdk.GITHASH=$(GITHASH) -X github.com/ovh/cds/sdk.BUILDTIME=$(BUILDTIME) -X github.com/ovh/cds/sdk.BINARY=$(TARGET_ENGINE) -X github.com/ovh/cds/sdk.DBMIGRATE=$(DBMIGRATE)"
TARGET_OS = $(if ${OS},${OS},windows darwin linux freebsd)
TARGET_ARCH = $(if ${ARCH},${ARCH},amd64 arm 386)

Expand Down
129 changes: 120 additions & 9 deletions engine/api/database/cli.go
Original file line number Diff line number Diff line change
@@ -1,16 +1,24 @@
package database

import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
"time"

"github.com/mholt/archiver"
"github.com/olekukonko/tablewriter"
"github.com/rubenv/sql-migrate"
"github.com/spf13/cobra"

"github.com/ovh/cds/engine/api/database/dbmigrate"
"github.com/ovh/cds/sdk"
"github.com/ovh/cds/sdk/cdsclient"
)

//DBCmd is the root command for database management
Expand All @@ -23,8 +31,13 @@ var DBCmd = &cobra.Command{
var upgradeCmd = &cobra.Command{
Use: "upgrade",
Short: "Upgrade schema",
Long: "Migrates the database to the most recent version available.",
Run: upgradeCmdFunc,
Long: `Migrates the database to the most recent version available.`,
Example: `engine database upgrade --db-password=your-password --db-sslmode=disable --db-name=cds --migrate-dir=./sql

# If the directory --migrate-dir is not up to date with the current version, this
# directory will be automatically updated with the release from https://github.com/ovh/cds/releases
`,
Run: upgradeCmdFunc,
}

var downgradeCmd = &cobra.Command{
Expand Down Expand Up @@ -85,22 +98,120 @@ type statusRow struct {
}

func upgradeCmdFunc(cmd *cobra.Command, args []string) {
source := migrate.FileMigrationSource{
Dir: sqlMigrateDir,
}

migrations, err := source.FindMigrations()
if err != nil {
sdk.Exit("Error: %v\n", err)
}

if sdk.VERSION != "snapshot" {
nbMigrateOnThisVersion, err := strconv.Atoi(sdk.DBMIGRATE)
if err == nil { // no err -> it's a real release
fmt.Printf("There are %d migrate files locally. Current engine needs %d files\n", len(migrations), nbMigrateOnThisVersion)
if nbMigrateOnThisVersion > len(migrations) {
fmt.Printf("This version %s should contains %d migrate files.\n", sdk.VERSION, nbMigrateOnThisVersion)
if err := downloadSQLTarGz(sdk.VERSION, "sql.tar.gz", sqlMigrateDir); err != nil {
sdk.Exit("Error on download sql.tar.gz: %v\n", err)
}
migrations, err := source.FindMigrations()
if err != nil {
sdk.Exit("Error: %v\n", err)
}
fmt.Printf("sql.tar.gz downloaded, there are now %d migrate files locally\n", len(migrations))
}
} else {
sdk.Exit("Invalid compilation flag DBMIGRATE")
}
}

if err := ApplyMigrations(migrate.Up, sqlMigrateDryRun, sqlMigrateLimitUp); err != nil {
sdk.Exit("Error: %s\n", err)
sdk.Exit("Error: %v\n", err)
}
}

// downloadSQLTarGz downloads sql.tar.gz from github release corresponding to the current engine version
// check status 200 on download
// then write sql.tar.gz file in tmpdir
// then unzip sql.tar.gz file
// then move all sql file to sqlMigrateDir directory
func downloadSQLTarGz(currentVersion string, artifactName string, migrateDir string) error {
if !strings.Contains(currentVersion, "+") {
return fmt.Errorf("invalid current version: %s, ersion must contains a '+'", currentVersion)
}
if _, err := os.Stat(migrateDir); os.IsNotExist(err) {
return fmt.Errorf("--migrate-dir does not exist: %s", migrateDir)
}
currentTag := currentVersion[:strings.LastIndex(currentVersion, "+")]
urlTarGZ := fmt.Sprintf("https://github.com/ovh/cds/releases/download/%s/%s", currentTag, artifactName)
fmt.Printf("Getting %s from github on %s...\n", artifactName, urlTarGZ)

httpClient := cdsclient.NewHTTPClient(60*time.Second, false)
resp, err := httpClient.Get(urlTarGZ)
if err != nil {
return fmt.Errorf("error while getting %s from Github: %v", artifactName, err)
}
defer resp.Body.Close()

if resp.StatusCode != 200 {
buf := new(bytes.Buffer)
if _, err := buf.ReadFrom(resp.Body); err == nil {
fmt.Printf("body returned from github: \n%s\n", buf.String())
}
return fmt.Errorf("error http code: %d, url called: %s", resp.StatusCode, urlTarGZ)
}

if err := sdk.CheckContentTypeBinary(resp); err != nil {
return fmt.Errorf("invalid content type: %s", err.Error())
}

tmpfile, err := ioutil.TempFile(os.TempDir(), artifactName)
if err != nil {
sdk.Exit(err.Error())
}
defer tmpfile.Close()

if _, err = io.Copy(tmpfile, resp.Body); err != nil {
return fmt.Errorf("error on creating file: %v", err.Error())
}

dest := tmpfile.Name() + "extract"
if err := archiver.TarGz.Open(tmpfile.Name(), dest); err != nil {
return fmt.Errorf("Unarchive %s failed: %v", artifactName, err)
}
// the directory dest/sql/ -> contains all sql files
fmt.Printf("Unarchive to %s\n", dest)

dirFiles := dest + "/sql"
files, err := ioutil.ReadDir(dirFiles)
if err != nil {
return fmt.Errorf("error on readDir %s", dirFiles)
}
for _, f := range files {
if strings.HasSuffix(f.Name(), ".sql") {
src := dirFiles + "/" + f.Name()
dest := migrateDir + "/" + filepath.Base(f.Name())
if err := os.Rename(src, dest); err != nil {
return fmt.Errorf("error on move %s to %s err:%v", src, dest, err)
}
}
}
return nil
}

func downgradeCmdFunc(cmd *cobra.Command, args []string) {
if err := ApplyMigrations(migrate.Down, sqlMigrateDryRun, sqlMigrateLimitDown); err != nil {
sdk.Exit("Error: %s\n", err)
sdk.Exit("Error: %v\n", err)
}
}

func statusCmdFunc(cmd *cobra.Command, args []string) {
var err error
connFactory, err = Init(connFactory.dbUser, connFactory.dbRole, connFactory.dbPassword, connFactory.dbName, connFactory.dbHost, connFactory.dbPort, connFactory.dbSSLMode, connFactory.dbConnectTimeout, connFactory.dbTimeout, connFactory.dbMaxConn)
if err != nil {
sdk.Exit("Error: %s\n", err)
sdk.Exit("Error: %v\n", err)
}

source := migrate.FileMigrationSource{
Expand All @@ -109,12 +220,12 @@ func statusCmdFunc(cmd *cobra.Command, args []string) {

migrations, err := source.FindMigrations()
if err != nil {
sdk.Exit("Error: %s\n", err)
sdk.Exit("Error: %v\n", err)
}

records, err := migrate.GetMigrationRecords(connFactory.DB(), "postgres")
if err != nil {
sdk.Exit("Error: %s\n", err)
sdk.Exit("Error: %v\n", err)
}

table := tablewriter.NewWriter(os.Stdout)
Expand Down Expand Up @@ -163,12 +274,12 @@ func ApplyMigrations(dir migrate.MigrationDirection, dryrun bool, limit int) err
var err error
connFactory, err = Init(connFactory.dbUser, connFactory.dbRole, connFactory.dbPassword, connFactory.dbName, connFactory.dbHost, connFactory.dbPort, connFactory.dbSSLMode, connFactory.dbConnectTimeout, connFactory.dbTimeout, connFactory.dbMaxConn)
if err != nil {
sdk.Exit("Error: %s\n", err)
sdk.Exit("Error: %v\n", err)
}

migrations, err := dbmigrate.Do(connFactory.DB, sqlMigrateDir, dir, dryrun, limit)
if err != nil {
sdk.Exit("Error: %s\n", err)
sdk.Exit("Error: %v\n", err)
}

if dryrun {
Expand Down
49 changes: 49 additions & 0 deletions engine/api/database/cli_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package database

import (
"os"
"testing"
)

func Test_downloadSQLTarGz(t *testing.T) {
type args struct {
currentVersion string
artifactName string
migrateDir string
}
tmpMigrateDir := os.TempDir()
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "valid version",
args: args{currentVersion: "0.37.3+cds.7708", artifactName: "sql.tar.gz", migrateDir: tmpMigrateDir},
wantErr: false,
}, {
name: "valid version, invalid dir",
args: args{currentVersion: "0.37.3+cds.7708", artifactName: "sql.tar.gz", migrateDir: "/tmp/foo"},
wantErr: true,
}, {
name: "invalid artifact name",
args: args{currentVersion: "0.37.3+cds.7708", artifactName: "test-artifact.tar.gz", migrateDir: tmpMigrateDir},
wantErr: true,
}, {
name: "invalid version without '+'",
args: args{currentVersion: "0.37.3", artifactName: "sql.tar.gz", migrateDir: tmpMigrateDir},
wantErr: true,
}, {
name: "invalid version",
args: args{currentVersion: "foo", artifactName: "sql.tar.gz", migrateDir: tmpMigrateDir},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := downloadSQLTarGz(tt.args.currentVersion, tt.args.artifactName, tt.args.migrateDir); (err != nil) != tt.wantErr {
t.Errorf("downloadSQLTarGz() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
2 changes: 1 addition & 1 deletion engine/update.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ var updateCmd = &cobra.Command{

resp, err := http.Get(urlBinary)
if err != nil {
sdk.Exit("Error while getting binary from CDS API: %s\n", err)
sdk.Exit("Error while getting binary from: %s err:%s\n", urlBinary, err)
}
defer resp.Body.Close()

Expand Down
8 changes: 7 additions & 1 deletion sdk/version.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ var (

//BINARY is set with -ldflags "-X github.com/ovh/cds/sdk.BINARY=$(BINARY)"
BINARY = ""

//DBMIGRATE is set with -ldflags "-X github.com/ovh/cds/sdk.DBMIGRATE=$(DBMIGRATE)"
// this flag contains the number of sql files to migrate for this version
DBMIGRATE = ""
)

func init() {
Expand All @@ -41,6 +45,7 @@ type Version struct {
OS string `json:"os"`
GitHash string `json:"git_hash"`
BuildTime string `json:"build_time"`
DBMigrate string `json:"db_migrate"`
}

// VersionCurrent returns the current version
Expand All @@ -51,10 +56,11 @@ func VersionCurrent() Version {
OS: GOOS,
GitHash: GITHASH,
BuildTime: BUILDTIME,
DBMigrate: DBMIGRATE,
}
}

// VersionString returns a string contains all about current version
func VersionString() string {
return fmt.Sprintf("CDS %s version:%s os:%s architecture:%s git.hash:%s build.time:%s", BINARY, VERSION, GOOS, GOARCH, GITHASH, BUILDTIME)
return fmt.Sprintf("CDS %s version:%s os:%s architecture:%s git.hash:%s build.time:%s db.migrate:%s", BINARY, VERSION, GOOS, GOARCH, GITHASH, BUILDTIME, DBMIGRATE)
}