From ba9f4998310f9494633b0432eeffbca8c872af32 Mon Sep 17 00:00:00 2001 From: Joseph Spurrier Date: Tue, 10 Jul 2018 00:37:38 -0400 Subject: [PATCH 01/14] Add the migration cli --- src/app/webapi/cmd/cliapp/cliapp.go | 9 +- src/app/webapi/internal/testrequest/form.go | 2 +- src/app/webapi/pkg/basemigrate/basemigrate.go | 128 ++++++++++++++++++ .../pkg/basemigrate/basemigrate_test.go | 36 +++++ src/app/webapi/pkg/basemigrate/changeset.go | 69 ++++++++++ src/app/webapi/pkg/basemigrate/helper.go | 46 +++++++ src/app/webapi/pkg/basemigrate/parse.go | 60 ++++++++ .../basemigrate/testdata/fail-duplicate.sql | 91 +++++++++++++ .../pkg/basemigrate/testdata/success.sql | 85 ++++++++++++ 9 files changed, 524 insertions(+), 2 deletions(-) create mode 100644 src/app/webapi/pkg/basemigrate/basemigrate.go create mode 100644 src/app/webapi/pkg/basemigrate/basemigrate_test.go create mode 100644 src/app/webapi/pkg/basemigrate/changeset.go create mode 100644 src/app/webapi/pkg/basemigrate/helper.go create mode 100644 src/app/webapi/pkg/basemigrate/parse.go create mode 100644 src/app/webapi/pkg/basemigrate/testdata/fail-duplicate.sql create mode 100644 src/app/webapi/pkg/basemigrate/testdata/success.sql diff --git a/src/app/webapi/cmd/cliapp/cliapp.go b/src/app/webapi/cmd/cliapp/cliapp.go index 6b9d327..182ca40 100644 --- a/src/app/webapi/cmd/cliapp/cliapp.go +++ b/src/app/webapi/cmd/cliapp/cliapp.go @@ -1,6 +1,7 @@ package main import ( + "app/webapi/pkg/basemigrate" "encoding/base64" "fmt" "log" @@ -14,7 +15,11 @@ import ( var ( app = kingpin.New("cliapp", "A command-line application to perform tasks for the webapi.") - cGenerate = app.Command("generate", "Generate 256 bit (32 byte) base64 encoded JWT.") + cGenerate = app.Command("generate", "Generate 256 bit (32 byte) base64 encoded JWT.") + cDB = app.Command("migrate", "Perform actions on the database.") + cDBAll = cDB.Command("all", "Apply all changesets to the database.") + cDBReset = cDB.Command("reset", "Run all rollbacks on the database.") + cDBAllFile = cDBAll.Arg("file", "Filename of the migration file.").Required().String() ) func main() { @@ -30,5 +35,7 @@ func main() { enc := base64.StdEncoding.EncodeToString(b) fmt.Println(enc) + case cDBAll.FullCommand(): + basemigrate.Migrate(*cDBAllFile, true) } } diff --git a/src/app/webapi/internal/testrequest/form.go b/src/app/webapi/internal/testrequest/form.go index ef31930..f4d0d62 100644 --- a/src/app/webapi/internal/testrequest/form.go +++ b/src/app/webapi/internal/testrequest/form.go @@ -1,13 +1,13 @@ package testrequest import ( - "app/webapi" "io" "net/http/httptest" "net/url" "strings" "testing" + "app/webapi" "app/webapi/component" ) diff --git a/src/app/webapi/pkg/basemigrate/basemigrate.go b/src/app/webapi/pkg/basemigrate/basemigrate.go new file mode 100644 index 0000000..f235dcc --- /dev/null +++ b/src/app/webapi/pkg/basemigrate/basemigrate.go @@ -0,0 +1,128 @@ +package basemigrate + +import ( + "database/sql" + "errors" + "fmt" + "os" + "strings" +) + +const ( + sqlChangelog = `CREATE TABLE IF NOT EXISTS databasechangelog ( + id varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + author varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + filename varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + dateexecuted datetime NOT NULL, + orderexecuted int(11) NOT NULL, + md5sum varchar(35) COLLATE utf8mb4_unicode_ci DEFAULT NULL, + description varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL, + tag varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL, + version varchar(20) COLLATE utf8mb4_unicode_ci DEFAULT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;` +) + +const ( + appVersion = "1.0" + elementChangeset = "--changeset " + elementRollback = "--rollback " +) + +var ( + // ErrInvalidHeader is when the changeset header is invalid. + ErrInvalidHeader = errors.New("invalid changeset header") + // ErrInvalidFormat is when a changeset is not found. + ErrInvalidFormat = errors.New("invalid changeset format") +) + +// ParseFile will parse a file into changesets. +func ParseFile(filename string) ([]*Changeset, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + return parse(f, filename) +} + +// Migrate will migrate a file. +func Migrate(filename string, verbose bool) (err error) { + db, err := connect() + if err != nil { + return err + } + + // Create the DATABASECHANGELOG. + _, err = db.Exec(sqlChangelog) + + // Get the changesets. + arr, err := ParseFile(filename) + if err != nil { + return err + } + + // Loop through each changeset. + for _, cs := range arr { + checksum := "" + newChecksum := cs.Checksum() + + // Determine if the changeset was already applied. + // Count the number of rows. + err = db.Get(&checksum, `SELECT md5sum + FROM databasechangelog + WHERE id = ? + AND author = ?`, cs.id, cs.author) + if err == nil { + // Determine if the checksums match. + if checksum != newChecksum { + return fmt.Errorf("checksum does not match - existing changeset %v:%v has checksum %v, but new changeset has checksum %v", + cs.author, cs.id, checksum, newChecksum) + } + + if verbose { + fmt.Printf("Changeset already applied: %v:%v\n", cs.author, cs.id) + } + continue + } else if err != nil && err != sql.ErrNoRows { + return err + } + + arrQueries := strings.Split(cs.Changes(), ";") + // Loop through each change. + for _, q := range arrQueries { + if len(q) == 0 { + continue + } + + // Execute the query. + _, err = db.Exec(q) + if err != nil { + return err + } + } + + // Count the number of rows. + count := 0 + err = db.Get(&count, `SELECT COUNT(*) FROM databasechangelog`) + if err != nil { + return err + } + + // Insert the record. + _, err = db.Exec(` + INSERT INTO databasechangelog + (id,author,filename,dateexecuted,orderexecuted,md5sum,description,version) + VALUES(?,?,?,NOW(),?,?,?,?) + `, cs.id, cs.author, cs.filename, count+1, newChecksum, cs.description, cs.version) + if err != nil { + return err + } + + if verbose { + fmt.Printf("Changeset applied: %v:%v\n", cs.author, cs.id) + } + } + + return +} diff --git a/src/app/webapi/pkg/basemigrate/basemigrate_test.go b/src/app/webapi/pkg/basemigrate/basemigrate_test.go new file mode 100644 index 0000000..a0a45c7 --- /dev/null +++ b/src/app/webapi/pkg/basemigrate/basemigrate_test.go @@ -0,0 +1,36 @@ +package basemigrate_test + +import ( + "testing" + + "app/webapi/pkg/basemigrate" + + "github.com/stretchr/testify/assert" +) + +func TestMigration(t *testing.T) { + err := basemigrate.Migrate("testdata/success.sql", false) + assert.Nil(t, err) +} + +func TestMigrationFailDuplicate(t *testing.T) { + err := basemigrate.Migrate("testdata/fail-duplicate.sql", false) + assert.Contains(t, err.Error(), "checksum does not match") +} + +func TestParse(t *testing.T) { + arr, err := basemigrate.ParseFile("testdata/success.sql") + assert.Nil(t, err) + assert.Equal(t, 5, len(arr)) + + //basemigrate.Debug(arr) + + /*for _, v := range arr { + fmt.Println(v.Changes()) + fmt.Println(v.Rollbacks()) + fmt.Println("MD5:", v.Checksum()) + break + } + + fmt.Println("Total:", len(arr))*/ +} diff --git a/src/app/webapi/pkg/basemigrate/changeset.go b/src/app/webapi/pkg/basemigrate/changeset.go new file mode 100644 index 0000000..e1ea952 --- /dev/null +++ b/src/app/webapi/pkg/basemigrate/changeset.go @@ -0,0 +1,69 @@ +package basemigrate + +import ( + "strings" +) + +// Changeset is a SQL changeset. +type Changeset struct { + id string + author string + filename string + md5 string + description string + version string + + change []string + rollback []string +} + +// ParseHeader will parse the header information. +func (cs *Changeset) ParseHeader(line string) error { + arr := strings.Split(line, ":") + if len(arr) != 2 { + return ErrInvalidHeader + } + + cs.author = arr[0] + cs.id = arr[1] + + return nil +} + +// SetFileInfo will set the file information. +func (cs *Changeset) SetFileInfo(filename string, description string, version string) { + cs.filename = filename + cs.description = description + cs.version = version +} + +// AddRollback will add a rollback command. +func (cs *Changeset) AddRollback(line string) { + if len(cs.rollback) == 0 { + cs.rollback = make([]string, 0) + } + cs.rollback = append(cs.rollback, line) +} + +// AddChange will add a change command. +func (cs *Changeset) AddChange(line string) { + if len(cs.change) == 0 { + cs.change = make([]string, 0) + } + cs.change = append(cs.change, line) +} + +// Changes will return all the changes. +func (cs *Changeset) Changes() string { + return strings.Join(cs.change, "\n") +} + +// Rollbacks will return all the rollbacks. +func (cs *Changeset) Rollbacks() string { + return strings.Join(cs.rollback, "\n") +} + +// Checksum returns an MD5 checksum for the changeset. +func (cs *Changeset) Checksum() string { + return md5sum(cs.Changes()) +} diff --git a/src/app/webapi/pkg/basemigrate/helper.go b/src/app/webapi/pkg/basemigrate/helper.go new file mode 100644 index 0000000..5575690 --- /dev/null +++ b/src/app/webapi/pkg/basemigrate/helper.go @@ -0,0 +1,46 @@ +package basemigrate + +import ( + "bytes" + "crypto/md5" + "fmt" + "io" + + "app/webapi/pkg/database" + + "github.com/jmoiron/sqlx" +) + +// connect will connect to the database. +func connect() (*sqlx.DB, error) { + dbc := new(database.Connection) + dbc.Hostname = "127.0.0.1" + dbc.Port = 3306 + dbc.Username = "root" + dbc.Password = "" + dbc.Database = "webapitest" + dbc.Parameter = "parseTime=true&allowNativePasswords=true" + + return dbc.Connect(true) +} + +// md5sum will return a checksum from a string. +func md5sum(s string) string { + h := md5.New() + r := bytes.NewReader([]byte(s)) + _, _ = io.Copy(h, r) + return fmt.Sprintf("%x", h.Sum(nil)) +} + +// Debug will return the SQL file. +func Debug(arr []*Changeset) { + for _, cs := range arr { + fmt.Printf("%v%v:%v\n", elementChangeset, cs.author, cs.id) + fmt.Println(cs.Changes()) + fmt.Printf("%v%v\n", elementRollback, cs.Rollbacks()) + fmt.Println("--md5", cs.Checksum()) + break + } + + fmt.Println("Total:", len(arr)) +} diff --git a/src/app/webapi/pkg/basemigrate/parse.go b/src/app/webapi/pkg/basemigrate/parse.go new file mode 100644 index 0000000..2b7cff2 --- /dev/null +++ b/src/app/webapi/pkg/basemigrate/parse.go @@ -0,0 +1,60 @@ +package basemigrate + +import ( + "bufio" + "io" + "path" + "strings" +) + +// parse will split the SQL migration into pieces. +func parse(r io.Reader, filename string) ([]*Changeset, error) { + scanner := bufio.NewScanner(r) + scanner.Split(bufio.ScanLines) + + // Array of changesets. + arr := make([]*Changeset, 0) + + for scanner.Scan() { + // Get the line without leading or trailing spaces. + line := strings.TrimSpace(scanner.Text()) + + // Skip blank lines. + if len(line) == 0 { + continue + } + + // Start recording the changeset. + if strings.HasPrefix(line, elementChangeset) { + // Create a new changeset. + cs := new(Changeset) + cs.ParseHeader(strings.TrimLeft(line, elementChangeset)) + cs.SetFileInfo(path.Base(filename), "sql", appVersion) + arr = append(arr, cs) + continue + } + + // If the length of the array is 0, then the first changeset is missing. + if len(arr) == 0 { + return nil, ErrInvalidFormat + } + + // Determine if the line is a rollback. + if strings.HasPrefix(line, elementRollback) { + cs := arr[len(arr)-1] + cs.AddRollback(strings.TrimLeft(line, elementRollback)) + continue + } + + // Determine if the line is comment, ignore it. + if strings.HasPrefix(line, "--") { + continue + } + + // Add the line as a changeset. + cs := arr[len(arr)-1] + cs.AddChange(line) + } + + return arr, nil +} diff --git a/src/app/webapi/pkg/basemigrate/testdata/fail-duplicate.sql b/src/app/webapi/pkg/basemigrate/testdata/fail-duplicate.sql new file mode 100644 index 0000000..82b92e2 --- /dev/null +++ b/src/app/webapi/pkg/basemigrate/testdata/fail-duplicate.sql @@ -0,0 +1,91 @@ +--changeset josephspurrier:1 +CREATE TABLE player ( + id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, + steam_id VARCHAR(191) NOT NULL, + account_name VARCHAR(191) NOT NULL, + + created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL DEFAULT NULL, + + PRIMARY KEY (id), + CONSTRAINT u_player_account_name UNIQUE (account_name) +); +--rollback DROP TABLE IF EXISTS player; +--changeset josephspurrier:2 +CREATE TABLE player_login ( + id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, + player_id BIGINT(10) UNSIGNED NOT NULL, + display_name VARCHAR(191) NOT NULL, + ip VARCHAR(191) NOT NULL, + login_at TIMESTAMP NOT NULL, + + created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL DEFAULT NULL, + + PRIMARY KEY (id), + FOREIGN KEY fk_player_login_player_id_player_id (player_id) + REFERENCES player(id) ON DELETE CASCADE +); +--rollback DROP TABLE IF EXISTS player_login; + +--changeset josephspurrier:3 +CREATE TABLE game ( + id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, + start_at TIMESTAMP NULL, + end_at TIMESTAMP NULL, + + created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL DEFAULT NULL, + + PRIMARY KEY (id) +); +--rollback DROP TABLE IF EXISTS game; + +--changeset josephspurrier:4 +CREATE TABLE game_winner ( + id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, + game_id BIGINT(10) UNSIGNED NOT NULL, + player_id BIGINT(10) UNSIGNED NOT NULL, + + created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL DEFAULT NULL, + + PRIMARY KEY (id), + FOREIGN KEY fk_game_winner_game_id_game_id (game_id) + REFERENCES game(id) ON DELETE CASCADE, + FOREIGN KEY fk_game_winner_player_id_player_id (player_id) + REFERENCES player(id) ON DELETE CASCADE +); +--rollback DROP TABLE IF EXISTS game_winner; + +--changeset josephspurrier:5 +CREATE TABLE game_player ( + id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, + game_id BIGINT(10) UNSIGNED NOT NULL, + player_id BIGINT(10) UNSIGNED NOT NULL, + kills SMALLINT(10) UNSIGNED NOT NULL, + deaths SMALLINT(10) UNSIGNED NOT NULL, + assists SMALLINT(10) UNSIGNED NOT NULL, + damage_dealt MEDIUMINT(10) UNSIGNED NOT NULL, + + created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL DEFAULT NULL, + + PRIMARY KEY (id), + FOREIGN KEY fk_game_player_game_id_game_id (game_id) + REFERENCES game(id) ON DELETE CASCADE, + FOREIGN KEY fk_game_player_player_id_player_id (player_id) + REFERENCES player(id) ON DELETE CASCADE +); +--rollback DROP TABLE IF EXISTS game_player; + +--changeset josephspurrier:5 +CREATE TABLE game_player2 ( + id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, +); +--rollback DROP TABLE IF EXISTS game_player2; \ No newline at end of file diff --git a/src/app/webapi/pkg/basemigrate/testdata/success.sql b/src/app/webapi/pkg/basemigrate/testdata/success.sql new file mode 100644 index 0000000..f334843 --- /dev/null +++ b/src/app/webapi/pkg/basemigrate/testdata/success.sql @@ -0,0 +1,85 @@ +--changeset josephspurrier:1 +CREATE TABLE player ( + id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, + steam_id VARCHAR(191) NOT NULL, + account_name VARCHAR(191) NOT NULL, + + created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL DEFAULT NULL, + + PRIMARY KEY (id), + CONSTRAINT u_player_account_name UNIQUE (account_name) +); +--rollback DROP TABLE IF EXISTS player; +--changeset josephspurrier:2 +CREATE TABLE player_login ( + id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, + player_id BIGINT(10) UNSIGNED NOT NULL, + display_name VARCHAR(191) NOT NULL, + ip VARCHAR(191) NOT NULL, + login_at TIMESTAMP NOT NULL, + + created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL DEFAULT NULL, + + PRIMARY KEY (id), + FOREIGN KEY fk_player_login_player_id_player_id (player_id) + REFERENCES player(id) ON DELETE CASCADE +); +--rollback DROP TABLE IF EXISTS player_login; + +--changeset josephspurrier:3 +CREATE TABLE game ( + id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, + start_at TIMESTAMP NULL, + end_at TIMESTAMP NULL, + + created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL DEFAULT NULL, + + PRIMARY KEY (id) +); +--rollback DROP TABLE IF EXISTS game; + +--changeset josephspurrier:4 +CREATE TABLE game_winner ( + id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, + game_id BIGINT(10) UNSIGNED NOT NULL, + player_id BIGINT(10) UNSIGNED NOT NULL, + + created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL DEFAULT NULL, + + PRIMARY KEY (id), + FOREIGN KEY fk_game_winner_game_id_game_id (game_id) + REFERENCES game(id) ON DELETE CASCADE, + FOREIGN KEY fk_game_winner_player_id_player_id (player_id) + REFERENCES player(id) ON DELETE CASCADE +); +--rollback DROP TABLE IF EXISTS game_winner; + +--changeset josephspurrier:5 +CREATE TABLE game_player ( + id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, + game_id BIGINT(10) UNSIGNED NOT NULL, + player_id BIGINT(10) UNSIGNED NOT NULL, + kills SMALLINT(10) UNSIGNED NOT NULL, + deaths SMALLINT(10) UNSIGNED NOT NULL, + assists SMALLINT(10) UNSIGNED NOT NULL, + damage_dealt MEDIUMINT(10) UNSIGNED NOT NULL, + + created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL DEFAULT NULL, + + PRIMARY KEY (id), + FOREIGN KEY fk_game_player_game_id_game_id (game_id) + REFERENCES game(id) ON DELETE CASCADE, + FOREIGN KEY fk_game_player_player_id_player_id (player_id) + REFERENCES player(id) ON DELETE CASCADE +); +--rollback DROP TABLE IF EXISTS game_player; \ No newline at end of file From a359e41cbb37074d7decd00f0232715577dec9a4 Mon Sep 17 00:00:00 2001 From: Joseph Spurrier Date: Tue, 10 Jul 2018 00:55:37 -0400 Subject: [PATCH 02/14] Add the SQL migration package --- src/app/webapi/cmd/cliapp/cliapp.go | 6 +- src/app/webapi/pkg/basemigrate/basemigrate.go | 4 +- .../pkg/basemigrate/basemigrate_test.go | 2 +- .../basemigrate/testdata/fail-duplicate.sql | 122 ++++++------------ .../pkg/basemigrate/testdata/success.sql | 114 ++++++---------- 5 files changed, 81 insertions(+), 167 deletions(-) diff --git a/src/app/webapi/cmd/cliapp/cliapp.go b/src/app/webapi/cmd/cliapp/cliapp.go index 182ca40..793cd78 100644 --- a/src/app/webapi/cmd/cliapp/cliapp.go +++ b/src/app/webapi/cmd/cliapp/cliapp.go @@ -36,6 +36,10 @@ func main() { enc := base64.StdEncoding.EncodeToString(b) fmt.Println(enc) case cDBAll.FullCommand(): - basemigrate.Migrate(*cDBAllFile, true) + err := basemigrate.Migrate(*cDBAllFile, true) + if err != nil { + fmt.Println(err) + os.Exit(1) + } } } diff --git a/src/app/webapi/pkg/basemigrate/basemigrate.go b/src/app/webapi/pkg/basemigrate/basemigrate.go index f235dcc..e73c8a5 100644 --- a/src/app/webapi/pkg/basemigrate/basemigrate.go +++ b/src/app/webapi/pkg/basemigrate/basemigrate.go @@ -85,7 +85,7 @@ func Migrate(filename string, verbose bool) (err error) { } continue } else if err != nil && err != sql.ErrNoRows { - return err + return fmt.Errorf("internal error on changeset %v:%v - %v", cs.author, cs.id, err.Error()) } arrQueries := strings.Split(cs.Changes(), ";") @@ -98,7 +98,7 @@ func Migrate(filename string, verbose bool) (err error) { // Execute the query. _, err = db.Exec(q) if err != nil { - return err + return fmt.Errorf("sql error on changeset %v:%v - %v", cs.author, cs.id, err.Error()) } } diff --git a/src/app/webapi/pkg/basemigrate/basemigrate_test.go b/src/app/webapi/pkg/basemigrate/basemigrate_test.go index a0a45c7..b60ac8d 100644 --- a/src/app/webapi/pkg/basemigrate/basemigrate_test.go +++ b/src/app/webapi/pkg/basemigrate/basemigrate_test.go @@ -21,7 +21,7 @@ func TestMigrationFailDuplicate(t *testing.T) { func TestParse(t *testing.T) { arr, err := basemigrate.ParseFile("testdata/success.sql") assert.Nil(t, err) - assert.Equal(t, 5, len(arr)) + assert.Equal(t, 3, len(arr)) //basemigrate.Debug(arr) diff --git a/src/app/webapi/pkg/basemigrate/testdata/fail-duplicate.sql b/src/app/webapi/pkg/basemigrate/testdata/fail-duplicate.sql index 82b92e2..fcd6846 100644 --- a/src/app/webapi/pkg/basemigrate/testdata/fail-duplicate.sql +++ b/src/app/webapi/pkg/basemigrate/testdata/fail-duplicate.sql @@ -1,91 +1,43 @@ --changeset josephspurrier:1 -CREATE TABLE player ( - id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, - steam_id VARCHAR(191) NOT NULL, - account_name VARCHAR(191) NOT NULL, - - created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, - deleted_at TIMESTAMP NULL DEFAULT NULL, - - PRIMARY KEY (id), - CONSTRAINT u_player_account_name UNIQUE (account_name) -); ---rollback DROP TABLE IF EXISTS player; ---changeset josephspurrier:2 -CREATE TABLE player_login ( - id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, - player_id BIGINT(10) UNSIGNED NOT NULL, - display_name VARCHAR(191) NOT NULL, - ip VARCHAR(191) NOT NULL, - login_at TIMESTAMP NOT NULL, - - created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, - deleted_at TIMESTAMP NULL DEFAULT NULL, - - PRIMARY KEY (id), - FOREIGN KEY fk_player_login_player_id_player_id (player_id) - REFERENCES player(id) ON DELETE CASCADE -); ---rollback DROP TABLE IF EXISTS player_login; - ---changeset josephspurrier:3 -CREATE TABLE game ( - id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, - start_at TIMESTAMP NULL, - end_at TIMESTAMP NULL, - - created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, - deleted_at TIMESTAMP NULL DEFAULT NULL, - +SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO'; +CREATE TABLE user_status ( + id TINYINT(1) UNSIGNED NOT NULL AUTO_INCREMENT, + + status VARCHAR(25) NOT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted TINYINT(1) UNSIGNED NOT NULL DEFAULT 0, + PRIMARY KEY (id) ); ---rollback DROP TABLE IF EXISTS game; +--rollback DROP TABLE user_status; ---changeset josephspurrier:4 -CREATE TABLE game_winner ( - id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, - game_id BIGINT(10) UNSIGNED NOT NULL, - player_id BIGINT(10) UNSIGNED NOT NULL, - - created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, - deleted_at TIMESTAMP NULL DEFAULT NULL, - - PRIMARY KEY (id), - FOREIGN KEY fk_game_winner_game_id_game_id (game_id) - REFERENCES game(id) ON DELETE CASCADE, - FOREIGN KEY fk_game_winner_player_id_player_id (player_id) - REFERENCES player(id) ON DELETE CASCADE -); ---rollback DROP TABLE IF EXISTS game_winner; - ---changeset josephspurrier:5 -CREATE TABLE game_player ( - id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, - game_id BIGINT(10) UNSIGNED NOT NULL, - player_id BIGINT(10) UNSIGNED NOT NULL, - kills SMALLINT(10) UNSIGNED NOT NULL, - deaths SMALLINT(10) UNSIGNED NOT NULL, - assists SMALLINT(10) UNSIGNED NOT NULL, - damage_dealt MEDIUMINT(10) UNSIGNED NOT NULL, - - created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, - deleted_at TIMESTAMP NULL DEFAULT NULL, - - PRIMARY KEY (id), - FOREIGN KEY fk_game_player_game_id_game_id (game_id) - REFERENCES game(id) ON DELETE CASCADE, - FOREIGN KEY fk_game_player_player_id_player_id (player_id) - REFERENCES player(id) ON DELETE CASCADE -); ---rollback DROP TABLE IF EXISTS game_player; +--changeset josephspurrier:2 +INSERT INTO `user_status` (`id`, `status`, `created_at`, `updated_at`, `deleted`) VALUES +(1, 'active', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0), +(2, 'inactive', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0); +--rollback TRUNCATE TABLE user_status; ---changeset josephspurrier:5 -CREATE TABLE game_player2 ( - id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, +--changeset josephspurrier:2 +SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO'; +CREATE TABLE user ( + id VARCHAR(36) NOT NULL, + + first_name VARCHAR(50) NOT NULL, + last_name VARCHAR(50) NOT NULL, + email VARCHAR(100) NOT NULL, + password CHAR(60) NOT NULL, + + status_id TINYINT(1) UNSIGNED NOT NULL DEFAULT 1, + + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP DEFAULT 0, + + UNIQUE KEY (email), + CONSTRAINT `f_user_status` FOREIGN KEY (`status_id`) REFERENCES `user_status` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, + + PRIMARY KEY (id) ); ---rollback DROP TABLE IF EXISTS game_player2; \ No newline at end of file +--rollback DROP TABLE user; \ No newline at end of file diff --git a/src/app/webapi/pkg/basemigrate/testdata/success.sql b/src/app/webapi/pkg/basemigrate/testdata/success.sql index f334843..6526d0a 100644 --- a/src/app/webapi/pkg/basemigrate/testdata/success.sql +++ b/src/app/webapi/pkg/basemigrate/testdata/success.sql @@ -1,85 +1,43 @@ --changeset josephspurrier:1 -CREATE TABLE player ( - id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, - steam_id VARCHAR(191) NOT NULL, - account_name VARCHAR(191) NOT NULL, - - created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, - deleted_at TIMESTAMP NULL DEFAULT NULL, - - PRIMARY KEY (id), - CONSTRAINT u_player_account_name UNIQUE (account_name) +SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO'; +CREATE TABLE user_status ( + id TINYINT(1) UNSIGNED NOT NULL AUTO_INCREMENT, + + status VARCHAR(25) NOT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted TINYINT(1) UNSIGNED NOT NULL DEFAULT 0, + + PRIMARY KEY (id) ); ---rollback DROP TABLE IF EXISTS player; ---changeset josephspurrier:2 -CREATE TABLE player_login ( - id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, - player_id BIGINT(10) UNSIGNED NOT NULL, - display_name VARCHAR(191) NOT NULL, - ip VARCHAR(191) NOT NULL, - login_at TIMESTAMP NOT NULL, - - created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, - deleted_at TIMESTAMP NULL DEFAULT NULL, +--rollback DROP TABLE user_status; - PRIMARY KEY (id), - FOREIGN KEY fk_player_login_player_id_player_id (player_id) - REFERENCES player(id) ON DELETE CASCADE -); ---rollback DROP TABLE IF EXISTS player_login; +--changeset josephspurrier:2 +INSERT INTO `user_status` (`id`, `status`, `created_at`, `updated_at`, `deleted`) VALUES +(1, 'active', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0), +(2, 'inactive', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0); +--rollback TRUNCATE TABLE user_status; --changeset josephspurrier:3 -CREATE TABLE game ( - id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, - start_at TIMESTAMP NULL, - end_at TIMESTAMP NULL, - - created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, - deleted_at TIMESTAMP NULL DEFAULT NULL, - +SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO'; +CREATE TABLE user ( + id VARCHAR(36) NOT NULL, + + first_name VARCHAR(50) NOT NULL, + last_name VARCHAR(50) NOT NULL, + email VARCHAR(100) NOT NULL, + password CHAR(60) NOT NULL, + + status_id TINYINT(1) UNSIGNED NOT NULL DEFAULT 1, + + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP DEFAULT 0, + + UNIQUE KEY (email), + CONSTRAINT `f_user_status` FOREIGN KEY (`status_id`) REFERENCES `user_status` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, + PRIMARY KEY (id) ); ---rollback DROP TABLE IF EXISTS game; - ---changeset josephspurrier:4 -CREATE TABLE game_winner ( - id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, - game_id BIGINT(10) UNSIGNED NOT NULL, - player_id BIGINT(10) UNSIGNED NOT NULL, - - created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, - deleted_at TIMESTAMP NULL DEFAULT NULL, - - PRIMARY KEY (id), - FOREIGN KEY fk_game_winner_game_id_game_id (game_id) - REFERENCES game(id) ON DELETE CASCADE, - FOREIGN KEY fk_game_winner_player_id_player_id (player_id) - REFERENCES player(id) ON DELETE CASCADE -); ---rollback DROP TABLE IF EXISTS game_winner; - ---changeset josephspurrier:5 -CREATE TABLE game_player ( - id BIGINT(10) UNSIGNED NOT NULL AUTO_INCREMENT, - game_id BIGINT(10) UNSIGNED NOT NULL, - player_id BIGINT(10) UNSIGNED NOT NULL, - kills SMALLINT(10) UNSIGNED NOT NULL, - deaths SMALLINT(10) UNSIGNED NOT NULL, - assists SMALLINT(10) UNSIGNED NOT NULL, - damage_dealt MEDIUMINT(10) UNSIGNED NOT NULL, - - created_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NULL DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, - deleted_at TIMESTAMP NULL DEFAULT NULL, - - PRIMARY KEY (id), - FOREIGN KEY fk_game_player_game_id_game_id (game_id) - REFERENCES game(id) ON DELETE CASCADE, - FOREIGN KEY fk_game_player_player_id_player_id (player_id) - REFERENCES player(id) ON DELETE CASCADE -); ---rollback DROP TABLE IF EXISTS game_player; \ No newline at end of file +--rollback DROP TABLE user; \ No newline at end of file From ca57db70cd64187f988e13a43bdbb6ef4d3f212d Mon Sep 17 00:00:00 2001 From: Joseph Spurrier Date: Wed, 11 Jul 2018 06:15:22 -0400 Subject: [PATCH 03/14] Add test commands, fix failing tests, standardize comments --- README.md | 19 ++++++++++++++++--- src/app/webapi/component/user/create.go | 2 +- src/app/webapi/component/user/index.go | 2 +- src/app/webapi/component/user/show.go | 2 +- src/app/webapi/component/user/update.go | 2 +- src/app/webapi/internal/testutil/database.go | 17 ++++++++++------- .../pkg/basemigrate/basemigrate_test.go | 4 ++++ src/app/webapi/pkg/structcopy/structcopy.go | 17 ++++++++--------- src/app/webapi/store/user.go | 13 +------------ 9 files changed, 43 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 6d6b9f7..012a26c 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,8 @@ You must use Go 1.7 or newer because it uses the http context. Use one of the following commands to start a MySQL container with Docker: -- Start MySQL without a password: `docker run -d -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=yes mysql:5.7` -- Start MySQL with a password: `docker run -d -p 3306:3306 -e MYSQL_ROOT_PASSWORD=somepassword mysql:5.7` +- Start MySQL without a password: `docker run -d --name=mysql57 -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=yes mysql:5.7` +- Start MySQL with a password: `docker run -d --name=mysql57 -p 3306:3306 -e MYSQL_ROOT_PASSWORD=somepassword mysql:5.7` Start MySQL and import `migration/mysql.sql` to create the database and tables. @@ -312,7 +312,20 @@ func (p *Endpoint) Index(w http.ResponseWriter, r *http.Request) (int, error) { You can disable logging on the server by setting an environment variable: `WEBAPI_LOG_LEVEL=none` -## Test Coverage +## Testing + +All the tests use a database called: `webapitest`. The quickest way to get it set up is: + +```bash +# Launch MySQL in docker container. +docker run -d --name=mysql57 -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=yes mysql:5.7 + +# Create the database via docker exec. +docker exec mysql57 sh -c 'exec mysql -uroot -e "CREATE DATABASE IF NOT EXISTS webapitest DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;"' + +# Or create the database manually. +CREATE DATABASE webapitest DEFAULT CHARSET = utf8 COLLATE = utf8_unicode_ci; +``` You can use these commands to run tests: diff --git a/src/app/webapi/component/user/create.go b/src/app/webapi/component/user/create.go index 1cac7d0..f0f58a4 100644 --- a/src/app/webapi/component/user/create.go +++ b/src/app/webapi/component/user/create.go @@ -48,7 +48,7 @@ func (p *Endpoint) Create(w http.ResponseWriter, r *http.Request) (int, error) { // Create the DB store. u := store.NewUser(p.DB, p.Q) - // Check for existing user. + // Check for existing item. exists, _, err := u.ExistsByField(u, "email", req.Email) if err != nil { return http.StatusInternalServerError, err diff --git a/src/app/webapi/component/user/index.go b/src/app/webapi/component/user/index.go index 8a16b7f..35aa4b4 100644 --- a/src/app/webapi/component/user/index.go +++ b/src/app/webapi/component/user/index.go @@ -1,10 +1,10 @@ package user import ( - "app/webapi/pkg/structcopy" "net/http" "app/webapi/model" + "app/webapi/pkg/structcopy" "app/webapi/store" ) diff --git a/src/app/webapi/component/user/show.go b/src/app/webapi/component/user/show.go index 2a132a5..61765dd 100644 --- a/src/app/webapi/component/user/show.go +++ b/src/app/webapi/component/user/show.go @@ -45,7 +45,7 @@ func (p *Endpoint) Show(w http.ResponseWriter, r *http.Request) (int, error) { // Create the DB store. u := store.NewUser(p.DB, p.Q) - // Get a user. + // Get an item by ID. exists, err := u.FindOneByID(u, req.UserID) if err != nil { return http.StatusInternalServerError, err diff --git a/src/app/webapi/component/user/update.go b/src/app/webapi/component/user/update.go index 061ea4d..f71850d 100644 --- a/src/app/webapi/component/user/update.go +++ b/src/app/webapi/component/user/update.go @@ -51,7 +51,7 @@ func (p *Endpoint) Update(w http.ResponseWriter, r *http.Request) (int, error) { // Create the DB store. u := store.NewUser(p.DB, p.Q) - // Determine if the user exists. + // Determine if the item exists. exists, err := u.ExistsByID(u, req.UserID) if err != nil { return http.StatusInternalServerError, err diff --git a/src/app/webapi/internal/testutil/database.go b/src/app/webapi/internal/testutil/database.go index 3699665..e28eb92 100644 --- a/src/app/webapi/internal/testutil/database.go +++ b/src/app/webapi/internal/testutil/database.go @@ -30,13 +30,18 @@ func ConnectDatabase(dbSpecificDB bool) *database.DBW { return dbw } -// LoadDatabase will set up the DB for the tests. -func LoadDatabase(t *testing.T) { +// ResetDatabase will drop and create the test database. +func ResetDatabase() { db := ConnectDatabase(false) db.Exec(`DROP DATABASE IF EXISTS webapitest`) db.Exec(`CREATE DATABASE webapitest DEFAULT CHARSET = utf8 COLLATE = utf8_unicode_ci`) +} - db = ConnectDatabase(true) +// LoadDatabase will set up the DB for the tests. +func LoadDatabase(t *testing.T) { + ResetDatabase() + + db := ConnectDatabase(true) b, err := ioutil.ReadFile("../../../../../migration/tables-only.sql") if err != nil { t.Error(err) @@ -57,11 +62,9 @@ func LoadDatabase(t *testing.T) { // LoadDatabaseFromFile will set up the DB for the tests. func LoadDatabaseFromFile(file string) { - db := ConnectDatabase(false) - db.Exec(`DROP DATABASE IF EXISTS webapitest`) - db.Exec(`CREATE DATABASE webapitest DEFAULT CHARSET = utf8 COLLATE = utf8_unicode_ci`) + ResetDatabase() - db = ConnectDatabase(true) + db := ConnectDatabase(true) b, err := ioutil.ReadFile(file) if err != nil { log.Println(err) diff --git a/src/app/webapi/pkg/basemigrate/basemigrate_test.go b/src/app/webapi/pkg/basemigrate/basemigrate_test.go index b60ac8d..f93e1c6 100644 --- a/src/app/webapi/pkg/basemigrate/basemigrate_test.go +++ b/src/app/webapi/pkg/basemigrate/basemigrate_test.go @@ -3,22 +3,26 @@ package basemigrate_test import ( "testing" + "app/webapi/internal/testutil" "app/webapi/pkg/basemigrate" "github.com/stretchr/testify/assert" ) func TestMigration(t *testing.T) { + testutil.ResetDatabase() err := basemigrate.Migrate("testdata/success.sql", false) assert.Nil(t, err) } func TestMigrationFailDuplicate(t *testing.T) { + testutil.ResetDatabase() err := basemigrate.Migrate("testdata/fail-duplicate.sql", false) assert.Contains(t, err.Error(), "checksum does not match") } func TestParse(t *testing.T) { + testutil.ResetDatabase() arr, err := basemigrate.ParseFile("testdata/success.sql") assert.Nil(t, err) assert.Equal(t, 3, len(arr)) diff --git a/src/app/webapi/pkg/structcopy/structcopy.go b/src/app/webapi/pkg/structcopy/structcopy.go index 2d782a4..e3920de 100644 --- a/src/app/webapi/pkg/structcopy/structcopy.go +++ b/src/app/webapi/pkg/structcopy/structcopy.go @@ -34,15 +34,14 @@ func ByTag(src interface{}, srcTag string, dst interface{}, dstTag string) (err // Loop through each field. keysSrc := vs.Type() keysDst := vd.Type() - for jS := 0; jS < vs.NumField(); jS++ { - fieldS := vs.Field(jS) - tagS := keysSrc.Field(jS).Tag - - for jD := 0; jD < vd.NumField(); jD++ { - fieldD := vd.Field(jD) - tagD := keysDst.Field(jD).Tag - - // Set the "status" field. + for jD := 0; jD < vd.NumField(); jD++ { + fieldD := vd.Field(jD) + tagD := keysDst.Field(jD).Tag + for jS := 0; jS < vs.NumField(); jS++ { + fieldS := vs.Field(jS) + tagS := keysSrc.Field(jS).Tag + + // If the tags match, copy the value from src to dst field. if tagS.Get(srcTag) == tagD.Get(dstTag) { if fieldS.Type() != fieldD.Type() { return fmt.Errorf("field types do not match - src type '%v' for tag '%v' do not match dst type '%v' for tag '%v'", diff --git a/src/app/webapi/store/user.go b/src/app/webapi/store/user.go index bb3a95e..75442f2 100644 --- a/src/app/webapi/store/user.go +++ b/src/app/webapi/store/user.go @@ -60,19 +60,13 @@ func (x UserGroup) PrimaryKey() string { return "id" } -// ***************************************************************************** -// Create -// ***************************************************************************** - // Create adds a new user. func (x *User) Create(firstName, lastName, email, password string) (string, error) { - // Generate a UUID. uuid, err := securegen.UUID() if err != nil { return "", err } - // Create the user. _, err = x.db.Exec(` INSERT INTO user (id, first_name, last_name, email, password, status_id) @@ -84,13 +78,8 @@ func (x *User) Create(firstName, lastName, email, password string) (string, erro return uuid, err } -// ***************************************************************************** -// Update -// ***************************************************************************** - -// Update makes changes to one entity. +// Update makes changes to a user. func (x *User) Update(ID, firstName, lastName, email, password string) (err error) { - // Update the entity. _, err = x.db.Exec(` UPDATE user SET From a31462492a66bc544910e69b55d8e2e6a63a9806 Mon Sep 17 00:00:00 2001 From: Joseph Spurrier Date: Sun, 15 Jul 2018 19:55:21 -0400 Subject: [PATCH 04/14] Add rollback migration --- src/app/webapi/cmd/cliapp/cliapp.go | 17 ++-- src/app/webapi/pkg/basemigrate/basemigrate.go | 90 ++++--------------- .../pkg/basemigrate/basemigrate_test.go | 6 +- src/app/webapi/pkg/basemigrate/migrate.go | 88 ++++++++++++++++++ src/app/webapi/pkg/basemigrate/parse.go | 4 +- src/app/webapi/pkg/basemigrate/reset.go | 87 ++++++++++++++++++ 6 files changed, 210 insertions(+), 82 deletions(-) create mode 100644 src/app/webapi/pkg/basemigrate/migrate.go create mode 100644 src/app/webapi/pkg/basemigrate/reset.go diff --git a/src/app/webapi/cmd/cliapp/cliapp.go b/src/app/webapi/cmd/cliapp/cliapp.go index 793cd78..462cd77 100644 --- a/src/app/webapi/cmd/cliapp/cliapp.go +++ b/src/app/webapi/cmd/cliapp/cliapp.go @@ -15,11 +15,12 @@ import ( var ( app = kingpin.New("cliapp", "A command-line application to perform tasks for the webapi.") - cGenerate = app.Command("generate", "Generate 256 bit (32 byte) base64 encoded JWT.") - cDB = app.Command("migrate", "Perform actions on the database.") - cDBAll = cDB.Command("all", "Apply all changesets to the database.") - cDBReset = cDB.Command("reset", "Run all rollbacks on the database.") - cDBAllFile = cDBAll.Arg("file", "Filename of the migration file.").Required().String() + cGenerate = app.Command("generate", "Generate 256 bit (32 byte) base64 encoded JWT.") + cDB = app.Command("migrate", "Perform actions on the database.") + cDBAll = cDB.Command("all", "Apply all changesets to the database.") + cDBAllFile = cDBAll.Arg("file", "Filename of the migration file.").Required().String() + cDBReset = cDB.Command("reset", "Run all rollbacks on the database.") + cDBResetFile = cDBReset.Arg("file", "Filename of the migration file.").Required().String() ) func main() { @@ -41,5 +42,11 @@ func main() { fmt.Println(err) os.Exit(1) } + case cDBReset.FullCommand(): + err := basemigrate.Reset(*cDBResetFile, true) + if err != nil { + fmt.Println(err) + os.Exit(1) + } } } diff --git a/src/app/webapi/pkg/basemigrate/basemigrate.go b/src/app/webapi/pkg/basemigrate/basemigrate.go index e73c8a5..b1123d7 100644 --- a/src/app/webapi/pkg/basemigrate/basemigrate.go +++ b/src/app/webapi/pkg/basemigrate/basemigrate.go @@ -1,11 +1,9 @@ package basemigrate import ( - "database/sql" "errors" "fmt" "os" - "strings" ) const ( @@ -35,94 +33,40 @@ var ( ErrInvalidFormat = errors.New("invalid changeset format") ) -// ParseFile will parse a file into changesets. -func ParseFile(filename string) ([]*Changeset, error) { +// ParseFileArray will parse a file into changesets. +func ParseFileArray(filename string) ([]*Changeset, error) { f, err := os.Open(filename) if err != nil { return nil, err } defer f.Close() - return parse(f, filename) + return parseToOrderedArray(f, filename) } -// Migrate will migrate a file. -func Migrate(filename string, verbose bool) (err error) { - db, err := connect() +// ParseFileMap will parse a file into a map. +func ParseFileMap(filename string) (map[string]Changeset, error) { + f, err := os.Open(filename) if err != nil { - return err + return nil, err } + defer f.Close() - // Create the DATABASECHANGELOG. - _, err = db.Exec(sqlChangelog) - - // Get the changesets. - arr, err := ParseFile(filename) + arr, err := parseToOrderedArray(f, filename) if err != nil { - return err + return nil, err } - // Loop through each changeset. - for _, cs := range arr { - checksum := "" - newChecksum := cs.Checksum() - - // Determine if the changeset was already applied. - // Count the number of rows. - err = db.Get(&checksum, `SELECT md5sum - FROM databasechangelog - WHERE id = ? - AND author = ?`, cs.id, cs.author) - if err == nil { - // Determine if the checksums match. - if checksum != newChecksum { - return fmt.Errorf("checksum does not match - existing changeset %v:%v has checksum %v, but new changeset has checksum %v", - cs.author, cs.id, checksum, newChecksum) - } - - if verbose { - fmt.Printf("Changeset already applied: %v:%v\n", cs.author, cs.id) - } - continue - } else if err != nil && err != sql.ErrNoRows { - return fmt.Errorf("internal error on changeset %v:%v - %v", cs.author, cs.id, err.Error()) - } - - arrQueries := strings.Split(cs.Changes(), ";") - // Loop through each change. - for _, q := range arrQueries { - if len(q) == 0 { - continue - } - - // Execute the query. - _, err = db.Exec(q) - if err != nil { - return fmt.Errorf("sql error on changeset %v:%v - %v", cs.author, cs.id, err.Error()) - } - } + m := make(map[string]Changeset) - // Count the number of rows. - count := 0 - err = db.Get(&count, `SELECT COUNT(*) FROM databasechangelog`) - if err != nil { - return err - } - - // Insert the record. - _, err = db.Exec(` - INSERT INTO databasechangelog - (id,author,filename,dateexecuted,orderexecuted,md5sum,description,version) - VALUES(?,?,?,NOW(),?,?,?,?) - `, cs.id, cs.author, cs.filename, count+1, newChecksum, cs.description, cs.version) - if err != nil { - return err + for _, cs := range arr { + id := fmt.Sprintf("%v:%v", cs.author, cs.id) + if _, found := m[id]; found { + return nil, errors.New("Duplicate entry found: " + id) } - if verbose { - fmt.Printf("Changeset applied: %v:%v\n", cs.author, cs.id) - } + m[id] = *cs } - return + return m, nil } diff --git a/src/app/webapi/pkg/basemigrate/basemigrate_test.go b/src/app/webapi/pkg/basemigrate/basemigrate_test.go index b60ac8d..310e285 100644 --- a/src/app/webapi/pkg/basemigrate/basemigrate_test.go +++ b/src/app/webapi/pkg/basemigrate/basemigrate_test.go @@ -11,6 +11,8 @@ import ( func TestMigration(t *testing.T) { err := basemigrate.Migrate("testdata/success.sql", false) assert.Nil(t, err) + err = basemigrate.Reset("testdata/success.sql", false) + assert.Nil(t, err) } func TestMigrationFailDuplicate(t *testing.T) { @@ -18,8 +20,8 @@ func TestMigrationFailDuplicate(t *testing.T) { assert.Contains(t, err.Error(), "checksum does not match") } -func TestParse(t *testing.T) { - arr, err := basemigrate.ParseFile("testdata/success.sql") +func TestParseArray(t *testing.T) { + arr, err := basemigrate.ParseFileArray("testdata/success.sql") assert.Nil(t, err) assert.Equal(t, 3, len(arr)) diff --git a/src/app/webapi/pkg/basemigrate/migrate.go b/src/app/webapi/pkg/basemigrate/migrate.go new file mode 100644 index 0000000..10dbe30 --- /dev/null +++ b/src/app/webapi/pkg/basemigrate/migrate.go @@ -0,0 +1,88 @@ +package basemigrate + +import ( + "database/sql" + "fmt" + "strings" +) + +// Migrate will migrate a file. +func Migrate(filename string, verbose bool) (err error) { + db, err := connect() + if err != nil { + return err + } + + // Create the DATABASECHANGELOG. + _, err = db.Exec(sqlChangelog) + + // Get the changesets. + arr, err := ParseFileArray(filename) + if err != nil { + return err + } + + // Loop through each changeset. + for _, cs := range arr { + checksum := "" + newChecksum := cs.Checksum() + + // Determine if the changeset was already applied. + // Count the number of rows. + err = db.Get(&checksum, `SELECT md5sum + FROM databasechangelog + WHERE id = ? + AND author = ?`, cs.id, cs.author) + if err == nil { + // Determine if the checksums match. + if checksum != newChecksum { + return fmt.Errorf("checksum does not match - existing changeset %v:%v has checksum %v, but new changeset has checksum %v", + cs.author, cs.id, checksum, newChecksum) + } + + if verbose { + fmt.Printf("Changeset already applied: %v:%v\n", cs.author, cs.id) + } + continue + } else if err != nil && err != sql.ErrNoRows { + return fmt.Errorf("internal error on changeset %v:%v - %v", cs.author, cs.id, err.Error()) + } + + arrQueries := strings.Split(cs.Changes(), ";") + // Loop through each change. + for _, q := range arrQueries { + if len(q) == 0 { + continue + } + + // Execute the query. + _, err = db.Exec(q) + if err != nil { + return fmt.Errorf("sql error on changeset %v:%v - %v", cs.author, cs.id, err.Error()) + } + } + + // Count the number of rows. + count := 0 + err = db.Get(&count, `SELECT COUNT(*) FROM databasechangelog`) + if err != nil { + return err + } + + // Insert the record. + _, err = db.Exec(` + INSERT INTO databasechangelog + (id,author,filename,dateexecuted,orderexecuted,md5sum,description,version) + VALUES(?,?,?,NOW(),?,?,?,?) + `, cs.id, cs.author, cs.filename, count+1, newChecksum, cs.description, cs.version) + if err != nil { + return err + } + + if verbose { + fmt.Printf("Changeset applied: %v:%v\n", cs.author, cs.id) + } + } + + return +} diff --git a/src/app/webapi/pkg/basemigrate/parse.go b/src/app/webapi/pkg/basemigrate/parse.go index 2b7cff2..16803f6 100644 --- a/src/app/webapi/pkg/basemigrate/parse.go +++ b/src/app/webapi/pkg/basemigrate/parse.go @@ -7,8 +7,8 @@ import ( "strings" ) -// parse will split the SQL migration into pieces. -func parse(r io.Reader, filename string) ([]*Changeset, error) { +// parseToOrderedArray will split the SQL migration into an ordered array. +func parseToOrderedArray(r io.Reader, filename string) ([]*Changeset, error) { scanner := bufio.NewScanner(r) scanner.Split(bufio.ScanLines) diff --git a/src/app/webapi/pkg/basemigrate/reset.go b/src/app/webapi/pkg/basemigrate/reset.go new file mode 100644 index 0000000..d300c9b --- /dev/null +++ b/src/app/webapi/pkg/basemigrate/reset.go @@ -0,0 +1,87 @@ +package basemigrate + +import ( + "errors" + "fmt" + "strings" +) + +// DBChangeset represents the database table records. +type DBChangeset struct { + ID string `db:"id"` + Author string `db:"author"` + Filename string `db:"filename"` + OrderExecuted int `db:"orderexecuted"` +} + +// Reset will remove all migrations. +func Reset(filename string, verbose bool) (err error) { + db, err := connect() + if err != nil { + return err + } + + // Get the changesets in a map. + m, err := ParseFileMap(filename) + if err != nil { + return err + } + + // Get each changeset from the database. + results := make([]DBChangeset, 0) + err = db.Select(&results, ` + SELECT id, author, filename, orderexecuted + FROM databasechangelog + ORDER BY orderexecuted DESC;`) + if err != nil { + return err + } + + if len(results) == 0 { + if verbose { + fmt.Println("No rollbacks to perform.") + return nil + } + } + + // Loop through each changeset. + for _, r := range results { + id := fmt.Sprintf("%v:%v", r.Author, r.ID) + + cs, ok := m[id] + if !ok { + return errors.New("changeset is missing: " + id) + } + + arrQueries := strings.Split(cs.Rollbacks(), ";") + + // Loop through each rollback. + for _, q := range arrQueries { + if len(q) == 0 { + continue + } + + // Execute the query. + _, err = db.Exec(q) + if err != nil { + return fmt.Errorf("sql error on rollback %v:%v - %v", cs.author, cs.id, err.Error()) + } + } + + // Delete the record. + _, err = db.Exec(` + DELETE FROM databasechangelog + WHERE id = ? AND author = ? AND filename = ? + LIMIT 1 + `, cs.id, cs.author, cs.filename) + if err != nil { + return err + } + + if verbose { + fmt.Printf("Rollback applied: %v:%v\n", cs.author, cs.id) + } + } + + return +} From 45fa0983976f27305f67a810a419967f045e69be Mon Sep 17 00:00:00 2001 From: Joseph Spurrier Date: Mon, 16 Jul 2018 05:34:48 -0400 Subject: [PATCH 05/14] Add database rollbacks --- src/app/webapi/cmd/cliapp/cliapp.go | 38 +++++-- src/app/webapi/internal/testutil/database.go | 17 +-- .../pkg/basemigrate/basemigrate_test.go | 100 +++++++++++++++--- src/app/webapi/pkg/basemigrate/helper.go | 13 +-- src/app/webapi/pkg/basemigrate/migrate.go | 15 ++- src/app/webapi/pkg/basemigrate/reset.go | 14 ++- src/app/webapi/pkg/database/connection.go | 16 +-- src/app/webapi/pkg/env/env.go | 68 ++++++++++++ src/app/webapi/pkg/env/env_test.go | 79 ++++++++++++++ 9 files changed, 313 insertions(+), 47 deletions(-) create mode 100644 src/app/webapi/pkg/env/env.go create mode 100644 src/app/webapi/pkg/env/env_test.go diff --git a/src/app/webapi/cmd/cliapp/cliapp.go b/src/app/webapi/cmd/cliapp/cliapp.go index 462cd77..f24d4af 100644 --- a/src/app/webapi/cmd/cliapp/cliapp.go +++ b/src/app/webapi/cmd/cliapp/cliapp.go @@ -1,12 +1,12 @@ package main import ( - "app/webapi/pkg/basemigrate" "encoding/base64" "fmt" "log" "os" + "app/webapi/pkg/basemigrate" "app/webapi/pkg/securegen" kingpin "gopkg.in/alecthomas/kingpin.v2" @@ -15,12 +15,21 @@ import ( var ( app = kingpin.New("cliapp", "A command-line application to perform tasks for the webapi.") - cGenerate = app.Command("generate", "Generate 256 bit (32 byte) base64 encoded JWT.") - cDB = app.Command("migrate", "Perform actions on the database.") - cDBAll = cDB.Command("all", "Apply all changesets to the database.") - cDBAllFile = cDBAll.Arg("file", "Filename of the migration file.").Required().String() + cGenerate = app.Command("generate", "Generate 256 bit (32 byte) base64 encoded JWT.") + cDB = app.Command("migrate", "Perform actions on the database.") + cDBAll = cDB.Command("all", "Apply all changesets to the database.") + cDBAllFile = cDBAll.Arg("file", "Filename of the migration file [string].").Required().String() + + cDBUp = cDB.Command("up", "Apply a specific number of changesets to the database.") + cDBUpCount = cDBUp.Arg("count", "Number of changesets [int].").Required().Int() + cDBUpFile = cDBUp.Arg("file", "Filename of the migration file [string].").Required().String() + cDBReset = cDB.Command("reset", "Run all rollbacks on the database.") - cDBResetFile = cDBReset.Arg("file", "Filename of the migration file.").Required().String() + cDBResetFile = cDBReset.Arg("file", "Filename of the migration file [string].").Required().String() + + cDBDown = cDB.Command("down", "Apply a specific number of rollbacks to the database.") + cDBDownCount = cDBDown.Arg("count", "Number of rollbacks [int].").Required().Int() + cDBDownFile = cDBDown.Arg("file", "Filename of the migration file [string].").Required().String() ) func main() { @@ -37,13 +46,26 @@ func main() { enc := base64.StdEncoding.EncodeToString(b) fmt.Println(enc) case cDBAll.FullCommand(): - err := basemigrate.Migrate(*cDBAllFile, true) + err := basemigrate.Migrate(*cDBAllFile, 0, true) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + case cDBUp.FullCommand(): + err := basemigrate.Migrate(*cDBUpFile, *cDBUpCount, true) if err != nil { fmt.Println(err) os.Exit(1) } case cDBReset.FullCommand(): - err := basemigrate.Reset(*cDBResetFile, true) + err := basemigrate.Reset(*cDBResetFile, 0, true) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + case cDBDown.FullCommand(): + err := basemigrate.Reset(*cDBDownFile, *cDBDownCount, true) if err != nil { fmt.Println(err) os.Exit(1) diff --git a/src/app/webapi/internal/testutil/database.go b/src/app/webapi/internal/testutil/database.go index 3699665..6bd56f5 100644 --- a/src/app/webapi/internal/testutil/database.go +++ b/src/app/webapi/internal/testutil/database.go @@ -30,13 +30,18 @@ func ConnectDatabase(dbSpecificDB bool) *database.DBW { return dbw } -// LoadDatabase will set up the DB for the tests. -func LoadDatabase(t *testing.T) { +// PrepDatabase will drop the test database and create a new test database. +func PrepDatabase() { db := ConnectDatabase(false) db.Exec(`DROP DATABASE IF EXISTS webapitest`) db.Exec(`CREATE DATABASE webapitest DEFAULT CHARSET = utf8 COLLATE = utf8_unicode_ci`) +} - db = ConnectDatabase(true) +// LoadDatabase will set up the DB for the tests. +func LoadDatabase(t *testing.T) { + PrepDatabase() + + db := ConnectDatabase(true) b, err := ioutil.ReadFile("../../../../../migration/tables-only.sql") if err != nil { t.Error(err) @@ -57,11 +62,9 @@ func LoadDatabase(t *testing.T) { // LoadDatabaseFromFile will set up the DB for the tests. func LoadDatabaseFromFile(file string) { - db := ConnectDatabase(false) - db.Exec(`DROP DATABASE IF EXISTS webapitest`) - db.Exec(`CREATE DATABASE webapitest DEFAULT CHARSET = utf8 COLLATE = utf8_unicode_ci`) + PrepDatabase() - db = ConnectDatabase(true) + db := ConnectDatabase(true) b, err := ioutil.ReadFile(file) if err != nil { log.Println(err) diff --git a/src/app/webapi/pkg/basemigrate/basemigrate_test.go b/src/app/webapi/pkg/basemigrate/basemigrate_test.go index 310e285..a904c7e 100644 --- a/src/app/webapi/pkg/basemigrate/basemigrate_test.go +++ b/src/app/webapi/pkg/basemigrate/basemigrate_test.go @@ -1,38 +1,110 @@ package basemigrate_test import ( + "os" "testing" + "app/webapi/internal/testutil" "app/webapi/pkg/basemigrate" "github.com/stretchr/testify/assert" ) +func setEnv() { + os.Setenv("DB_HOSTNAME", "127.0.0.1") + os.Setenv("DB_PORT", "3306") + os.Setenv("DB_USERNAME", "root") + os.Setenv("DB_PASSWORD", "") + os.Setenv("DB_DATABASE", "webapitest") + os.Setenv("DB_PARAMETER", "parseTime=true&allowNativePasswords=true") +} + +func unsetEnv() { + os.Unsetenv("DB_HOSTNAME") + os.Unsetenv("DB_PORT") + os.Unsetenv("DB_USERNAME") + os.Unsetenv("DB_PASSWORD") + os.Unsetenv("DB_DATABASE") + os.Unsetenv("DB_PARAMETER") +} func TestMigration(t *testing.T) { - err := basemigrate.Migrate("testdata/success.sql", false) + setEnv() + defer unsetEnv() + + testutil.PrepDatabase() + db := testutil.ConnectDatabase(true) + + // Run migration. + err := basemigrate.Migrate("testdata/success.sql", 0, false) + assert.Nil(t, err) + + // Count the records. + rows := 0 + err = db.Get(&rows, `SELECT count(*) from databasechangelog`) + assert.Nil(t, err) + assert.Equal(t, 3, rows) + + // Run migration again. + err = basemigrate.Migrate("testdata/success.sql", 0, false) + assert.Nil(t, err) + + // Remove all migrations. + err = basemigrate.Reset("testdata/success.sql", 0, false) + assert.Nil(t, err) + + rows = 0 + err = db.Get(&rows, `SELECT count(*) from databasechangelog`) + assert.Nil(t, err) + assert.Equal(t, 0, rows) + + // Remove all migrations again. + err = basemigrate.Reset("testdata/success.sql", 0, false) assert.Nil(t, err) - err = basemigrate.Reset("testdata/success.sql", false) + + // Run 2 migrations. + err = basemigrate.Migrate("testdata/success.sql", 2, false) + assert.Nil(t, err) + + rows = 0 + err = db.Get(&rows, `SELECT count(*) from databasechangelog`) + assert.Nil(t, err) + assert.Equal(t, 2, rows) + + // Remove 1 migration. + err = basemigrate.Reset("testdata/success.sql", 1, false) + assert.Nil(t, err) + + rows = 0 + err = db.Get(&rows, `SELECT count(*) from databasechangelog`) assert.Nil(t, err) + assert.Equal(t, 1, rows) } func TestMigrationFailDuplicate(t *testing.T) { - err := basemigrate.Migrate("testdata/fail-duplicate.sql", false) + setEnv() + defer unsetEnv() + + testutil.PrepDatabase() + db := testutil.ConnectDatabase(true) + + err := basemigrate.Migrate("testdata/fail-duplicate.sql", 0, false) assert.Contains(t, err.Error(), "checksum does not match") + + rows := 0 + err = db.Get(&rows, `SELECT count(*) from databasechangelog`) + assert.Nil(t, err) + assert.Equal(t, 2, rows) } -func TestParseArray(t *testing.T) { +func TestParse(t *testing.T) { + setEnv() + defer unsetEnv() + arr, err := basemigrate.ParseFileArray("testdata/success.sql") assert.Nil(t, err) assert.Equal(t, 3, len(arr)) - //basemigrate.Debug(arr) - - /*for _, v := range arr { - fmt.Println(v.Changes()) - fmt.Println(v.Rollbacks()) - fmt.Println("MD5:", v.Checksum()) - break - } - - fmt.Println("Total:", len(arr))*/ + m, err := basemigrate.ParseFileMap("testdata/success.sql") + assert.Nil(t, err) + assert.Equal(t, 3, len(m)) } diff --git a/src/app/webapi/pkg/basemigrate/helper.go b/src/app/webapi/pkg/basemigrate/helper.go index 5575690..ae64e8f 100644 --- a/src/app/webapi/pkg/basemigrate/helper.go +++ b/src/app/webapi/pkg/basemigrate/helper.go @@ -7,6 +7,7 @@ import ( "io" "app/webapi/pkg/database" + "app/webapi/pkg/env" "github.com/jmoiron/sqlx" ) @@ -14,12 +15,12 @@ import ( // connect will connect to the database. func connect() (*sqlx.DB, error) { dbc := new(database.Connection) - dbc.Hostname = "127.0.0.1" - dbc.Port = 3306 - dbc.Username = "root" - dbc.Password = "" - dbc.Database = "webapitest" - dbc.Parameter = "parseTime=true&allowNativePasswords=true" + + // Load the struct from environment variables. + err := env.Unmarshal(dbc) + if err != nil { + return nil, err + } return dbc.Connect(true) } diff --git a/src/app/webapi/pkg/basemigrate/migrate.go b/src/app/webapi/pkg/basemigrate/migrate.go index 10dbe30..adffbc6 100644 --- a/src/app/webapi/pkg/basemigrate/migrate.go +++ b/src/app/webapi/pkg/basemigrate/migrate.go @@ -6,8 +6,9 @@ import ( "strings" ) -// Migrate will migrate a file. -func Migrate(filename string, verbose bool) (err error) { +// Migrate will perform all the migrations in a file. If max is 0, all +// migrations are run. +func Migrate(filename string, max int, verbose bool) (err error) { db, err := connect() if err != nil { return err @@ -22,6 +23,8 @@ func Migrate(filename string, verbose bool) (err error) { return err } + maxCounter := 0 + // Loop through each changeset. for _, cs := range arr { checksum := "" @@ -82,6 +85,14 @@ func Migrate(filename string, verbose bool) (err error) { if verbose { fmt.Printf("Changeset applied: %v:%v\n", cs.author, cs.id) } + + // Only perform the maxium number of changes based on the max value. + maxCounter++ + if max != 0 { + if maxCounter >= max { + break + } + } } return diff --git a/src/app/webapi/pkg/basemigrate/reset.go b/src/app/webapi/pkg/basemigrate/reset.go index d300c9b..0a6b5aa 100644 --- a/src/app/webapi/pkg/basemigrate/reset.go +++ b/src/app/webapi/pkg/basemigrate/reset.go @@ -14,8 +14,8 @@ type DBChangeset struct { OrderExecuted int `db:"orderexecuted"` } -// Reset will remove all migrations. -func Reset(filename string, verbose bool) (err error) { +// Reset will remove all migrations. If max is 0, all rollbacks are run. +func Reset(filename string, max int, verbose bool) (err error) { db, err := connect() if err != nil { return err @@ -44,6 +44,8 @@ func Reset(filename string, verbose bool) (err error) { } } + maxCounter := 0 + // Loop through each changeset. for _, r := range results { id := fmt.Sprintf("%v:%v", r.Author, r.ID) @@ -81,6 +83,14 @@ func Reset(filename string, verbose bool) (err error) { if verbose { fmt.Printf("Rollback applied: %v:%v\n", cs.author, cs.id) } + + // Only perform the maxium number of changes based on the max value. + maxCounter++ + if max != 0 { + if maxCounter >= max { + break + } + } } return diff --git a/src/app/webapi/pkg/database/connection.go b/src/app/webapi/pkg/database/connection.go index 630b198..40df6d2 100644 --- a/src/app/webapi/pkg/database/connection.go +++ b/src/app/webapi/pkg/database/connection.go @@ -10,14 +10,14 @@ import ( // Connection holds the details for the MySQL connection. type Connection struct { - Username string `json:"Username"` - Password string `json:"Password"` - Database string `json:"Database"` - Charset string `json:"Charset"` - Collation string `json:"Collation"` - Hostname string `json:"Hostname"` - Port int `json:"Port"` - Parameter string `json:"Parameter"` + Username string `json:"Username" env:"DB_USERNAME"` + Password string `json:"Password" env:"DB_PASSWORD"` + Database string `json:"Database" env:"DB_DATABASE"` + Charset string `json:"Charset" env:"DB_CHARSET"` + Collation string `json:"Collation" env:"DB_COLLATION"` + Hostname string `json:"Hostname" env:"DB_HOSTNAME"` + Port int `json:"Port" env:"DB_PORT"` + Parameter string `json:"Parameter" env:"DB_PARAMETER"` } // ***************************************************************************** diff --git a/src/app/webapi/pkg/env/env.go b/src/app/webapi/pkg/env/env.go new file mode 100644 index 0000000..83132f7 --- /dev/null +++ b/src/app/webapi/pkg/env/env.go @@ -0,0 +1,68 @@ +// Package env will fill a struct from environment variables. +package env + +import ( + "fmt" + "os" + "reflect" + "strconv" +) + +// Unmarshal will fill a struct from environment variables. It supports struct +// values of string, int, and bool. +func Unmarshal(dst interface{}) (err error) { + // Ensure a pointer is passed in. + vdst := reflect.ValueOf(dst) + if vdst.Kind() != reflect.Ptr { + return fmt.Errorf("dst type not pointer - expected 'struct pointer' but got '%v'", vdst.Kind()) + } + + // Ensure a struct is passed in. + vd := reflect.Indirect(reflect.ValueOf(dst)) + if vd.Kind() != reflect.Struct { + return fmt.Errorf("dst type not struct - expected 'struct pointer' but got '%v pointer'", vd.Kind()) + } + + // Loop through each field. + keys := vd.Type() + for j := 0; j < vd.NumField(); j++ { + field := keys.Field(j) + tag := keys.Field(j).Tag + + // Get the env tag. + envname := tag.Get("env") + + // Get the environment variable from the tag. + val := os.Getenv(envname) + + // If the environment variable exists, set the value. + if len(val) > 0 { + vr := reflect.ValueOf(val) + + // If the types are the same, then set the field. + if vr.Type() == field.Type { + vd.Field(j).Set(vr) + continue + } + + // If the types are not the same, perform type conversion. + f := vd.Field(j) + switch f.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + i64, err := strconv.ParseInt(val, 10, 0) + if err != nil { + return err + } + f.SetInt(i64) + case reflect.Bool: + b, err := strconv.ParseBool(val) + if err != nil { + return err + } + f.SetBool(b) + } + } + } + + return +} diff --git a/src/app/webapi/pkg/env/env_test.go b/src/app/webapi/pkg/env/env_test.go new file mode 100644 index 0000000..731fc63 --- /dev/null +++ b/src/app/webapi/pkg/env/env_test.go @@ -0,0 +1,79 @@ +package env_test + +import ( + "os" + "testing" + + "app/webapi/pkg/env" + + "github.com/stretchr/testify/assert" +) + +// Connection holds the details for the MySQL connection. +type Connection struct { + Username string `json:"Username" env:"DB_USERNAME"` + Password string `json:"Password" env:"DB_PASSWORD"` + Database string `json:"Database" env:"DB_DATABASE"` + Port int `json:"Port" env:"DB_PORT"` + SSL bool `json:"SSL" env:"DB_SSL"` +} + +func TestUnmarshalEmpty(t *testing.T) { + c := new(Connection) + err := env.Unmarshal(c) + assert.Nil(t, err) + + assert.Equal(t, "", c.Username) + assert.Equal(t, "", c.Password) + assert.Equal(t, "", c.Database) + assert.Equal(t, 0, c.Port) + assert.Equal(t, false, c.SSL) +} + +func TestUnmarshalFilled(t *testing.T) { + os.Setenv("DB_USERNAME", "a") + os.Setenv("DB_PASSWORD", "B") + os.Setenv("DB_DATABASE", "c123") + os.Setenv("DB_PORT", "3306") + os.Setenv("DB_SSL", "TRUE") + + c := new(Connection) + err := env.Unmarshal(c) + assert.Nil(t, err) + + assert.Equal(t, "a", c.Username) + assert.Equal(t, "B", c.Password) + assert.Equal(t, "c123", c.Database) + assert.Equal(t, 3306, c.Port) + assert.Equal(t, true, c.SSL) + + os.Unsetenv("DB_USERNAME") + os.Unsetenv("DB_PASSWORD") + os.Unsetenv("DB_DATABASE") + os.Unsetenv("DB_PORT") + os.Unsetenv("DB_SSL") +} + +func TestUnmarshalError(t *testing.T) { + c := "string" + err := env.Unmarshal(c) + assert.Contains(t, err.Error(), "type not pointer") + + d := "string" + err = env.Unmarshal(&d) + assert.Contains(t, err.Error(), "type not struct") + + os.Setenv("DB_SSL", "TRUEX") + f := new(Connection) + err = env.Unmarshal(f) + assert.NotNil(t, err) + assert.Equal(t, false, f.SSL) + os.Unsetenv("DB_SSL") + + os.Setenv("DB_PORT", "monkey") + g := new(Connection) + err = env.Unmarshal(f) + assert.NotNil(t, err) + assert.Equal(t, false, g.SSL) + os.Unsetenv("DB_PORT") +} From 18abe8bd3207369199a73ed1c492278de292a19b Mon Sep 17 00:00:00 2001 From: Joseph Spurrier Date: Mon, 16 Jul 2018 05:59:26 -0400 Subject: [PATCH 06/14] Move basemigrate to internal --- src/app/webapi/cmd/cliapp/cliapp.go | 6 +- src/app/webapi/cmd/cliapp/cliapp_test.go | 171 ++++++++++++++++++ .../cliapp}/testdata/success.sql | 0 .../basemigrate/basemigrate.go | 0 .../basemigrate/basemigrate_test.go | 2 +- .../basemigrate/changeset.go | 0 .../{pkg => internal}/basemigrate/helper.go | 0 .../{pkg => internal}/basemigrate/migrate.go | 0 .../{pkg => internal}/basemigrate/parse.go | 0 .../{pkg => internal}/basemigrate/reset.go | 0 .../basemigrate/testdata/fail-duplicate.sql | 0 .../internal/basemigrate/testdata/success.sql | 43 +++++ 12 files changed, 218 insertions(+), 4 deletions(-) rename src/app/webapi/{pkg/basemigrate => cmd/cliapp}/testdata/success.sql (100%) rename src/app/webapi/{pkg => internal}/basemigrate/basemigrate.go (100%) rename src/app/webapi/{pkg => internal}/basemigrate/basemigrate_test.go (98%) rename src/app/webapi/{pkg => internal}/basemigrate/changeset.go (100%) rename src/app/webapi/{pkg => internal}/basemigrate/helper.go (100%) rename src/app/webapi/{pkg => internal}/basemigrate/migrate.go (100%) rename src/app/webapi/{pkg => internal}/basemigrate/parse.go (100%) rename src/app/webapi/{pkg => internal}/basemigrate/reset.go (100%) rename src/app/webapi/{pkg => internal}/basemigrate/testdata/fail-duplicate.sql (100%) create mode 100644 src/app/webapi/internal/basemigrate/testdata/success.sql diff --git a/src/app/webapi/cmd/cliapp/cliapp.go b/src/app/webapi/cmd/cliapp/cliapp.go index f24d4af..609c9bb 100644 --- a/src/app/webapi/cmd/cliapp/cliapp.go +++ b/src/app/webapi/cmd/cliapp/cliapp.go @@ -3,10 +3,9 @@ package main import ( "encoding/base64" "fmt" - "log" "os" - "app/webapi/pkg/basemigrate" + "app/webapi/internal/basemigrate" "app/webapi/pkg/securegen" kingpin "gopkg.in/alecthomas/kingpin.v2" @@ -40,7 +39,8 @@ func main() { case cGenerate.FullCommand(): b, err := securegen.Bytes(32) if err != nil { - log.Fatal(err) + fmt.Println(err) + os.Exit(1) } enc := base64.StdEncoding.EncodeToString(b) diff --git a/src/app/webapi/cmd/cliapp/cliapp_test.go b/src/app/webapi/cmd/cliapp/cliapp_test.go index 774d466..1626159 100644 --- a/src/app/webapi/cmd/cliapp/cliapp_test.go +++ b/src/app/webapi/cmd/cliapp/cliapp_test.go @@ -6,6 +6,8 @@ import ( "os" "testing" + "app/webapi/internal/testutil" + "github.com/stretchr/testify/assert" ) @@ -37,3 +39,172 @@ func TestGenerate(t *testing.T) { // Ensure the length is 32 bytes. assert.Equal(t, 32, len(s)) } + +func setEnv() { + os.Setenv("DB_HOSTNAME", "127.0.0.1") + os.Setenv("DB_PORT", "3306") + os.Setenv("DB_USERNAME", "root") + os.Setenv("DB_PASSWORD", "") + os.Setenv("DB_DATABASE", "webapitest") + os.Setenv("DB_PARAMETER", "parseTime=true&allowNativePasswords=true") +} + +func unsetEnv() { + os.Unsetenv("DB_HOSTNAME") + os.Unsetenv("DB_PORT") + os.Unsetenv("DB_USERNAME") + os.Unsetenv("DB_PASSWORD") + os.Unsetenv("DB_DATABASE") + os.Unsetenv("DB_PARAMETER") +} +func TestMigrationAll(t *testing.T) { + setEnv() + defer unsetEnv() + + testutil.ResetDatabase() + db := testutil.ConnectDatabase(true) + + // Set the arguments. + os.Args = make([]string, 4) + os.Args[0] = "cliapp" + os.Args[1] = "migrate" + os.Args[2] = "all" + os.Args[3] = "testdata/success.sql" + + // Redirect stdout. + backupd := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + + // Call the application. + main() + + // Get the output. + w.Close() + out, err := ioutil.ReadAll(r) + assert.Nil(t, err) + os.Stdout = backupd + + assert.Contains(t, string(out), "Changeset applied") + + // Count the records. + rows := 0 + err = db.Get(&rows, `SELECT count(*) from databasechangelog`) + assert.Nil(t, err) + assert.Equal(t, 3, rows) +} + +func TestMigrationReset(t *testing.T) { + TestMigrationAll(t) + + setEnv() + defer unsetEnv() + + db := testutil.ConnectDatabase(true) + + // Set the arguments. + os.Args = make([]string, 4) + os.Args[0] = "cliapp" + os.Args[1] = "migrate" + os.Args[2] = "reset" + os.Args[3] = "testdata/success.sql" + + // Redirect stdout. + backupd := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + + // Call the application. + main() + + // Get the output. + w.Close() + out, err := ioutil.ReadAll(r) + assert.Nil(t, err) + os.Stdout = backupd + + assert.Contains(t, string(out), "Rollback applied") + + // Count the records. + rows := 0 + err = db.Get(&rows, `SELECT count(*) from databasechangelog`) + assert.Nil(t, err) + assert.Equal(t, 0, rows) +} + +func TestMigrationUp(t *testing.T) { + setEnv() + defer unsetEnv() + + testutil.ResetDatabase() + db := testutil.ConnectDatabase(true) + + // Set the arguments. + os.Args = make([]string, 5) + os.Args[0] = "cliapp" + os.Args[1] = "migrate" + os.Args[2] = "up" + os.Args[3] = "2" + os.Args[4] = "testdata/success.sql" + + // Redirect stdout. + backupd := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + + // Call the application. + main() + + // Get the output. + w.Close() + out, err := ioutil.ReadAll(r) + assert.Nil(t, err) + os.Stdout = backupd + + assert.Contains(t, string(out), "Changeset applied") + + // Count the records. + rows := 0 + err = db.Get(&rows, `SELECT count(*) from databasechangelog`) + assert.Nil(t, err) + assert.Equal(t, 2, rows) +} + +func TestMigrationDown(t *testing.T) { + TestMigrationUp(t) + + setEnv() + defer unsetEnv() + + db := testutil.ConnectDatabase(true) + + // Set the arguments. + os.Args = make([]string, 5) + os.Args[0] = "cliapp" + os.Args[1] = "migrate" + os.Args[2] = "down" + os.Args[3] = "1" + os.Args[4] = "testdata/success.sql" + + // Redirect stdout. + backupd := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + + // Call the application. + main() + + // Get the output. + w.Close() + out, err := ioutil.ReadAll(r) + assert.Nil(t, err) + os.Stdout = backupd + + assert.Contains(t, string(out), "Rollback applied") + + // Count the records. + rows := 0 + err = db.Get(&rows, `SELECT count(*) from databasechangelog`) + assert.Nil(t, err) + assert.Equal(t, 1, rows) +} diff --git a/src/app/webapi/pkg/basemigrate/testdata/success.sql b/src/app/webapi/cmd/cliapp/testdata/success.sql similarity index 100% rename from src/app/webapi/pkg/basemigrate/testdata/success.sql rename to src/app/webapi/cmd/cliapp/testdata/success.sql diff --git a/src/app/webapi/pkg/basemigrate/basemigrate.go b/src/app/webapi/internal/basemigrate/basemigrate.go similarity index 100% rename from src/app/webapi/pkg/basemigrate/basemigrate.go rename to src/app/webapi/internal/basemigrate/basemigrate.go diff --git a/src/app/webapi/pkg/basemigrate/basemigrate_test.go b/src/app/webapi/internal/basemigrate/basemigrate_test.go similarity index 98% rename from src/app/webapi/pkg/basemigrate/basemigrate_test.go rename to src/app/webapi/internal/basemigrate/basemigrate_test.go index eaaf18c..5c04615 100644 --- a/src/app/webapi/pkg/basemigrate/basemigrate_test.go +++ b/src/app/webapi/internal/basemigrate/basemigrate_test.go @@ -4,8 +4,8 @@ import ( "os" "testing" + "app/webapi/internal/basemigrate" "app/webapi/internal/testutil" - "app/webapi/pkg/basemigrate" "github.com/stretchr/testify/assert" ) diff --git a/src/app/webapi/pkg/basemigrate/changeset.go b/src/app/webapi/internal/basemigrate/changeset.go similarity index 100% rename from src/app/webapi/pkg/basemigrate/changeset.go rename to src/app/webapi/internal/basemigrate/changeset.go diff --git a/src/app/webapi/pkg/basemigrate/helper.go b/src/app/webapi/internal/basemigrate/helper.go similarity index 100% rename from src/app/webapi/pkg/basemigrate/helper.go rename to src/app/webapi/internal/basemigrate/helper.go diff --git a/src/app/webapi/pkg/basemigrate/migrate.go b/src/app/webapi/internal/basemigrate/migrate.go similarity index 100% rename from src/app/webapi/pkg/basemigrate/migrate.go rename to src/app/webapi/internal/basemigrate/migrate.go diff --git a/src/app/webapi/pkg/basemigrate/parse.go b/src/app/webapi/internal/basemigrate/parse.go similarity index 100% rename from src/app/webapi/pkg/basemigrate/parse.go rename to src/app/webapi/internal/basemigrate/parse.go diff --git a/src/app/webapi/pkg/basemigrate/reset.go b/src/app/webapi/internal/basemigrate/reset.go similarity index 100% rename from src/app/webapi/pkg/basemigrate/reset.go rename to src/app/webapi/internal/basemigrate/reset.go diff --git a/src/app/webapi/pkg/basemigrate/testdata/fail-duplicate.sql b/src/app/webapi/internal/basemigrate/testdata/fail-duplicate.sql similarity index 100% rename from src/app/webapi/pkg/basemigrate/testdata/fail-duplicate.sql rename to src/app/webapi/internal/basemigrate/testdata/fail-duplicate.sql diff --git a/src/app/webapi/internal/basemigrate/testdata/success.sql b/src/app/webapi/internal/basemigrate/testdata/success.sql new file mode 100644 index 0000000..6526d0a --- /dev/null +++ b/src/app/webapi/internal/basemigrate/testdata/success.sql @@ -0,0 +1,43 @@ +--changeset josephspurrier:1 +SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO'; +CREATE TABLE user_status ( + id TINYINT(1) UNSIGNED NOT NULL AUTO_INCREMENT, + + status VARCHAR(25) NOT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted TINYINT(1) UNSIGNED NOT NULL DEFAULT 0, + + PRIMARY KEY (id) +); +--rollback DROP TABLE user_status; + +--changeset josephspurrier:2 +INSERT INTO `user_status` (`id`, `status`, `created_at`, `updated_at`, `deleted`) VALUES +(1, 'active', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0), +(2, 'inactive', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0); +--rollback TRUNCATE TABLE user_status; + +--changeset josephspurrier:3 +SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO'; +CREATE TABLE user ( + id VARCHAR(36) NOT NULL, + + first_name VARCHAR(50) NOT NULL, + last_name VARCHAR(50) NOT NULL, + email VARCHAR(100) NOT NULL, + password CHAR(60) NOT NULL, + + status_id TINYINT(1) UNSIGNED NOT NULL DEFAULT 1, + + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP DEFAULT 0, + + UNIQUE KEY (email), + CONSTRAINT `f_user_status` FOREIGN KEY (`status_id`) REFERENCES `user_status` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, + + PRIMARY KEY (id) +); +--rollback DROP TABLE user; \ No newline at end of file From 6511c540a7adee2379d52982e31b0a33284a70d6 Mon Sep 17 00:00:00 2001 From: Joseph Spurrier Date: Mon, 16 Jul 2018 22:31:45 -0400 Subject: [PATCH 07/14] Add an include function to migrations --- README.md | 60 ++++++++++++--- migration/{mysql.sql => mysql-v0.sql} | 38 +++------ src/app/webapi/cmd/cliapp/cliapp_test.go | 4 - .../internal/basemigrate/basemigrate.go | 41 +--------- .../internal/basemigrate/basemigrate_test.go | 77 +++++++++++-------- src/app/webapi/internal/basemigrate/helper.go | 13 ---- .../webapi/internal/basemigrate/migrate.go | 5 +- src/app/webapi/internal/basemigrate/parse.go | 68 +++++++++++++++- src/app/webapi/internal/basemigrate/reset.go | 4 +- .../internal/basemigrate/testdata/child1.sql | 20 +++++ .../internal/basemigrate/testdata/child2.sql | 22 ++++++ .../internal/basemigrate/testdata/parent.sql | 2 + src/app/webapi/internal/testutil/database.go | 52 +++++++------ 13 files changed, 251 insertions(+), 155 deletions(-) rename migration/{mysql.sql => mysql-v0.sql} (52%) create mode 100644 src/app/webapi/internal/basemigrate/testdata/child1.sql create mode 100644 src/app/webapi/internal/basemigrate/testdata/child2.sql create mode 100644 src/app/webapi/internal/basemigrate/testdata/parent.sql diff --git a/README.md b/README.md index 012a26c..5d694c7 100644 --- a/README.md +++ b/README.md @@ -18,20 +18,62 @@ You must use Go 1.7 or newer because it uses the http context. ## Quick Start with MySQL -Use one of the following commands to start a MySQL container with Docker: +Use the following commands to start a MySQL container with Docker: -- Start MySQL without a password: `docker run -d --name=mysql57 -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=yes mysql:5.7` -- Start MySQL with a password: `docker run -d --name=mysql57 -p 3306:3306 -e MYSQL_ROOT_PASSWORD=somepassword mysql:5.7` +```bash +# Start MySQL without a password. +docker run -d --name=mysql57 -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=yes mysql:5.7 +# or start MySQL with a password. +docker run -d --name=mysql57 -p 3306:3306 -e MYSQL_ROOT_PASSWORD=somepassword mysql:5.7 + +# Create the database via docker exec. +docker exec mysql57 sh -c 'exec mysql -uroot -e "CREATE DATABASE IF NOT EXISTS webapi DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;"' +# Or create the database manually. +CREATE DATABASE webapi DEFAULT CHARSET = utf8 COLLATE = utf8_unicode_ci; + +# CD to the CLI tool. +cd src/app/webapi/cmd/cliapp + +# Build the CLI tool. +go build + +# Apply the database migrations without a password. +DB_USERNAME=root DB_HOSTNAME=127.0.0.1 DB_PORT=3306 DB_DATABASE=webapi ./cliapp migrate all ../../../../../migration/mysql-v0.sql +# or apply the database migrations with a password. +DB_USERNAME=root DB_PASSWORD=somepassword DB_HOSTNAME=127.0.0.1 DB_PORT=3306 DB_DATABASE=webapi ./cliapp migrate all ../../../../../migration/mysql-v0.sql +``` + +Using the database connection information above, follow the steps to set up the `config.json` file: -Start MySQL and import `migration/mysql.sql` to create the database and tables. +```bash +# Copy the config.json from the root of the project to the CLI app folder. +cp config.json src/app/webapi/cmd/webapi/config.json + +# Edit the `Database` section so the connection information matches your MySQL instance. +# The database password is read from the `config.json` file, but is overwritten by the environment variable, `DB_PASSWORD`, if it is set. + +# Generate a base64 encoded secret. +./cliapp generate + +# Add the encoded secret above to the `JWT.Secret` section of the config. +``` -Copy `config.json` to `src/app/webapi/cmd/webapi/config.json` and edit the **Database** section so the connection information matches your MySQL instance. Also add a base64 encoded `JWT.Secret` to the config. You can generate it using the command line app in the repo - run these commands: -- `cd src/app/webapi/cmd/cliapp` -- `go run cliapp.go generate` +Now you can start the API. -The database password is read from the `config.json` first, but is overwritten by the environment variable, `DB_PASSWORD`, if it is set. +```bash +# CD to the webapi app folder. +cd src/app/webapi/cmd/webapi + +# Build the app. +go build + +# Run the app. +./webapi + +# Open your browser to this URL to see the **welcome** message and status **OK**: http://localhost/v1 +``` -Build and run from the root directory. Open your REST client to: http://localhost/v1. You should see the **welcome** message and status **OK**. +To interact with the API, open your favorite REST client. You'll need to authenticate with at http://localhost/v1/auth before you can use any of the user endpoints. Once you have a token, add it to the request header with a name of `Authorization` and with a value of `Bearer {TOKEN HERE}`. To create a user, send a POST request to http://localhost/v1/user with the following fields: first_name, last_name, email, and password. diff --git a/migration/mysql.sql b/migration/mysql-v0.sql similarity index 52% rename from migration/mysql.sql rename to migration/mysql-v0.sql index 7a2cba0..6526d0a 100644 --- a/migration/mysql.sql +++ b/migration/mysql-v0.sql @@ -1,27 +1,5 @@ -/* ***************************************************************************** -// Setup the preferences -// ****************************************************************************/ -SET NAMES utf8 COLLATE 'utf8_unicode_ci'; -SET foreign_key_checks = 1; -SET time_zone = '+00:00'; +--changeset josephspurrier:1 SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO'; -SET CHARACTER SET utf8; - -/* ***************************************************************************** -// Remove old database -// ****************************************************************************/ -DROP DATABASE IF EXISTS webapi; - -/* ***************************************************************************** -// Create new database -// ****************************************************************************/ -CREATE DATABASE webapi DEFAULT CHARSET = utf8 COLLATE = utf8_unicode_ci; -USE webapi; - -/* ***************************************************************************** -// Create the tables -// ****************************************************************************/ - CREATE TABLE user_status ( id TINYINT(1) UNSIGNED NOT NULL AUTO_INCREMENT, @@ -33,7 +11,16 @@ CREATE TABLE user_status ( PRIMARY KEY (id) ); +--rollback DROP TABLE user_status; +--changeset josephspurrier:2 +INSERT INTO `user_status` (`id`, `status`, `created_at`, `updated_at`, `deleted`) VALUES +(1, 'active', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0), +(2, 'inactive', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0); +--rollback TRUNCATE TABLE user_status; + +--changeset josephspurrier:3 +SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO'; CREATE TABLE user ( id VARCHAR(36) NOT NULL, @@ -53,7 +40,4 @@ CREATE TABLE user ( PRIMARY KEY (id) ); - -INSERT INTO `user_status` (`id`, `status`, `created_at`, `updated_at`, `deleted`) VALUES -(1, 'active', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0), -(2, 'inactive', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0); \ No newline at end of file +--rollback DROP TABLE user; \ No newline at end of file diff --git a/src/app/webapi/cmd/cliapp/cliapp_test.go b/src/app/webapi/cmd/cliapp/cliapp_test.go index 1626159..72a6de7 100644 --- a/src/app/webapi/cmd/cliapp/cliapp_test.go +++ b/src/app/webapi/cmd/cliapp/cliapp_test.go @@ -59,7 +59,6 @@ func unsetEnv() { } func TestMigrationAll(t *testing.T) { setEnv() - defer unsetEnv() testutil.ResetDatabase() db := testutil.ConnectDatabase(true) @@ -98,7 +97,6 @@ func TestMigrationReset(t *testing.T) { TestMigrationAll(t) setEnv() - defer unsetEnv() db := testutil.ConnectDatabase(true) @@ -134,7 +132,6 @@ func TestMigrationReset(t *testing.T) { func TestMigrationUp(t *testing.T) { setEnv() - defer unsetEnv() testutil.ResetDatabase() db := testutil.ConnectDatabase(true) @@ -174,7 +171,6 @@ func TestMigrationDown(t *testing.T) { TestMigrationUp(t) setEnv() - defer unsetEnv() db := testutil.ConnectDatabase(true) diff --git a/src/app/webapi/internal/basemigrate/basemigrate.go b/src/app/webapi/internal/basemigrate/basemigrate.go index b1123d7..d0421db 100644 --- a/src/app/webapi/internal/basemigrate/basemigrate.go +++ b/src/app/webapi/internal/basemigrate/basemigrate.go @@ -2,8 +2,6 @@ package basemigrate import ( "errors" - "fmt" - "os" ) const ( @@ -24,6 +22,7 @@ const ( appVersion = "1.0" elementChangeset = "--changeset " elementRollback = "--rollback " + elementInclude = "--include " ) var ( @@ -32,41 +31,3 @@ var ( // ErrInvalidFormat is when a changeset is not found. ErrInvalidFormat = errors.New("invalid changeset format") ) - -// ParseFileArray will parse a file into changesets. -func ParseFileArray(filename string) ([]*Changeset, error) { - f, err := os.Open(filename) - if err != nil { - return nil, err - } - defer f.Close() - - return parseToOrderedArray(f, filename) -} - -// ParseFileMap will parse a file into a map. -func ParseFileMap(filename string) (map[string]Changeset, error) { - f, err := os.Open(filename) - if err != nil { - return nil, err - } - defer f.Close() - - arr, err := parseToOrderedArray(f, filename) - if err != nil { - return nil, err - } - - m := make(map[string]Changeset) - - for _, cs := range arr { - id := fmt.Sprintf("%v:%v", cs.author, cs.id) - if _, found := m[id]; found { - return nil, errors.New("Duplicate entry found: " + id) - } - - m[id] = *cs - } - - return m, nil -} diff --git a/src/app/webapi/internal/basemigrate/basemigrate_test.go b/src/app/webapi/internal/basemigrate/basemigrate_test.go index 5c04615..e1a08db 100644 --- a/src/app/webapi/internal/basemigrate/basemigrate_test.go +++ b/src/app/webapi/internal/basemigrate/basemigrate_test.go @@ -1,7 +1,6 @@ package basemigrate_test import ( - "os" "testing" "app/webapi/internal/basemigrate" @@ -10,27 +9,7 @@ import ( "github.com/stretchr/testify/assert" ) -func setEnv() { - os.Setenv("DB_HOSTNAME", "127.0.0.1") - os.Setenv("DB_PORT", "3306") - os.Setenv("DB_USERNAME", "root") - os.Setenv("DB_PASSWORD", "") - os.Setenv("DB_DATABASE", "webapitest") - os.Setenv("DB_PARAMETER", "parseTime=true&allowNativePasswords=true") -} - -func unsetEnv() { - os.Unsetenv("DB_HOSTNAME") - os.Unsetenv("DB_PORT") - os.Unsetenv("DB_USERNAME") - os.Unsetenv("DB_PASSWORD") - os.Unsetenv("DB_DATABASE") - os.Unsetenv("DB_PARAMETER") -} func TestMigration(t *testing.T) { - setEnv() - defer unsetEnv() - testutil.ResetDatabase() db := testutil.ConnectDatabase(true) @@ -81,9 +60,6 @@ func TestMigration(t *testing.T) { } func TestMigrationFailDuplicate(t *testing.T) { - setEnv() - defer unsetEnv() - testutil.ResetDatabase() db := testutil.ConnectDatabase(true) @@ -94,17 +70,56 @@ func TestMigrationFailDuplicate(t *testing.T) { err = db.Get(&rows, `SELECT count(*) from databasechangelog`) assert.Nil(t, err) assert.Equal(t, 2, rows) + + testutil.ResetDatabase() } -func TestParse(t *testing.T) { - setEnv() - defer unsetEnv() +func TestInclude(t *testing.T) { + testutil.ResetDatabase() + db := testutil.ConnectDatabase(true) + + // Run migration. + err := basemigrate.Migrate("testdata/parent.sql", 0, false) + assert.Nil(t, err) + + // Count the records. + rows := 0 + err = db.Get(&rows, `SELECT count(*) from databasechangelog`) + assert.Nil(t, err) + assert.Equal(t, 3, rows) + + // Run migration again. + err = basemigrate.Migrate("testdata/parent.sql", 0, false) + assert.Nil(t, err) + + // Remove all migrations. + err = basemigrate.Reset("testdata/parent.sql", 0, false) + assert.Nil(t, err) + + rows = 0 + err = db.Get(&rows, `SELECT count(*) from databasechangelog`) + assert.Nil(t, err) + assert.Equal(t, 0, rows) + + // Remove all migrations again. + err = basemigrate.Reset("testdata/parent.sql", 0, false) + assert.Nil(t, err) + + // Run 2 migrations. + err = basemigrate.Migrate("testdata/parent.sql", 2, false) + assert.Nil(t, err) - arr, err := basemigrate.ParseFileArray("testdata/success.sql") + rows = 0 + err = db.Get(&rows, `SELECT count(*) from databasechangelog`) assert.Nil(t, err) - assert.Equal(t, 3, len(arr)) + assert.Equal(t, 2, rows) - m, err := basemigrate.ParseFileMap("testdata/success.sql") + // Remove 1 migration. + err = basemigrate.Reset("testdata/parent.sql", 1, false) assert.Nil(t, err) - assert.Equal(t, 3, len(m)) + + rows = 0 + err = db.Get(&rows, `SELECT count(*) from databasechangelog`) + assert.Nil(t, err) + assert.Equal(t, 1, rows) } diff --git a/src/app/webapi/internal/basemigrate/helper.go b/src/app/webapi/internal/basemigrate/helper.go index ae64e8f..aa96261 100644 --- a/src/app/webapi/internal/basemigrate/helper.go +++ b/src/app/webapi/internal/basemigrate/helper.go @@ -32,16 +32,3 @@ func md5sum(s string) string { _, _ = io.Copy(h, r) return fmt.Sprintf("%x", h.Sum(nil)) } - -// Debug will return the SQL file. -func Debug(arr []*Changeset) { - for _, cs := range arr { - fmt.Printf("%v%v:%v\n", elementChangeset, cs.author, cs.id) - fmt.Println(cs.Changes()) - fmt.Printf("%v%v\n", elementRollback, cs.Rollbacks()) - fmt.Println("--md5", cs.Checksum()) - break - } - - fmt.Println("Total:", len(arr)) -} diff --git a/src/app/webapi/internal/basemigrate/migrate.go b/src/app/webapi/internal/basemigrate/migrate.go index adffbc6..b93f6dc 100644 --- a/src/app/webapi/internal/basemigrate/migrate.go +++ b/src/app/webapi/internal/basemigrate/migrate.go @@ -18,7 +18,7 @@ func Migrate(filename string, max int, verbose bool) (err error) { _, err = db.Exec(sqlChangelog) // Get the changesets. - arr, err := ParseFileArray(filename) + arr, err := parseFileToArray(filename) if err != nil { return err } @@ -35,7 +35,8 @@ func Migrate(filename string, max int, verbose bool) (err error) { err = db.Get(&checksum, `SELECT md5sum FROM databasechangelog WHERE id = ? - AND author = ?`, cs.id, cs.author) + AND author = ? + AND filename = ?`, cs.id, cs.author, cs.filename) if err == nil { // Determine if the checksums match. if checksum != newChecksum { diff --git a/src/app/webapi/internal/basemigrate/parse.go b/src/app/webapi/internal/basemigrate/parse.go index 16803f6..835c3c8 100644 --- a/src/app/webapi/internal/basemigrate/parse.go +++ b/src/app/webapi/internal/basemigrate/parse.go @@ -2,13 +2,28 @@ package basemigrate import ( "bufio" + "errors" + "fmt" "io" + "os" "path" + "path/filepath" "strings" ) -// parseToOrderedArray will split the SQL migration into an ordered array. -func parseToOrderedArray(r io.Reader, filename string) ([]*Changeset, error) { +// parseFileToArray will parse a file into changesets. +func parseFileToArray(filename string) ([]*Changeset, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + return parseToArray(f, filename) +} + +// parseToArray will split the SQL migration into an ordered array. +func parseToArray(r io.Reader, filename string) ([]*Changeset, error) { scanner := bufio.NewScanner(r) scanner.Split(bufio.ScanLines) @@ -24,11 +39,24 @@ func parseToOrderedArray(r io.Reader, filename string) ([]*Changeset, error) { continue } + // Determine if the line is an `include`. + if strings.HasPrefix(line, elementInclude) { + // Load the file and add to the array. + fp := strings.TrimPrefix(line, elementInclude) + rfp := filepath.Join(filepath.Dir(filename), fp) + cs, err := parseFileToArray(rfp) + if err != nil { + return nil, err + } + arr = append(arr, cs...) + continue + } + // Start recording the changeset. if strings.HasPrefix(line, elementChangeset) { // Create a new changeset. cs := new(Changeset) - cs.ParseHeader(strings.TrimLeft(line, elementChangeset)) + cs.ParseHeader(strings.TrimPrefix(line, elementChangeset)) cs.SetFileInfo(path.Base(filename), "sql", appVersion) arr = append(arr, cs) continue @@ -42,7 +70,7 @@ func parseToOrderedArray(r io.Reader, filename string) ([]*Changeset, error) { // Determine if the line is a rollback. if strings.HasPrefix(line, elementRollback) { cs := arr[len(arr)-1] - cs.AddRollback(strings.TrimLeft(line, elementRollback)) + cs.AddRollback(strings.TrimPrefix(line, elementRollback)) continue } @@ -58,3 +86,35 @@ func parseToOrderedArray(r io.Reader, filename string) ([]*Changeset, error) { return arr, nil } + +// parseFileToMap will parse a file into a map. +func parseFileToMap(filename string) (map[string]Changeset, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + return parseToMap(f, filename) +} + +// parseToMap will parse a reader to a map. +func parseToMap(r io.Reader, filename string) (map[string]Changeset, error) { + arr, err := parseToArray(r, filename) + if err != nil { + return nil, err + } + + m := make(map[string]Changeset) + + for _, cs := range arr { + id := fmt.Sprintf("%v:%v:%v", cs.author, cs.id, cs.filename) + if _, found := m[id]; found { + return nil, errors.New("Duplicate entry found: " + id) + } + + m[id] = *cs + } + + return m, nil +} diff --git a/src/app/webapi/internal/basemigrate/reset.go b/src/app/webapi/internal/basemigrate/reset.go index 0a6b5aa..1f44761 100644 --- a/src/app/webapi/internal/basemigrate/reset.go +++ b/src/app/webapi/internal/basemigrate/reset.go @@ -22,7 +22,7 @@ func Reset(filename string, max int, verbose bool) (err error) { } // Get the changesets in a map. - m, err := ParseFileMap(filename) + m, err := parseFileToMap(filename) if err != nil { return err } @@ -48,7 +48,7 @@ func Reset(filename string, max int, verbose bool) (err error) { // Loop through each changeset. for _, r := range results { - id := fmt.Sprintf("%v:%v", r.Author, r.ID) + id := fmt.Sprintf("%v:%v:%v", r.Author, r.ID, r.Filename) cs, ok := m[id] if !ok { diff --git a/src/app/webapi/internal/basemigrate/testdata/child1.sql b/src/app/webapi/internal/basemigrate/testdata/child1.sql new file mode 100644 index 0000000..c509931 --- /dev/null +++ b/src/app/webapi/internal/basemigrate/testdata/child1.sql @@ -0,0 +1,20 @@ +--changeset josephspurrier:1 +SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO'; +CREATE TABLE user_status ( + id TINYINT(1) UNSIGNED NOT NULL AUTO_INCREMENT, + + status VARCHAR(25) NOT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted TINYINT(1) UNSIGNED NOT NULL DEFAULT 0, + + PRIMARY KEY (id) +); +--rollback DROP TABLE user_status; + +--changeset josephspurrier:2 +INSERT INTO `user_status` (`id`, `status`, `created_at`, `updated_at`, `deleted`) VALUES +(1, 'active', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0), +(2, 'inactive', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0); +--rollback TRUNCATE TABLE user_status; \ No newline at end of file diff --git a/src/app/webapi/internal/basemigrate/testdata/child2.sql b/src/app/webapi/internal/basemigrate/testdata/child2.sql new file mode 100644 index 0000000..3fc8bf0 --- /dev/null +++ b/src/app/webapi/internal/basemigrate/testdata/child2.sql @@ -0,0 +1,22 @@ +--changeset josephspurrier:3 +SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO'; +CREATE TABLE user ( + id VARCHAR(36) NOT NULL, + + first_name VARCHAR(50) NOT NULL, + last_name VARCHAR(50) NOT NULL, + email VARCHAR(100) NOT NULL, + password CHAR(60) NOT NULL, + + status_id TINYINT(1) UNSIGNED NOT NULL DEFAULT 1, + + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP DEFAULT 0, + + UNIQUE KEY (email), + CONSTRAINT `f_user_status` FOREIGN KEY (`status_id`) REFERENCES `user_status` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, + + PRIMARY KEY (id) +); +--rollback DROP TABLE user; \ No newline at end of file diff --git a/src/app/webapi/internal/basemigrate/testdata/parent.sql b/src/app/webapi/internal/basemigrate/testdata/parent.sql new file mode 100644 index 0000000..5502cb4 --- /dev/null +++ b/src/app/webapi/internal/basemigrate/testdata/parent.sql @@ -0,0 +1,2 @@ +--include child1.sql +--include child2.sql \ No newline at end of file diff --git a/src/app/webapi/internal/testutil/database.go b/src/app/webapi/internal/testutil/database.go index e28eb92..b3516a8 100644 --- a/src/app/webapi/internal/testutil/database.go +++ b/src/app/webapi/internal/testutil/database.go @@ -1,24 +1,37 @@ package testutil import ( + "fmt" "io/ioutil" "log" "os" "strings" "testing" + "app/webapi/internal/basemigrate" + "app/webapi/pkg/env" + "app/webapi/pkg/database" ) +func setEnv() { + os.Setenv("DB_HOSTNAME", "127.0.0.1") + os.Setenv("DB_PORT", "3306") + os.Setenv("DB_USERNAME", "root") + os.Setenv("DB_PASSWORD", "") + os.Setenv("DB_DATABASE", "webapitest") + os.Setenv("DB_PARAMETER", "parseTime=true&allowNativePasswords=true") +} + // ConnectDatabase returns a test database connection. func ConnectDatabase(dbSpecificDB bool) *database.DBW { dbc := new(database.Connection) - dbc.Hostname = "127.0.0.1" - dbc.Port = 3306 - dbc.Username = "root" - dbc.Password = "" - dbc.Database = "webapitest" - dbc.Parameter = "parseTime=true&allowNativePasswords=true" + setEnv() + + err := env.Unmarshal(dbc) + if err != nil { + log.Println("DB ENV Error:", err) + } connection, err := dbc.Connect(dbSpecificDB) if err != nil { @@ -33,30 +46,23 @@ func ConnectDatabase(dbSpecificDB bool) *database.DBW { // ResetDatabase will drop and create the test database. func ResetDatabase() { db := ConnectDatabase(false) - db.Exec(`DROP DATABASE IF EXISTS webapitest`) - db.Exec(`CREATE DATABASE webapitest DEFAULT CHARSET = utf8 COLLATE = utf8_unicode_ci`) + _, err := db.Exec(`DROP DATABASE IF EXISTS webapitest`) + if err != nil { + fmt.Println(err) + } + _, err = db.Exec(`CREATE DATABASE webapitest DEFAULT CHARSET = utf8 COLLATE = utf8_unicode_ci`) + if err != nil { + fmt.Println(err) + } } // LoadDatabase will set up the DB for the tests. func LoadDatabase(t *testing.T) { ResetDatabase() - db := ConnectDatabase(true) - b, err := ioutil.ReadFile("../../../../../migration/tables-only.sql") + err := basemigrate.Migrate("../../../../../migration/mysql-v0.sql", 0, false) if err != nil { - t.Error(err) - } - - // Split each statement. - stmts := strings.Split(string(b), ";") - for i, s := range stmts { - if i == len(stmts)-1 { - break - } - _, err = db.Exec(s) - if err != nil { - t.Error(err) - } + log.Println("DB Error:", err) } } From 0c03e9b22756994263c2e751c11850a578df591f Mon Sep 17 00:00:00 2001 From: Joseph Spurrier Date: Tue, 17 Jul 2018 05:36:23 -0400 Subject: [PATCH 08/14] Ensure all DB tests can be run in parallel --- src/app/webapi/cmd/cliapp/cliapp.go | 9 +- src/app/webapi/cmd/cliapp/cliapp_test.go | 70 ++++++----- src/app/webapi/cmd/hooks/main.go | 8 +- src/app/webapi/component/auth/index_test.go | 11 +- src/app/webapi/component/core_mock.go | 10 +- src/app/webapi/component/root/index_test.go | 6 +- src/app/webapi/component/user/create_test.go | 24 ++-- .../webapi/component/user/destroy_all_test.go | 20 ++-- src/app/webapi/component/user/destroy_test.go | 6 +- src/app/webapi/component/user/index_test.go | 12 +- src/app/webapi/component/user/show_test.go | 12 +- src/app/webapi/component/user/update_test.go | 12 +- .../internal/basemigrate/basemigrate_test.go | 41 +++---- src/app/webapi/internal/basemigrate/helper.go | 4 +- .../webapi/internal/basemigrate/migrate.go | 4 +- src/app/webapi/internal/basemigrate/reset.go | 4 +- src/app/webapi/internal/testutil/database.go | 111 ++++++++++-------- src/app/webapi/pkg/env/env.go | 4 +- src/app/webapi/pkg/env/env_test.go | 38 ++++-- 19 files changed, 243 insertions(+), 163 deletions(-) diff --git a/src/app/webapi/cmd/cliapp/cliapp.go b/src/app/webapi/cmd/cliapp/cliapp.go index 609c9bb..473a4c5 100644 --- a/src/app/webapi/cmd/cliapp/cliapp.go +++ b/src/app/webapi/cmd/cliapp/cliapp.go @@ -16,6 +16,7 @@ var ( cGenerate = app.Command("generate", "Generate 256 bit (32 byte) base64 encoded JWT.") cDB = app.Command("migrate", "Perform actions on the database.") + cDBPrefix = cDB.Flag("envprefix", "Prefix for environment variables.").String() cDBAll = cDB.Command("all", "Apply all changesets to the database.") cDBAllFile = cDBAll.Arg("file", "Filename of the migration file [string].").Required().String() @@ -46,26 +47,26 @@ func main() { enc := base64.StdEncoding.EncodeToString(b) fmt.Println(enc) case cDBAll.FullCommand(): - err := basemigrate.Migrate(*cDBAllFile, 0, true) + err := basemigrate.Migrate(*cDBAllFile, *cDBPrefix, 0, true) if err != nil { fmt.Println(err) os.Exit(1) } case cDBUp.FullCommand(): - err := basemigrate.Migrate(*cDBUpFile, *cDBUpCount, true) + err := basemigrate.Migrate(*cDBUpFile, *cDBPrefix, *cDBUpCount, true) if err != nil { fmt.Println(err) os.Exit(1) } case cDBReset.FullCommand(): - err := basemigrate.Reset(*cDBResetFile, 0, true) + err := basemigrate.Reset(*cDBResetFile, *cDBPrefix, 0, true) if err != nil { fmt.Println(err) os.Exit(1) } case cDBDown.FullCommand(): - err := basemigrate.Reset(*cDBDownFile, *cDBDownCount, true) + err := basemigrate.Reset(*cDBDownFile, *cDBPrefix, *cDBDownCount, true) if err != nil { fmt.Println(err) os.Exit(1) diff --git a/src/app/webapi/cmd/cliapp/cliapp_test.go b/src/app/webapi/cmd/cliapp/cliapp_test.go index 72a6de7..ef31b22 100644 --- a/src/app/webapi/cmd/cliapp/cliapp_test.go +++ b/src/app/webapi/cmd/cliapp/cliapp_test.go @@ -7,6 +7,7 @@ import ( "testing" "app/webapi/internal/testutil" + "app/webapi/pkg/database" "github.com/stretchr/testify/assert" ) @@ -40,35 +41,22 @@ func TestGenerate(t *testing.T) { assert.Equal(t, 32, len(s)) } -func setEnv() { - os.Setenv("DB_HOSTNAME", "127.0.0.1") - os.Setenv("DB_PORT", "3306") - os.Setenv("DB_USERNAME", "root") - os.Setenv("DB_PASSWORD", "") - os.Setenv("DB_DATABASE", "webapitest") - os.Setenv("DB_PARAMETER", "parseTime=true&allowNativePasswords=true") -} - -func unsetEnv() { - os.Unsetenv("DB_HOSTNAME") - os.Unsetenv("DB_PORT") - os.Unsetenv("DB_USERNAME") - os.Unsetenv("DB_PASSWORD") - os.Unsetenv("DB_DATABASE") - os.Unsetenv("DB_PARAMETER") -} func TestMigrationAll(t *testing.T) { - setEnv() + _, unique := migrateAll(t) + testutil.TeardownDatabase(unique) +} - testutil.ResetDatabase() - db := testutil.ConnectDatabase(true) +func migrateAll(t *testing.T) (*database.DBW, string) { + db, unique := testutil.SetupDatabase() // Set the arguments. - os.Args = make([]string, 4) + os.Args = make([]string, 6) os.Args[0] = "cliapp" os.Args[1] = "migrate" os.Args[2] = "all" os.Args[3] = "testdata/success.sql" + os.Args[4] = "--envprefix" + os.Args[5] = unique // Redirect stdout. backupd := os.Stdout @@ -91,21 +79,21 @@ func TestMigrationAll(t *testing.T) { err = db.Get(&rows, `SELECT count(*) from databasechangelog`) assert.Nil(t, err) assert.Equal(t, 3, rows) + + return db, unique } func TestMigrationReset(t *testing.T) { - TestMigrationAll(t) - - setEnv() - - db := testutil.ConnectDatabase(true) + db, unique := migrateAll(t) // Set the arguments. - os.Args = make([]string, 4) + os.Args = make([]string, 6) os.Args[0] = "cliapp" os.Args[1] = "migrate" os.Args[2] = "reset" os.Args[3] = "testdata/success.sql" + os.Args[4] = "--envprefix" + os.Args[5] = unique // Redirect stdout. backupd := os.Stdout @@ -128,21 +116,27 @@ func TestMigrationReset(t *testing.T) { err = db.Get(&rows, `SELECT count(*) from databasechangelog`) assert.Nil(t, err) assert.Equal(t, 0, rows) + + testutil.TeardownDatabase(unique) } func TestMigrationUp(t *testing.T) { - setEnv() + _, unique := migrateUp(t) + testutil.TeardownDatabase(unique) +} - testutil.ResetDatabase() - db := testutil.ConnectDatabase(true) +func migrateUp(t *testing.T) (*database.DBW, string) { + db, unique := testutil.SetupDatabase() // Set the arguments. - os.Args = make([]string, 5) + os.Args = make([]string, 7) os.Args[0] = "cliapp" os.Args[1] = "migrate" os.Args[2] = "up" os.Args[3] = "2" os.Args[4] = "testdata/success.sql" + os.Args[5] = "--envprefix" + os.Args[6] = unique // Redirect stdout. backupd := os.Stdout @@ -165,22 +159,22 @@ func TestMigrationUp(t *testing.T) { err = db.Get(&rows, `SELECT count(*) from databasechangelog`) assert.Nil(t, err) assert.Equal(t, 2, rows) + + return db, unique } func TestMigrationDown(t *testing.T) { - TestMigrationUp(t) - - setEnv() - - db := testutil.ConnectDatabase(true) + db, unique := migrateUp(t) // Set the arguments. - os.Args = make([]string, 5) + os.Args = make([]string, 7) os.Args[0] = "cliapp" os.Args[1] = "migrate" os.Args[2] = "down" os.Args[3] = "1" os.Args[4] = "testdata/success.sql" + os.Args[5] = "--envprefix" + os.Args[6] = unique // Redirect stdout. backupd := os.Stdout @@ -203,4 +197,6 @@ func TestMigrationDown(t *testing.T) { err = db.Get(&rows, `SELECT count(*) from databasechangelog`) assert.Nil(t, err) assert.Equal(t, 1, rows) + + testutil.TeardownDatabase(unique) } diff --git a/src/app/webapi/cmd/hooks/main.go b/src/app/webapi/cmd/hooks/main.go index b42a112..ff41721 100644 --- a/src/app/webapi/cmd/hooks/main.go +++ b/src/app/webapi/cmd/hooks/main.go @@ -88,9 +88,9 @@ func main() { // Set the Authorization header. t.Request.Headers["Authorization"] = "Bearer " + token - // Load the tables. - testutil.LoadDatabaseFromFile("../../../migration/tables-only.sql") - core, _ := component.NewCoreMock() + // Load the database with test data. + db, unique := testutil.LoadDatabaseFromFile("../../../migration/tables-only.sql") + core, _ := component.NewCoreMock(db) mux := router.New() user.New(core).Routes(mux) @@ -118,6 +118,8 @@ func main() { if t.Request.URI == "/v1/user/USERID" { t.FullPath = "/v1/user/" + id1 } + + testutil.TeardownDatabase(unique) }) if false { diff --git a/src/app/webapi/component/auth/index_test.go b/src/app/webapi/component/auth/index_test.go index 51f8bfc..4071f38 100644 --- a/src/app/webapi/component/auth/index_test.go +++ b/src/app/webapi/component/auth/index_test.go @@ -10,13 +10,15 @@ import ( "app/webapi/component" "app/webapi/internal/testrequest" + "app/webapi/internal/testutil" "app/webapi/model" "github.com/stretchr/testify/assert" ) func TestIndex(t *testing.T) { - core, m := component.NewCoreMock() + db, unique := testutil.LoadDatabase() + core, m := component.NewCoreMock(db) m.Token.GenerateFunc = func(userID string, duration time.Duration) (string, error) { b := []byte("0123456789ABCDEF0123456789ABCDEF") @@ -33,10 +35,13 @@ func TestIndex(t *testing.T) { assert.Equal(t, http.StatusOK, w.Code) assert.Equal(t, "OK", r.Body.Status) assert.Equal(t, "MDEyMzQ1Njc4OUFCQ0RFRjAxMjM0NTY3ODlBQkNERUY=", r.Body.Data.Token) + + testutil.TeardownDatabase(unique) } func TestIndexError(t *testing.T) { - core, m := component.NewCoreMock() + db, unique := testutil.LoadDatabase() + core, m := component.NewCoreMock(db) m.Token.GenerateFunc = func(userID string, duration time.Duration) (string, error) { return "", errors.New("generate error") @@ -51,4 +56,6 @@ func TestIndexError(t *testing.T) { assert.Equal(t, http.StatusInternalServerError, w.Code) assert.Equal(t, "Internal Server Error", r.Body.Status) assert.Equal(t, "generate error", r.Body.Message) + + testutil.TeardownDatabase(unique) } diff --git a/src/app/webapi/component/core_mock.go b/src/app/webapi/component/core_mock.go index bbcb10b..829a2bc 100644 --- a/src/app/webapi/component/core_mock.go +++ b/src/app/webapi/component/core_mock.go @@ -4,22 +4,22 @@ import ( "app/webapi/internal/bind" "app/webapi/internal/response" "app/webapi/internal/testutil" + "app/webapi/pkg/database" "app/webapi/pkg/query" ) // NewCoreMock returns all mocked dependencies. -func NewCoreMock() (Core, *CoreMock) { +func NewCoreMock(db *database.DBW) (Core, *CoreMock) { ml := new(testutil.MockLogger) - md := testutil.ConnectDatabase(true) - mq := query.New(md) + mq := query.New(db) mt := new(testutil.MockToken) resp := response.New() binder := bind.New() - core := NewCore(ml, md, mq, binder, resp, mt) + core := NewCore(ml, db, mq, binder, resp, mt) m := &CoreMock{ Log: ml, - DB: md, + DB: db, Q: mq, Bind: binder, Response: resp, diff --git a/src/app/webapi/component/root/index_test.go b/src/app/webapi/component/root/index_test.go index 64abbdf..2280f79 100644 --- a/src/app/webapi/component/root/index_test.go +++ b/src/app/webapi/component/root/index_test.go @@ -7,13 +7,15 @@ import ( "app/webapi/component" "app/webapi/internal/testrequest" + "app/webapi/internal/testutil" "app/webapi/model" "github.com/stretchr/testify/assert" ) func TestIndex(t *testing.T) { - core, _ := component.NewCoreMock() + db, unique := testutil.LoadDatabase() + core, _ := component.NewCoreMock(db) w := testrequest.SendForm(t, core, "GET", "/v1", nil) @@ -24,4 +26,6 @@ func TestIndex(t *testing.T) { assert.Equal(t, http.StatusOK, w.Code) assert.Equal(t, "OK", r.Body.Status) assert.Equal(t, "hello", r.Body.Message) + + testutil.TeardownDatabase(unique) } diff --git a/src/app/webapi/component/user/create_test.go b/src/app/webapi/component/user/create_test.go index 672cf8a..392f0de 100644 --- a/src/app/webapi/component/user/create_test.go +++ b/src/app/webapi/component/user/create_test.go @@ -17,8 +17,8 @@ import ( ) func TestCreate(t *testing.T) { - testutil.LoadDatabase(t) - core, _ := component.NewCoreMock() + db, unique := testutil.LoadDatabase() + core, _ := component.NewCoreMock(db) form := url.Values{} form.Add("first_name", "John") @@ -35,11 +35,13 @@ func TestCreate(t *testing.T) { assert.Equal(t, http.StatusCreated, w.Code) assert.Equal(t, "Created", r.Body.Status) assert.Equal(t, 36, len(r.Body.RecordID)) + + testutil.TeardownDatabase(unique) } func TestCreateUserAlreadyExists(t *testing.T) { - testutil.LoadDatabase(t) - core, _ := component.NewCoreMock() + db, unique := testutil.LoadDatabase() + core, _ := component.NewCoreMock(db) u := store.NewUser(core.DB, core.Q) _, err := u.Create("John", "Smith", "jsmith@example.com", "password") @@ -60,11 +62,13 @@ func TestCreateUserAlreadyExists(t *testing.T) { assert.Equal(t, http.StatusBadRequest, w.Code) assert.Equal(t, "Bad Request", r.Body.Status) assert.Equal(t, "user already exists", r.Body.Message) + + testutil.TeardownDatabase(unique) } func TestCreateBadEmail(t *testing.T) { - testutil.LoadDatabase(t) - core, _ := component.NewCoreMock() + db, unique := testutil.LoadDatabase() + core, _ := component.NewCoreMock(db) form := url.Values{} form.Add("first_name", "John") @@ -81,19 +85,23 @@ func TestCreateBadEmail(t *testing.T) { assert.Equal(t, http.StatusBadRequest, w.Code) assert.Equal(t, "Bad Request", r.Body.Status) assert.Contains(t, w.Body.String(), `failed`) + + testutil.TeardownDatabase(unique) } func TestCreateValidation(t *testing.T) { for _, v := range []string{ "POST /v1/user", } { - testutil.LoadDatabase(t) - core, _ := component.NewCoreMock() + db, unique := testutil.LoadDatabase() + core, _ := component.NewCoreMock(db) arr := strings.Split(v, " ") w := testrequest.SendForm(t, core, arr[0], arr[1], nil) assert.Equal(t, http.StatusBadRequest, w.Code) + + testutil.TeardownDatabase(unique) } } diff --git a/src/app/webapi/component/user/destroy_all_test.go b/src/app/webapi/component/user/destroy_all_test.go index 4960987..1723173 100644 --- a/src/app/webapi/component/user/destroy_all_test.go +++ b/src/app/webapi/component/user/destroy_all_test.go @@ -1,7 +1,6 @@ package user_test import ( - "app/webapi/model" "encoding/json" "net/http" "strings" @@ -10,14 +9,15 @@ import ( "app/webapi/component" "app/webapi/internal/testrequest" "app/webapi/internal/testutil" + "app/webapi/model" "app/webapi/store" "github.com/stretchr/testify/assert" ) func TestDestroyAll(t *testing.T) { - testutil.LoadDatabase(t) - core, _ := component.NewCoreMock() + db, unique := testutil.LoadDatabase() + core, _ := component.NewCoreMock(db) u := store.NewUser(core.DB, core.Q) _, err := u.Create("John", "Smith", "jsmith@example.com", "password") @@ -32,11 +32,13 @@ func TestDestroyAll(t *testing.T) { assert.Equal(t, http.StatusOK, w.Code) assert.Equal(t, "OK", r.Body.Status) assert.Equal(t, "users deleted", r.Body.Message) + + testutil.TeardownDatabase(unique) } func TestDestroyAllNoUsers(t *testing.T) { - testutil.LoadDatabase(t) - core, _ := component.NewCoreMock() + db, unique := testutil.LoadDatabase() + core, _ := component.NewCoreMock(db) w := testrequest.SendForm(t, core, "DELETE", "/v1/user", nil) @@ -47,19 +49,23 @@ func TestDestroyAllNoUsers(t *testing.T) { assert.Equal(t, http.StatusBadRequest, w.Code) assert.Equal(t, "Bad Request", r.Body.Status) assert.Equal(t, "no users to delete", r.Body.Message) + + testutil.TeardownDatabase(unique) } func TestDestroyValidation(t *testing.T) { for _, v := range []string{ "DELETE /v1/user/1", } { - testutil.LoadDatabase(t) - core, _ := component.NewCoreMock() + db, unique := testutil.LoadDatabase() + core, _ := component.NewCoreMock(db) arr := strings.Split(v, " ") w := testrequest.SendForm(t, core, arr[0], arr[1], nil) assert.Equal(t, http.StatusBadRequest, w.Code) + + testutil.TeardownDatabase(unique) } } diff --git a/src/app/webapi/component/user/destroy_test.go b/src/app/webapi/component/user/destroy_test.go index da14772..3f045be 100644 --- a/src/app/webapi/component/user/destroy_test.go +++ b/src/app/webapi/component/user/destroy_test.go @@ -16,8 +16,8 @@ import ( ) func TestDestroy(t *testing.T) { - testutil.LoadDatabase(t) - core, _ := component.NewCoreMock() + db, unique := testutil.LoadDatabase() + core, _ := component.NewCoreMock(db) u := store.NewUser(core.DB, core.Q) ID, err := u.Create("John", "Smith", "jsmith@example.com", "password") @@ -42,4 +42,6 @@ func TestDestroy(t *testing.T) { found, err := u.FindOneByID(u, ID) assert.Nil(t, err) assert.False(t, found) + + testutil.TeardownDatabase(unique) } diff --git a/src/app/webapi/component/user/index_test.go b/src/app/webapi/component/user/index_test.go index b3f1ec3..7524b25 100644 --- a/src/app/webapi/component/user/index_test.go +++ b/src/app/webapi/component/user/index_test.go @@ -15,8 +15,8 @@ import ( ) func TestIndexEmpty(t *testing.T) { - testutil.LoadDatabase(t) - core, _ := component.NewCoreMock() + db, unique := testutil.LoadDatabase() + core, _ := component.NewCoreMock(db) w := testrequest.SendForm(t, core, "GET", "/v1/user", nil) @@ -27,11 +27,13 @@ func TestIndexEmpty(t *testing.T) { assert.Equal(t, http.StatusOK, w.Code) assert.Equal(t, "OK", r.Body.Status) assert.Equal(t, 0, len(r.Body.Data)) + + testutil.TeardownDatabase(unique) } func TestIndexOne(t *testing.T) { - testutil.LoadDatabase(t) - core, _ := component.NewCoreMock() + db, unique := testutil.LoadDatabase() + core, _ := component.NewCoreMock(db) u := store.NewUser(core.DB, core.Q) _, err := u.Create("John", "Smith", "jsmith@example.com", "password") @@ -48,4 +50,6 @@ func TestIndexOne(t *testing.T) { assert.Equal(t, "John", r.Body.Data[0].FirstName) assert.Equal(t, "Smith", r.Body.Data[0].LastName) assert.Equal(t, "jsmith@example.com", r.Body.Data[0].Email) + + testutil.TeardownDatabase(unique) } diff --git a/src/app/webapi/component/user/show_test.go b/src/app/webapi/component/user/show_test.go index d498bf1..20c919d 100644 --- a/src/app/webapi/component/user/show_test.go +++ b/src/app/webapi/component/user/show_test.go @@ -15,8 +15,8 @@ import ( ) func TestShowOne(t *testing.T) { - testutil.LoadDatabase(t) - core, _ := component.NewCoreMock() + db, unique := testutil.LoadDatabase() + core, _ := component.NewCoreMock(db) u := store.NewUser(core.DB, core.Q) ID, err := u.Create("John", "Smith", "jsmith@example.com", "password") @@ -34,11 +34,13 @@ func TestShowOne(t *testing.T) { assert.Equal(t, "John", r.Body.Data[0].FirstName) assert.Equal(t, "Smith", r.Body.Data[0].LastName) assert.Equal(t, "jsmith@example.com", r.Body.Data[0].Email) + + testutil.TeardownDatabase(unique) } func TestShowNotFound(t *testing.T) { - testutil.LoadDatabase(t) - core, _ := component.NewCoreMock() + db, unique := testutil.LoadDatabase() + core, _ := component.NewCoreMock(db) w := testrequest.SendForm(t, core, "GET", "/v1/user/1", nil) @@ -49,4 +51,6 @@ func TestShowNotFound(t *testing.T) { assert.Equal(t, http.StatusBadRequest, w.Code) assert.Equal(t, "Bad Request", r.Body.Status) assert.Equal(t, "user not found", r.Body.Message) + + testutil.TeardownDatabase(unique) } diff --git a/src/app/webapi/component/user/update_test.go b/src/app/webapi/component/user/update_test.go index 4d2c171..ba548db 100644 --- a/src/app/webapi/component/user/update_test.go +++ b/src/app/webapi/component/user/update_test.go @@ -16,8 +16,8 @@ import ( ) func TestUpdateUserAllFields(t *testing.T) { - testutil.LoadDatabase(t) - core, _ := component.NewCoreMock() + db, unique := testutil.LoadDatabase() + core, _ := component.NewCoreMock(db) u := store.NewUser(core.DB, core.Q) ID, err := u.Create("John", "Smith", "jsmith@example.com", "password") @@ -46,11 +46,13 @@ func TestUpdateUserAllFields(t *testing.T) { assert.Equal(t, "Smith2", u.LastName) assert.Equal(t, "jsmith3@example.com", u.Email) assert.Equal(t, "password4", u.Password) + + testutil.TeardownDatabase(unique) } func TestUpdateMissingFields(t *testing.T) { - testutil.LoadDatabase(t) - core, _ := component.NewCoreMock() + db, unique := testutil.LoadDatabase() + core, _ := component.NewCoreMock(db) u := store.NewUser(core.DB, core.Q) ID, err := u.Create("John", "Smith", "jsmith@example.com", "password") @@ -76,4 +78,6 @@ func TestUpdateMissingFields(t *testing.T) { assert.Equal(t, "Smith", u.LastName) assert.Equal(t, "jsmith@example.com", u.Email) assert.Equal(t, "password", u.Password) + + testutil.TeardownDatabase(unique) } diff --git a/src/app/webapi/internal/basemigrate/basemigrate_test.go b/src/app/webapi/internal/basemigrate/basemigrate_test.go index e1a08db..a6a8938 100644 --- a/src/app/webapi/internal/basemigrate/basemigrate_test.go +++ b/src/app/webapi/internal/basemigrate/basemigrate_test.go @@ -10,11 +10,10 @@ import ( ) func TestMigration(t *testing.T) { - testutil.ResetDatabase() - db := testutil.ConnectDatabase(true) + db, unique := testutil.SetupDatabase() // Run migration. - err := basemigrate.Migrate("testdata/success.sql", 0, false) + err := basemigrate.Migrate("testdata/success.sql", unique, 0, false) assert.Nil(t, err) // Count the records. @@ -24,11 +23,11 @@ func TestMigration(t *testing.T) { assert.Equal(t, 3, rows) // Run migration again. - err = basemigrate.Migrate("testdata/success.sql", 0, false) + err = basemigrate.Migrate("testdata/success.sql", unique, 0, false) assert.Nil(t, err) // Remove all migrations. - err = basemigrate.Reset("testdata/success.sql", 0, false) + err = basemigrate.Reset("testdata/success.sql", unique, 0, false) assert.Nil(t, err) rows = 0 @@ -37,11 +36,11 @@ func TestMigration(t *testing.T) { assert.Equal(t, 0, rows) // Remove all migrations again. - err = basemigrate.Reset("testdata/success.sql", 0, false) + err = basemigrate.Reset("testdata/success.sql", unique, 0, false) assert.Nil(t, err) // Run 2 migrations. - err = basemigrate.Migrate("testdata/success.sql", 2, false) + err = basemigrate.Migrate("testdata/success.sql", unique, 2, false) assert.Nil(t, err) rows = 0 @@ -50,20 +49,21 @@ func TestMigration(t *testing.T) { assert.Equal(t, 2, rows) // Remove 1 migration. - err = basemigrate.Reset("testdata/success.sql", 1, false) + err = basemigrate.Reset("testdata/success.sql", unique, 1, false) assert.Nil(t, err) rows = 0 err = db.Get(&rows, `SELECT count(*) from databasechangelog`) assert.Nil(t, err) assert.Equal(t, 1, rows) + + testutil.TeardownDatabase(unique) } func TestMigrationFailDuplicate(t *testing.T) { - testutil.ResetDatabase() - db := testutil.ConnectDatabase(true) + db, unique := testutil.SetupDatabase() - err := basemigrate.Migrate("testdata/fail-duplicate.sql", 0, false) + err := basemigrate.Migrate("testdata/fail-duplicate.sql", unique, 0, false) assert.Contains(t, err.Error(), "checksum does not match") rows := 0 @@ -71,15 +71,14 @@ func TestMigrationFailDuplicate(t *testing.T) { assert.Nil(t, err) assert.Equal(t, 2, rows) - testutil.ResetDatabase() + testutil.TeardownDatabase(unique) } func TestInclude(t *testing.T) { - testutil.ResetDatabase() - db := testutil.ConnectDatabase(true) + db, unique := testutil.SetupDatabase() // Run migration. - err := basemigrate.Migrate("testdata/parent.sql", 0, false) + err := basemigrate.Migrate("testdata/parent.sql", unique, 0, false) assert.Nil(t, err) // Count the records. @@ -89,11 +88,11 @@ func TestInclude(t *testing.T) { assert.Equal(t, 3, rows) // Run migration again. - err = basemigrate.Migrate("testdata/parent.sql", 0, false) + err = basemigrate.Migrate("testdata/parent.sql", unique, 0, false) assert.Nil(t, err) // Remove all migrations. - err = basemigrate.Reset("testdata/parent.sql", 0, false) + err = basemigrate.Reset("testdata/parent.sql", unique, 0, false) assert.Nil(t, err) rows = 0 @@ -102,11 +101,11 @@ func TestInclude(t *testing.T) { assert.Equal(t, 0, rows) // Remove all migrations again. - err = basemigrate.Reset("testdata/parent.sql", 0, false) + err = basemigrate.Reset("testdata/parent.sql", unique, 0, false) assert.Nil(t, err) // Run 2 migrations. - err = basemigrate.Migrate("testdata/parent.sql", 2, false) + err = basemigrate.Migrate("testdata/parent.sql", unique, 2, false) assert.Nil(t, err) rows = 0 @@ -115,11 +114,13 @@ func TestInclude(t *testing.T) { assert.Equal(t, 2, rows) // Remove 1 migration. - err = basemigrate.Reset("testdata/parent.sql", 1, false) + err = basemigrate.Reset("testdata/parent.sql", unique, 1, false) assert.Nil(t, err) rows = 0 err = db.Get(&rows, `SELECT count(*) from databasechangelog`) assert.Nil(t, err) assert.Equal(t, 1, rows) + + testutil.TeardownDatabase(unique) } diff --git a/src/app/webapi/internal/basemigrate/helper.go b/src/app/webapi/internal/basemigrate/helper.go index aa96261..6a5b037 100644 --- a/src/app/webapi/internal/basemigrate/helper.go +++ b/src/app/webapi/internal/basemigrate/helper.go @@ -13,11 +13,11 @@ import ( ) // connect will connect to the database. -func connect() (*sqlx.DB, error) { +func connect(prefix string) (*sqlx.DB, error) { dbc := new(database.Connection) // Load the struct from environment variables. - err := env.Unmarshal(dbc) + err := env.Unmarshal(dbc, prefix) if err != nil { return nil, err } diff --git a/src/app/webapi/internal/basemigrate/migrate.go b/src/app/webapi/internal/basemigrate/migrate.go index b93f6dc..28aebc0 100644 --- a/src/app/webapi/internal/basemigrate/migrate.go +++ b/src/app/webapi/internal/basemigrate/migrate.go @@ -8,8 +8,8 @@ import ( // Migrate will perform all the migrations in a file. If max is 0, all // migrations are run. -func Migrate(filename string, max int, verbose bool) (err error) { - db, err := connect() +func Migrate(filename string, prefix string, max int, verbose bool) (err error) { + db, err := connect(prefix) if err != nil { return err } diff --git a/src/app/webapi/internal/basemigrate/reset.go b/src/app/webapi/internal/basemigrate/reset.go index 1f44761..3eee5aa 100644 --- a/src/app/webapi/internal/basemigrate/reset.go +++ b/src/app/webapi/internal/basemigrate/reset.go @@ -15,8 +15,8 @@ type DBChangeset struct { } // Reset will remove all migrations. If max is 0, all rollbacks are run. -func Reset(filename string, max int, verbose bool) (err error) { - db, err := connect() +func Reset(filename string, prefix string, max int, verbose bool) (err error) { + db, err := connect(prefix) if err != nil { return err } diff --git a/src/app/webapi/internal/testutil/database.go b/src/app/webapi/internal/testutil/database.go index b3516a8..53dc6ea 100644 --- a/src/app/webapi/internal/testutil/database.go +++ b/src/app/webapi/internal/testutil/database.go @@ -2,40 +2,49 @@ package testutil import ( "fmt" - "io/ioutil" "log" + "math/rand" "os" - "strings" - "testing" + "time" "app/webapi/internal/basemigrate" - "app/webapi/pkg/env" - "app/webapi/pkg/database" + "app/webapi/pkg/env" ) -func setEnv() { - os.Setenv("DB_HOSTNAME", "127.0.0.1") - os.Setenv("DB_PORT", "3306") - os.Setenv("DB_USERNAME", "root") - os.Setenv("DB_PASSWORD", "") - os.Setenv("DB_DATABASE", "webapitest") - os.Setenv("DB_PARAMETER", "parseTime=true&allowNativePasswords=true") +func init() { + rand.Seed(time.Now().UnixNano()) } -// ConnectDatabase returns a test database connection. -func ConnectDatabase(dbSpecificDB bool) *database.DBW { - dbc := new(database.Connection) - setEnv() +func setEnv(unique string) { + os.Setenv(unique+"DB_HOSTNAME", "127.0.0.1") + os.Setenv(unique+"DB_PORT", "3306") + os.Setenv(unique+"DB_USERNAME", "root") + os.Setenv(unique+"DB_PASSWORD", "") + os.Setenv(unique+"DB_DATABASE", "webapitest"+unique) + os.Setenv(unique+"DB_PARAMETER", "parseTime=true&allowNativePasswords=true") +} - err := env.Unmarshal(dbc) +func unsetEnv(unique string) { + os.Unsetenv(unique + "DB_HOSTNAME") + os.Unsetenv(unique + "DB_PORT") + os.Unsetenv(unique + "DB_USERNAME") + os.Unsetenv(unique + "DB_PASSWORD") + os.Unsetenv(unique + "DB_DATABASE") + os.Unsetenv(unique + "DB_PARAMETER") +} + +// connectDatabase returns a test database connection. +func connectDatabase(dbSpecificDB bool, unique string) *database.DBW { + dbc := new(database.Connection) + err := env.Unmarshal(dbc, unique) if err != nil { - log.Println("DB ENV Error:", err) + fmt.Println("DB ENV Error:", err) } connection, err := dbc.Connect(dbSpecificDB) if err != nil { - log.Println("DB Error:", err) + fmt.Println("DB Error:", err) } dbw := database.New(connection) @@ -43,49 +52,57 @@ func ConnectDatabase(dbSpecificDB bool) *database.DBW { return dbw } -// ResetDatabase will drop and create the test database. -func ResetDatabase() { - db := ConnectDatabase(false) - _, err := db.Exec(`DROP DATABASE IF EXISTS webapitest`) +// SetupDatabase will create the test database and set the environment +// variables. +func SetupDatabase() (*database.DBW, string) { + unique := "T" + fmt.Sprint(rand.Intn(500)) + setEnv(unique) + + db := connectDatabase(false, unique) + _, err := db.Exec(`DROP DATABASE IF EXISTS webapitest` + unique) if err != nil { - fmt.Println(err) + fmt.Println("DB DROP SETUP Error:", err) } - _, err = db.Exec(`CREATE DATABASE webapitest DEFAULT CHARSET = utf8 COLLATE = utf8_unicode_ci`) + _, err = db.Exec(`CREATE DATABASE webapitest` + unique + ` DEFAULT CHARSET = utf8 COLLATE = utf8_unicode_ci`) if err != nil { - fmt.Println(err) + fmt.Println("DB CREATE Error:", err) } + + return connectDatabase(true, unique), unique } -// LoadDatabase will set up the DB for the tests. -func LoadDatabase(t *testing.T) { - ResetDatabase() +// TeardownDatabase will destroy the test database and unset the environment +// variables. +func TeardownDatabase(unique string) { + db := connectDatabase(false, unique) + _, err := db.Exec(`DROP DATABASE IF EXISTS webapitest` + unique) + if err != nil { + fmt.Println("DB DROP TEARDOWN Error:", err) + } + + unsetEnv(unique) +} - err := basemigrate.Migrate("../../../../../migration/mysql-v0.sql", 0, false) +// LoadDatabase will set up the DB and apply migrations for the tests. +func LoadDatabase() (*database.DBW, string) { + db, unique := SetupDatabase() + + err := basemigrate.Migrate("../../../../../migration/mysql-v0.sql", unique, 0, false) if err != nil { log.Println("DB Error:", err) } + + return db, unique } // LoadDatabaseFromFile will set up the DB for the tests. -func LoadDatabaseFromFile(file string) { - ResetDatabase() +func LoadDatabaseFromFile(file string) (*database.DBW, string) { + db, unique := SetupDatabase() - db := ConnectDatabase(true) - b, err := ioutil.ReadFile(file) + err := basemigrate.Migrate(file, unique, 0, false) if err != nil { - log.Println(err) - os.Exit(1) + log.Println("DB Error:", err) } - // Split each statement. - stmts := strings.Split(string(b), ";") - for i, s := range stmts { - if i == len(stmts)-1 { - break - } - _, err = db.Exec(s) - if err != nil { - log.Println(err) - } - } + return db, unique } diff --git a/src/app/webapi/pkg/env/env.go b/src/app/webapi/pkg/env/env.go index 83132f7..759fe34 100644 --- a/src/app/webapi/pkg/env/env.go +++ b/src/app/webapi/pkg/env/env.go @@ -10,7 +10,7 @@ import ( // Unmarshal will fill a struct from environment variables. It supports struct // values of string, int, and bool. -func Unmarshal(dst interface{}) (err error) { +func Unmarshal(dst interface{}, prefix string) (err error) { // Ensure a pointer is passed in. vdst := reflect.ValueOf(dst) if vdst.Kind() != reflect.Ptr { @@ -33,7 +33,7 @@ func Unmarshal(dst interface{}) (err error) { envname := tag.Get("env") // Get the environment variable from the tag. - val := os.Getenv(envname) + val := os.Getenv(prefix + envname) // If the environment variable exists, set the value. if len(val) > 0 { diff --git a/src/app/webapi/pkg/env/env_test.go b/src/app/webapi/pkg/env/env_test.go index 731fc63..3a8a182 100644 --- a/src/app/webapi/pkg/env/env_test.go +++ b/src/app/webapi/pkg/env/env_test.go @@ -20,7 +20,7 @@ type Connection struct { func TestUnmarshalEmpty(t *testing.T) { c := new(Connection) - err := env.Unmarshal(c) + err := env.Unmarshal(c, "") assert.Nil(t, err) assert.Equal(t, "", c.Username) @@ -38,7 +38,7 @@ func TestUnmarshalFilled(t *testing.T) { os.Setenv("DB_SSL", "TRUE") c := new(Connection) - err := env.Unmarshal(c) + err := env.Unmarshal(c, "") assert.Nil(t, err) assert.Equal(t, "a", c.Username) @@ -54,25 +54,49 @@ func TestUnmarshalFilled(t *testing.T) { os.Unsetenv("DB_SSL") } +func TestUnmarshalFilledPrefix(t *testing.T) { + os.Setenv("TEST_DB_USERNAME", "a") + os.Setenv("TEST_DB_PASSWORD", "B") + os.Setenv("TEST_DB_DATABASE", "c123") + os.Setenv("TEST_DB_PORT", "3306") + os.Setenv("TEST_DB_SSL", "TRUE") + + c := new(Connection) + err := env.Unmarshal(c, "TEST_") + assert.Nil(t, err) + + assert.Equal(t, "a", c.Username) + assert.Equal(t, "B", c.Password) + assert.Equal(t, "c123", c.Database) + assert.Equal(t, 3306, c.Port) + assert.Equal(t, true, c.SSL) + + os.Unsetenv("TEST_DB_USERNAME") + os.Unsetenv("TEST_DB_PASSWORD") + os.Unsetenv("TEST_DB_DATABASE") + os.Unsetenv("TEST_DB_PORT") + os.Unsetenv("TEST_DB_SSL") +} + func TestUnmarshalError(t *testing.T) { c := "string" - err := env.Unmarshal(c) + err := env.Unmarshal(c, "") assert.Contains(t, err.Error(), "type not pointer") d := "string" - err = env.Unmarshal(&d) + err = env.Unmarshal(&d, "") assert.Contains(t, err.Error(), "type not struct") os.Setenv("DB_SSL", "TRUEX") f := new(Connection) - err = env.Unmarshal(f) + err = env.Unmarshal(f, "") assert.NotNil(t, err) assert.Equal(t, false, f.SSL) os.Unsetenv("DB_SSL") - os.Setenv("DB_PORT", "monkey") + os.Setenv("DB_PORT", "bad") g := new(Connection) - err = env.Unmarshal(f) + err = env.Unmarshal(f, "") assert.NotNil(t, err) assert.Equal(t, false, g.SSL) os.Unsetenv("DB_PORT") From 4e9dd9c5d1989127d709d4d751c3d166aea55500 Mon Sep 17 00:00:00 2001 From: Joseph Spurrier Date: Tue, 17 Jul 2018 05:46:52 -0400 Subject: [PATCH 09/14] Fix hooks database loading --- src/app/webapi/cmd/hooks/main.go | 2 +- src/app/webapi/internal/testutil/database.go | 30 +++++++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/app/webapi/cmd/hooks/main.go b/src/app/webapi/cmd/hooks/main.go index ff41721..43ff8de 100644 --- a/src/app/webapi/cmd/hooks/main.go +++ b/src/app/webapi/cmd/hooks/main.go @@ -89,7 +89,7 @@ func main() { t.Request.Headers["Authorization"] = "Bearer " + token // Load the database with test data. - db, unique := testutil.LoadDatabaseFromFile("../../../migration/tables-only.sql") + db, unique := testutil.LoadDatabaseFromFile("../../../migration/tables-only.sql", false) core, _ := component.NewCoreMock(db) mux := router.New() diff --git a/src/app/webapi/internal/testutil/database.go b/src/app/webapi/internal/testutil/database.go index 53dc6ea..92bdc2b 100644 --- a/src/app/webapi/internal/testutil/database.go +++ b/src/app/webapi/internal/testutil/database.go @@ -85,19 +85,29 @@ func TeardownDatabase(unique string) { // LoadDatabase will set up the DB and apply migrations for the tests. func LoadDatabase() (*database.DBW, string) { - db, unique := SetupDatabase() - - err := basemigrate.Migrate("../../../../../migration/mysql-v0.sql", unique, 0, false) - if err != nil { - log.Println("DB Error:", err) - } - - return db, unique + return LoadDatabaseFromFile("../../../../../migration/mysql-v0.sql", true) } // LoadDatabaseFromFile will set up the DB for the tests. -func LoadDatabaseFromFile(file string) (*database.DBW, string) { - db, unique := SetupDatabase() +func LoadDatabaseFromFile(file string, usePrefix bool) (*database.DBW, string) { + unique := "" + var db *database.DBW + + if usePrefix { + db, unique = SetupDatabase() + } else { + db = connectDatabase(false, "") + _, err := db.Exec(`DROP DATABASE IF EXISTS webapitest`) + if err != nil { + fmt.Println("DB DROP SETUP Error:", err) + } + _, err = db.Exec(`CREATE DATABASE webapitest DEFAULT CHARSET = utf8 COLLATE = utf8_unicode_ci`) + if err != nil { + fmt.Println("DB CREATE Error:", err) + } + + db = connectDatabase(true, unique) + } err := basemigrate.Migrate(file, unique, 0, false) if err != nil { From dd0cf0df817f9af4abd3355f848f717db34c71be Mon Sep 17 00:00:00 2001 From: Joseph Spurrier Date: Tue, 17 Jul 2018 05:50:28 -0400 Subject: [PATCH 10/14] Add missing env set --- src/app/webapi/internal/testutil/database.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app/webapi/internal/testutil/database.go b/src/app/webapi/internal/testutil/database.go index 92bdc2b..103eb74 100644 --- a/src/app/webapi/internal/testutil/database.go +++ b/src/app/webapi/internal/testutil/database.go @@ -96,6 +96,7 @@ func LoadDatabaseFromFile(file string, usePrefix bool) (*database.DBW, string) { if usePrefix { db, unique = SetupDatabase() } else { + setEnv(unique) db = connectDatabase(false, "") _, err := db.Exec(`DROP DATABASE IF EXISTS webapitest`) if err != nil { From 6034202e2bfbe52c80a11c9bf0354cb1b49570ef Mon Sep 17 00:00:00 2001 From: Joseph Spurrier Date: Tue, 17 Jul 2018 05:56:29 -0400 Subject: [PATCH 11/14] Remove the old migration --- migration/tables-only.sql | 41 ------------------- src/app/webapi/cmd/hooks/main.go | 2 +- .../webapi/internal/basemigrate/migrate.go | 1 + 3 files changed, 2 insertions(+), 42 deletions(-) delete mode 100644 migration/tables-only.sql diff --git a/migration/tables-only.sql b/migration/tables-only.sql deleted file mode 100644 index 9db5064..0000000 --- a/migration/tables-only.sql +++ /dev/null @@ -1,41 +0,0 @@ -SET NAMES utf8 COLLATE 'utf8_unicode_ci'; -SET foreign_key_checks = 1; -SET time_zone = '+00:00'; -SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO'; -SET CHARACTER SET utf8; - -CREATE TABLE IF NOT EXISTS user_status ( - id TINYINT(1) UNSIGNED NOT NULL AUTO_INCREMENT, - - status VARCHAR(25) NOT NULL, - - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - deleted TINYINT(1) UNSIGNED NOT NULL DEFAULT 0, - - PRIMARY KEY (id) -); - -CREATE TABLE IF NOT EXISTS user ( - id VARCHAR(36) NOT NULL, - - first_name VARCHAR(50) NOT NULL, - last_name VARCHAR(50) NOT NULL, - email VARCHAR(100) NOT NULL, - password CHAR(60) NOT NULL, - - status_id TINYINT(1) UNSIGNED NOT NULL DEFAULT 1, - - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - deleted_at TIMESTAMP DEFAULT 0, - - UNIQUE KEY (email), - CONSTRAINT `f_user_status` FOREIGN KEY (`status_id`) REFERENCES `user_status` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, - - PRIMARY KEY (id) -); - -INSERT INTO `user_status` (`id`, `status`, `created_at`, `updated_at`, `deleted`) VALUES -(1, 'active', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0), -(2, 'inactive', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0); \ No newline at end of file diff --git a/src/app/webapi/cmd/hooks/main.go b/src/app/webapi/cmd/hooks/main.go index 43ff8de..19acab3 100644 --- a/src/app/webapi/cmd/hooks/main.go +++ b/src/app/webapi/cmd/hooks/main.go @@ -89,7 +89,7 @@ func main() { t.Request.Headers["Authorization"] = "Bearer " + token // Load the database with test data. - db, unique := testutil.LoadDatabaseFromFile("../../../migration/tables-only.sql", false) + db, unique := testutil.LoadDatabaseFromFile("../../../migration/mysql-v0.sql", false) core, _ := component.NewCoreMock(db) mux := router.New() diff --git a/src/app/webapi/internal/basemigrate/migrate.go b/src/app/webapi/internal/basemigrate/migrate.go index 28aebc0..ca9d28e 100644 --- a/src/app/webapi/internal/basemigrate/migrate.go +++ b/src/app/webapi/internal/basemigrate/migrate.go @@ -53,6 +53,7 @@ func Migrate(filename string, prefix string, max int, verbose bool) (err error) } arrQueries := strings.Split(cs.Changes(), ";") + // Loop through each change. for _, q := range arrQueries { if len(q) == 0 { From 8c889dd04d0dbadb4e6031f5d24105aa99f0a589 Mon Sep 17 00:00:00 2001 From: Joseph Spurrier Date: Tue, 17 Jul 2018 06:16:14 -0400 Subject: [PATCH 12/14] Add transaction to migration --- .../webapi/internal/basemigrate/migrate.go | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/app/webapi/internal/basemigrate/migrate.go b/src/app/webapi/internal/basemigrate/migrate.go index ca9d28e..fc0a5c9 100644 --- a/src/app/webapi/internal/basemigrate/migrate.go +++ b/src/app/webapi/internal/basemigrate/migrate.go @@ -8,7 +8,7 @@ import ( // Migrate will perform all the migrations in a file. If max is 0, all // migrations are run. -func Migrate(filename string, prefix string, max int, verbose bool) (err error) { +func Migrate(filename string, prefix string, max int, verbose bool) error { db, err := connect(prefix) if err != nil { return err @@ -16,6 +16,9 @@ func Migrate(filename string, prefix string, max int, verbose bool) (err error) // Create the DATABASECHANGELOG. _, err = db.Exec(sqlChangelog) + if err != nil { + return err + } // Get the changesets. arr, err := parseFileToArray(filename) @@ -54,6 +57,11 @@ func Migrate(filename string, prefix string, max int, verbose bool) (err error) arrQueries := strings.Split(cs.Changes(), ";") + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("sql error begin transaction - %v", err.Error()) + } + // Loop through each change. for _, q := range arrQueries { if len(q) == 0 { @@ -61,12 +69,21 @@ func Migrate(filename string, prefix string, max int, verbose bool) (err error) } // Execute the query. - _, err = db.Exec(q) + _, err = tx.Exec(q) if err != nil { return fmt.Errorf("sql error on changeset %v:%v - %v", cs.author, cs.id, err.Error()) } } + err = tx.Commit() + if err != nil { + errr := tx.Rollback() + if errr != nil { + return fmt.Errorf("sql error on rollback changeset %v:%v - %v", cs.author, cs.id, errr.Error()) + } + return fmt.Errorf("sql error on commit changeset %v:%v - %v", cs.author, cs.id, err.Error()) + } + // Count the number of rows. count := 0 err = db.Get(&count, `SELECT COUNT(*) FROM databasechangelog`) @@ -97,5 +114,5 @@ func Migrate(filename string, prefix string, max int, verbose bool) (err error) } } - return + return nil } From 2b5a014e918c4fd185ccd06f4f46f71979afc5b6 Mon Sep 17 00:00:00 2001 From: Joseph Spurrier Date: Tue, 17 Jul 2018 06:19:09 -0400 Subject: [PATCH 13/14] Add transaction to reset --- src/app/webapi/internal/basemigrate/migrate.go | 4 ++-- src/app/webapi/internal/basemigrate/reset.go | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/app/webapi/internal/basemigrate/migrate.go b/src/app/webapi/internal/basemigrate/migrate.go index fc0a5c9..7cfafaa 100644 --- a/src/app/webapi/internal/basemigrate/migrate.go +++ b/src/app/webapi/internal/basemigrate/migrate.go @@ -79,9 +79,9 @@ func Migrate(filename string, prefix string, max int, verbose bool) error { if err != nil { errr := tx.Rollback() if errr != nil { - return fmt.Errorf("sql error on rollback changeset %v:%v - %v", cs.author, cs.id, errr.Error()) + return fmt.Errorf("sql error on commit rollback %v:%v - %v", cs.author, cs.id, errr.Error()) } - return fmt.Errorf("sql error on commit changeset %v:%v - %v", cs.author, cs.id, err.Error()) + return fmt.Errorf("sql error on commit %v:%v - %v", cs.author, cs.id, err.Error()) } // Count the number of rows. diff --git a/src/app/webapi/internal/basemigrate/reset.go b/src/app/webapi/internal/basemigrate/reset.go index 3eee5aa..0d8b899 100644 --- a/src/app/webapi/internal/basemigrate/reset.go +++ b/src/app/webapi/internal/basemigrate/reset.go @@ -57,6 +57,11 @@ func Reset(filename string, prefix string, max int, verbose bool) (err error) { arrQueries := strings.Split(cs.Rollbacks(), ";") + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("sql error begin transaction - %v", err.Error()) + } + // Loop through each rollback. for _, q := range arrQueries { if len(q) == 0 { @@ -64,12 +69,21 @@ func Reset(filename string, prefix string, max int, verbose bool) (err error) { } // Execute the query. - _, err = db.Exec(q) + _, err = tx.Exec(q) if err != nil { return fmt.Errorf("sql error on rollback %v:%v - %v", cs.author, cs.id, err.Error()) } } + err = tx.Commit() + if err != nil { + errr := tx.Rollback() + if errr != nil { + return fmt.Errorf("sql error on commit rollback %v:%v - %v", cs.author, cs.id, errr.Error()) + } + return fmt.Errorf("sql error on commit %v:%v - %v", cs.author, cs.id, err.Error()) + } + // Delete the record. _, err = db.Exec(` DELETE FROM databasechangelog From a6a971deb891b1f63c4fe8d015af00b3da461722 Mon Sep 17 00:00:00 2001 From: Joseph Spurrier Date: Tue, 17 Jul 2018 06:22:09 -0400 Subject: [PATCH 14/14] Remove teardown --- src/app/webapi/cmd/hooks/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app/webapi/cmd/hooks/main.go b/src/app/webapi/cmd/hooks/main.go index 19acab3..0f1466b 100644 --- a/src/app/webapi/cmd/hooks/main.go +++ b/src/app/webapi/cmd/hooks/main.go @@ -89,7 +89,7 @@ func main() { t.Request.Headers["Authorization"] = "Bearer " + token // Load the database with test data. - db, unique := testutil.LoadDatabaseFromFile("../../../migration/mysql-v0.sql", false) + db, _ := testutil.LoadDatabaseFromFile("../../../migration/mysql-v0.sql", false) core, _ := component.NewCoreMock(db) mux := router.New() @@ -119,7 +119,7 @@ func main() { t.FullPath = "/v1/user/" + id1 } - testutil.TeardownDatabase(unique) + //testutil.TeardownDatabase(unique) }) if false {