Add an include function to migrations

README.md

@@ -18,20 +18,62 @@ You must use Go 1.7 or newer because it uses the http context.
 
 ## Quick Start with MySQL
 
-Use one of the following commands to start a MySQL container with Docker:
+Use the following commands to start a MySQL container with Docker:
 
-- Start MySQL without a password: `docker run -d --name=mysql57 -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=yes mysql:5.7`
-- Start MySQL with a password: `docker run -d --name=mysql57 -p 3306:3306 -e MYSQL_ROOT_PASSWORD=somepassword mysql:5.7`
+```bash
+# Start MySQL without a password.
+docker run -d --name=mysql57 -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=yes mysql:5.7
+# or start MySQL with a password.
+docker run -d --name=mysql57 -p 3306:3306 -e MYSQL_ROOT_PASSWORD=somepassword mysql:5.7
+
+# Create the database via docker exec.
+docker exec mysql57 sh -c 'exec mysql -uroot -e "CREATE DATABASE IF NOT EXISTS webapi DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;"'
+# Or create the database manually.
+CREATE DATABASE webapi DEFAULT CHARSET = utf8 COLLATE = utf8_unicode_ci;
+
+# CD to the CLI tool.
+cd src/app/webapi/cmd/cliapp
+
+# Build the CLI tool.
+go build
+
+# Apply the database migrations without a password.
+DB_USERNAME=root DB_HOSTNAME=127.0.0.1 DB_PORT=3306 DB_DATABASE=webapi ./cliapp migrate all ../../../../../migration/mysql-v0.sql
+# or apply the database migrations with a password.
+DB_USERNAME=root DB_PASSWORD=somepassword DB_HOSTNAME=127.0.0.1 DB_PORT=3306 DB_DATABASE=webapi ./cliapp migrate all ../../../../../migration/mysql-v0.sql
+```
+
+Using the database connection information above, follow the steps to set up the `config.json` file:
 
-Start MySQL and import `migration/mysql.sql` to create the database and tables.
+```bash
+# Copy the config.json from the root of the project to the CLI app folder.
+cp config.json src/app/webapi/cmd/webapi/config.json
+
+# Edit the `Database` section so the connection information matches your MySQL instance.
+# The database password is read from the `config.json` file, but is overwritten by the environment variable, `DB_PASSWORD`, if it is set.
+
+# Generate a base64 encoded secret.
+./cliapp generate
+
+# Add the encoded secret above to the `JWT.Secret` section of the config.
+```
 
-Copy `config.json` to `src/app/webapi/cmd/webapi/config.json` and edit the **Database** section so the connection information matches your MySQL instance. Also add a base64 encoded `JWT.Secret` to the config. You can generate it using the command line app in the repo - run these commands:
-- `cd src/app/webapi/cmd/cliapp`
-- `go run cliapp.go generate`
+Now you can start the API.
 
-The database password is read from the `config.json` first, but is overwritten by the environment variable, `DB_PASSWORD`, if it is set.
+```bash
+# CD to the webapi app folder.
+cd src/app/webapi/cmd/webapi
+
+# Build the app.
+go build
+
+# Run the app.
+./webapi
+
+# Open your browser to this URL to see the **welcome** message and status **OK**: http://localhost/v1
+```
 
-Build and run from the root directory. Open your REST client to: http://localhost/v1. You should see the **welcome** message and status **OK**.
+To interact with the API, open your favorite REST client.
 
 You'll need to authenticate with at http://localhost/v1/auth before you can use any of the user endpoints. Once you have a token, add it to the request header with a name of `Authorization` and with a value of `Bearer {TOKEN HERE}`. To create a user, send a POST request to http://localhost/v1/user with the following fields: first_name, last_name, email, and password.
 

migration/mysql.sql → migration/mysql-v0.sql

@@ -1,27 +1,5 @@
-/* *****************************************************************************
-// Setup the preferences
-// ****************************************************************************/
-SET NAMES utf8 COLLATE 'utf8_unicode_ci';
-SET foreign_key_checks = 1;
-SET time_zone = '+00:00';
+--changeset josephspurrier:1
 SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO';
-SET CHARACTER SET utf8;
-
-/* *****************************************************************************
-// Remove old database
-// ****************************************************************************/
-DROP DATABASE IF EXISTS webapi;
-
-/* *****************************************************************************
-// Create new database
-// ****************************************************************************/
-CREATE DATABASE webapi DEFAULT CHARSET = utf8 COLLATE = utf8_unicode_ci;
-USE webapi;
-
-/* *****************************************************************************
-// Create the tables
-// ****************************************************************************/
-
 CREATE TABLE user_status (
     id TINYINT(1) UNSIGNED NOT NULL AUTO_INCREMENT,
 
@@ -33,7 +11,16 @@ CREATE TABLE user_status (
 
     PRIMARY KEY (id)
 );
+--rollback DROP TABLE user_status;
 
+--changeset josephspurrier:2
+INSERT INTO `user_status` (`id`, `status`, `created_at`, `updated_at`, `deleted`) VALUES
+(1, 'active',   CURRENT_TIMESTAMP,  CURRENT_TIMESTAMP,  0),
+(2, 'inactive', CURRENT_TIMESTAMP,  CURRENT_TIMESTAMP,  0);
+--rollback TRUNCATE TABLE user_status;
+
+--changeset josephspurrier:3
+SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO';
 CREATE TABLE user (
     id VARCHAR(36) NOT NULL,
 
@@ -53,7 +40,4 @@ CREATE TABLE user (
 
     PRIMARY KEY (id)
 );
-
-INSERT INTO `user_status` (`id`, `status`, `created_at`, `updated_at`, `deleted`) VALUES
-(1, 'active',   CURRENT_TIMESTAMP,  CURRENT_TIMESTAMP,  0),
-(2, 'inactive', CURRENT_TIMESTAMP,  CURRENT_TIMESTAMP,  0);
+--rollback DROP TABLE user;

src/app/webapi/cmd/cliapp/cliapp_test.go

@@ -59,7 +59,6 @@ func unsetEnv() {
 }
 func TestMigrationAll(t *testing.T) {
 	setEnv()
-	defer unsetEnv()
 
 	testutil.ResetDatabase()
 	db := testutil.ConnectDatabase(true)
@@ -98,7 +97,6 @@ func TestMigrationReset(t *testing.T) {
 	TestMigrationAll(t)
 
 	setEnv()
-	defer unsetEnv()
 
 	db := testutil.ConnectDatabase(true)
 
@@ -134,7 +132,6 @@ func TestMigrationReset(t *testing.T) {
 
 func TestMigrationUp(t *testing.T) {
 	setEnv()
-	defer unsetEnv()
 
 	testutil.ResetDatabase()
 	db := testutil.ConnectDatabase(true)
@@ -174,7 +171,6 @@ func TestMigrationDown(t *testing.T) {
 	TestMigrationUp(t)
 
 	setEnv()
-	defer unsetEnv()
 
 	db := testutil.ConnectDatabase(true)
 

src/app/webapi/internal/basemigrate/basemigrate.go

@@ -2,8 +2,6 @@ package basemigrate
 
 import (
 	"errors"
-	"fmt"
-	"os"
 )
 
 const (
@@ -24,6 +22,7 @@ const (
 	appVersion       = "1.0"
 	elementChangeset = "--changeset "
 	elementRollback  = "--rollback "
+	elementInclude   = "--include "
 )
 
 var (
@@ -32,41 +31,3 @@ var (
 	// ErrInvalidFormat is when a changeset is not found.
 	ErrInvalidFormat = errors.New("invalid changeset format")
 )
-
-// ParseFileArray will parse a file into changesets.
-func ParseFileArray(filename string) ([]*Changeset, error) {
-	f, err := os.Open(filename)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
-	return parseToOrderedArray(f, filename)
-}
-
-// ParseFileMap will parse a file into a map.
-func ParseFileMap(filename string) (map[string]Changeset, error) {
-	f, err := os.Open(filename)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
-	arr, err := parseToOrderedArray(f, filename)
-	if err != nil {
-		return nil, err
-	}
-
-	m := make(map[string]Changeset)
-
-	for _, cs := range arr {
-		id := fmt.Sprintf("%v:%v", cs.author, cs.id)
-		if _, found := m[id]; found {
-			return nil, errors.New("Duplicate entry found: " + id)
-		}
-
-		m[id] = *cs
-	}
-
-	return m, nil
-}

src/app/webapi/internal/basemigrate/basemigrate_test.go

@@ -1,7 +1,6 @@
 package basemigrate_test
 
 import (
-	"os"
 	"testing"
 
 	"app/webapi/internal/basemigrate"
@@ -10,27 +9,7 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func setEnv() {
-	os.Setenv("DB_HOSTNAME", "127.0.0.1")
-	os.Setenv("DB_PORT", "3306")
-	os.Setenv("DB_USERNAME", "root")
-	os.Setenv("DB_PASSWORD", "")
-	os.Setenv("DB_DATABASE", "webapitest")
-	os.Setenv("DB_PARAMETER", "parseTime=true&allowNativePasswords=true")
-}
-
-func unsetEnv() {
-	os.Unsetenv("DB_HOSTNAME")
-	os.Unsetenv("DB_PORT")
-	os.Unsetenv("DB_USERNAME")
-	os.Unsetenv("DB_PASSWORD")
-	os.Unsetenv("DB_DATABASE")
-	os.Unsetenv("DB_PARAMETER")
-}
 func TestMigration(t *testing.T) {
-	setEnv()
-	defer unsetEnv()
-
 	testutil.ResetDatabase()
 	db := testutil.ConnectDatabase(true)
 
@@ -81,9 +60,6 @@ func TestMigration(t *testing.T) {
 }
 
 func TestMigrationFailDuplicate(t *testing.T) {
-	setEnv()
-	defer unsetEnv()
-
 	testutil.ResetDatabase()
 	db := testutil.ConnectDatabase(true)
 
@@ -94,17 +70,56 @@ func TestMigrationFailDuplicate(t *testing.T) {
 	err = db.Get(&rows, `SELECT count(*) from databasechangelog`)
 	assert.Nil(t, err)
 	assert.Equal(t, 2, rows)
+
+	testutil.ResetDatabase()
 }
 
-func TestParse(t *testing.T) {
-	setEnv()
-	defer unsetEnv()
+func TestInclude(t *testing.T) {
+	testutil.ResetDatabase()
+	db := testutil.ConnectDatabase(true)
+
+	// Run migration.
+	err := basemigrate.Migrate("testdata/parent.sql", 0, false)
+	assert.Nil(t, err)
+
+	// Count the records.
+	rows := 0
+	err = db.Get(&rows, `SELECT count(*) from databasechangelog`)
+	assert.Nil(t, err)
+	assert.Equal(t, 3, rows)
+
+	// Run migration again.
+	err = basemigrate.Migrate("testdata/parent.sql", 0, false)
+	assert.Nil(t, err)
+
+	// Remove all migrations.
+	err = basemigrate.Reset("testdata/parent.sql", 0, false)
+	assert.Nil(t, err)
+
+	rows = 0
+	err = db.Get(&rows, `SELECT count(*) from databasechangelog`)
+	assert.Nil(t, err)
+	assert.Equal(t, 0, rows)
+
+	// Remove all migrations again.
+	err = basemigrate.Reset("testdata/parent.sql", 0, false)
+	assert.Nil(t, err)
+
+	// Run 2 migrations.
+	err = basemigrate.Migrate("testdata/parent.sql", 2, false)
+	assert.Nil(t, err)
 
-	arr, err := basemigrate.ParseFileArray("testdata/success.sql")
+	rows = 0
+	err = db.Get(&rows, `SELECT count(*) from databasechangelog`)
 	assert.Nil(t, err)
-	assert.Equal(t, 3, len(arr))
+	assert.Equal(t, 2, rows)
 
-	m, err := basemigrate.ParseFileMap("testdata/success.sql")
+	// Remove 1 migration.
+	err = basemigrate.Reset("testdata/parent.sql", 1, false)
 	assert.Nil(t, err)
-	assert.Equal(t, 3, len(m))
+
+	rows = 0
+	err = db.Get(&rows, `SELECT count(*) from databasechangelog`)
+	assert.Nil(t, err)
+	assert.Equal(t, 1, rows)
 }

src/app/webapi/internal/basemigrate/helper.go

@@ -32,16 +32,3 @@ func md5sum(s string) string {
 	_, _ = io.Copy(h, r)
 	return fmt.Sprintf("%x", h.Sum(nil))
 }
-
-// Debug will return the SQL file.
-func Debug(arr []*Changeset) {
-	for _, cs := range arr {
-		fmt.Printf("%v%v:%v\n", elementChangeset, cs.author, cs.id)
-		fmt.Println(cs.Changes())
-		fmt.Printf("%v%v\n", elementRollback, cs.Rollbacks())
-		fmt.Println("--md5", cs.Checksum())
-		break
-	}
-
-	fmt.Println("Total:", len(arr))
-}

src/app/webapi/internal/basemigrate/migrate.go

@@ -18,7 +18,7 @@ func Migrate(filename string, max int, verbose bool) (err error) {
 	_, err = db.Exec(sqlChangelog)
 
 	// Get the changesets.
-	arr, err := ParseFileArray(filename)
+	arr, err := parseFileToArray(filename)
 	if err != nil {
 		return err
 	}
@@ -35,7 +35,8 @@ func Migrate(filename string, max int, verbose bool) (err error) {
 		err = db.Get(&checksum, `SELECT md5sum
 			FROM databasechangelog
 			WHERE id = ?
-			AND author = ?`, cs.id, cs.author)
+			AND author = ?
+			AND filename = ?`, cs.id, cs.author, cs.filename)
 		if err == nil {
 			// Determine if the checksums match.
 			if checksum != newChecksum {