feat(experimental): add internal migrate package and SessionLocker in…

…terface (#606)

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/jackc/pgx/v5 v5.4.3
 	github.com/microsoft/go-mssqldb v1.6.0
 	github.com/ory/dockertest/v3 v3.10.0
+	github.com/sethvargo/go-retry v0.2.4
 	github.com/vertica/vertica-sql-go v1.3.3
 	github.com/ziutek/mymysql v1.5.4
 	go.uber.org/multierr v1.11.0

go.sum

@@ -127,6 +127,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qq
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
 github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
+github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec=
+github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw=
 github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=

internal/migrate/doc.go

@@ -0,0 +1,9 @@
+// Package migrate defines a Migration struct and implements the migration logic for executing Go
+// and SQL migrations.
+//
+//   - For Go migrations, only *sql.Tx and *sql.DB are supported. *sql.Conn is not supported.
+//   - For SQL migrations, all three are supported.
+//
+// Lastly, SQL migrations are lazily parsed. This means that the SQL migration is parsed the first
+// time it is executed.
+package migrate

internal/migrate/migration.go

@@ -0,0 +1,166 @@
+package migrate
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+
+	"github.com/pressly/goose/v3/internal/sqlextended"
+)
+
+type Migration struct {
+	// Fullpath is the full path to the migration file.
+	//
+	// Example: /path/to/migrations/123_create_users_table.go
+	Fullpath string
+	// Version is the version of the migration.
+	Version int64
+	// Type is the type of migration.
+	Type MigrationType
+	// A migration is either a Go migration or a SQL migration, but never both.
+	//
+	// Note, the SQLParsed field is used to determine if the SQL migration has been parsed. This is
+	// an optimization to avoid parsing the SQL migration if it is never required. Also, the
+	// majority of the time migrations are incremental, so it is likely that the user will only want
+	// to run the last few migrations, and there is no need to parse ALL prior migrations.
+	//
+	// Exactly one of these fields will be set:
+	Go *Go
+	// -- or --
+	SQLParsed bool
+	SQL       *SQL
+}
+
+type MigrationType int
+
+const (
+	TypeGo MigrationType = iota + 1
+	TypeSQL
+)
+
+func (t MigrationType) String() string {
+	switch t {
+	case TypeGo:
+		return "go"
+	case TypeSQL:
+		return "sql"
+	default:
+		// This should never happen.
+		return "unknown"
+	}
+}
+
+func (m *Migration) UseTx() bool {
+	switch m.Type {
+	case TypeGo:
+		return m.Go.UseTx
+	case TypeSQL:
+		return m.SQL.UseTx
+	default:
+		// This should never happen.
+		panic("unknown migration type: use tx")
+	}
+}
+
+func (m *Migration) IsEmpty(direction bool) bool {
+	switch m.Type {
+	case TypeGo:
+		return m.Go.IsEmpty(direction)
+	case TypeSQL:
+		return m.SQL.IsEmpty(direction)
+	default:
+		// This should never happen.
+		panic("unknown migration type: is empty")
+	}
+}
+
+func (m *Migration) GetSQLStatements(direction bool) ([]string, error) {
+	if m.Type != TypeSQL {
+		return nil, fmt.Errorf("expected sql migration, got %s: no sql statements", m.Type)
+	}
+	if m.SQL == nil {
+		return nil, errors.New("sql migration has not been initialized")
+	}
+	if !m.SQLParsed {
+		return nil, errors.New("sql migration has not been parsed")
+	}
+	if direction {
+		return m.SQL.UpStatements, nil
+	}
+	return m.SQL.DownStatements, nil
+}
+
+type Go struct {
+	// We used an explicit bool instead of relying on a pointer because registered funcs may be nil.
+	// These are still valid Go and versioned migrations, but they are just empty.
+	//
+	// For example: goose.AddMigration(nil, nil)
+	UseTx bool
+
+	// Only one of these func pairs will be set:
+	UpFn, DownFn func(context.Context, *sql.Tx) error
+	// -- or --
+	UpFnNoTx, DownFnNoTx func(context.Context, *sql.DB) error
+}
+
+func (g *Go) IsEmpty(direction bool) bool {
+	if direction {
+		return g.UpFn == nil && g.UpFnNoTx == nil
+	}
+	return g.DownFn == nil && g.DownFnNoTx == nil
+}
+
+func (g *Go) run(ctx context.Context, tx *sql.Tx, direction bool) error {
+	var fn func(context.Context, *sql.Tx) error
+	if direction {
+		fn = g.UpFn
+	} else {
+		fn = g.DownFn
+	}
+	if fn != nil {
+		return fn(ctx, tx)
+	}
+	return nil
+}
+
+func (g *Go) runNoTx(ctx context.Context, db *sql.DB, direction bool) error {
+	var fn func(context.Context, *sql.DB) error
+	if direction {
+		fn = g.UpFnNoTx
+	} else {
+		fn = g.DownFnNoTx
+	}
+	if fn != nil {
+		return fn(ctx, db)
+	}
+	return nil
+}
+
+type SQL struct {
+	UseTx          bool
+	UpStatements   []string
+	DownStatements []string
+}
+
+func (s *SQL) IsEmpty(direction bool) bool {
+	if direction {
+		return len(s.UpStatements) == 0
+	}
+	return len(s.DownStatements) == 0
+}
+
+func (s *SQL) run(ctx context.Context, db sqlextended.DBTxConn, direction bool) error {
+	var statements []string
+	if direction {
+		statements = s.UpStatements
+	} else {
+		statements = s.DownStatements
+	}
+	for _, stmt := range statements {
+		if _, err := db.ExecContext(ctx, stmt); err != nil {
+			return err
+		}
+	}
+	return nil
+}

internal/migrate/parse.go

@@ -0,0 +1,75 @@
+package migrate
+
+import (
+	"bytes"
+	"io"
+	"io/fs"
+
+	"github.com/pressly/goose/v3/internal/sqlparser"
+)
+
+// ParseSQL parses all SQL migrations in BOTH directions. If a migration has already been parsed, it
+// will not be parsed again.
+//
+// Important: This function will mutate SQL migrations.
+func ParseSQL(fsys fs.FS, debug bool, migrations []*Migration) error {
+	for _, m := range migrations {
+		if m.Type == TypeSQL && !m.SQLParsed {
+			parsedSQLMigration, err := parseSQL(fsys, m.Fullpath, parseAll, debug)
+			if err != nil {
+				return err
+			}
+			m.SQLParsed = true
+			m.SQL = parsedSQLMigration
+		}
+	}
+	return nil
+}
+
+// parse is used to determine which direction to parse the SQL migration.
+type parse int
+
+const (
+	// parseAll parses all SQL statements in BOTH directions.
+	parseAll parse = iota + 1
+	// parseUp parses all SQL statements in the UP direction.
+	parseUp
+	// parseDown parses all SQL statements in the DOWN direction.
+	parseDown
+)
+
+func parseSQL(fsys fs.FS, filename string, p parse, debug bool) (*SQL, error) {
+	r, err := fsys.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	by, err := io.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+	if err := r.Close(); err != nil {
+		return nil, err
+	}
+	s := new(SQL)
+	if p == parseAll || p == parseUp {
+		s.UpStatements, s.UseTx, err = sqlparser.ParseSQLMigration(
+			bytes.NewReader(by),
+			sqlparser.DirectionUp,
+			debug,
+		)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if p == parseAll || p == parseDown {
+		s.DownStatements, s.UseTx, err = sqlparser.ParseSQLMigration(
+			bytes.NewReader(by),
+			sqlparser.DirectionDown,
+			debug,
+		)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return s, nil
+}

internal/migrate/run.go

@@ -0,0 +1,53 @@
+package migrate
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"path/filepath"
+)
+
+// Run runs the migration inside of a transaction.
+func (m *Migration) Run(ctx context.Context, tx *sql.Tx, direction bool) error {
+	switch m.Type {
+	case TypeSQL:
+		if m.SQL == nil || !m.SQLParsed {
+			return fmt.Errorf("tx: sql migration has not been parsed")
+		}
+		return m.SQL.run(ctx, tx, direction)
+	case TypeGo:
+		return m.Go.run(ctx, tx, direction)
+	}
+	// This should never happen.
+	return fmt.Errorf("tx: failed to run migration %s: neither sql or go", filepath.Base(m.Fullpath))
+}
+
+// RunNoTx runs the migration without a transaction.
+func (m *Migration) RunNoTx(ctx context.Context, db *sql.DB, direction bool) error {
+	switch m.Type {
+	case TypeSQL:
+		if m.SQL == nil || !m.SQLParsed {
+			return fmt.Errorf("db: sql migration has not been parsed")
+		}
+		return m.SQL.run(ctx, db, direction)
+	case TypeGo:
+		return m.Go.runNoTx(ctx, db, direction)
+	}
+	// This should never happen.
+	return fmt.Errorf("db: failed to run migration %s: neither sql or go", filepath.Base(m.Fullpath))
+}
+
+// RunConn runs the migration without a transaction using the provided connection.
+func (m *Migration) RunConn(ctx context.Context, conn *sql.Conn, direction bool) error {
+	switch m.Type {
+	case TypeSQL:
+		if m.SQL == nil || !m.SQLParsed {
+			return fmt.Errorf("conn: sql migration has not been parsed")
+		}
+		return m.SQL.run(ctx, conn, direction)
+	case TypeGo:
+		return fmt.Errorf("conn: go migrations are not supported with *sql.Conn")
+	}
+	// This should never happen.
+	return fmt.Errorf("conn: failed to run migration %s: neither sql or go", filepath.Base(m.Fullpath))
+}

internal/sqlextended/sqlextended.go

@@ -11,7 +11,7 @@ import (
 // There is a long outstanding issue to formalize a std lib interface, but alas... See:
 // https://github.com/golang/go/issues/14468
 type DBTxConn interface {
-	ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error)
+	ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
 	QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
 	QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
 }