feat: Add database fixtures for testing migrations (#4858)

coderd/database/migrations/create_fixture.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+# Naming the fixture is optional, if missing, the name of the latest
+# migration will be used.
+#
+# Usage:
+# ./create_fixture
+# ./create_fixture name of fixture
+# ./create_fixture "name of fixture"
+# ./create_fixture name_of_fixture
+
+set -euo pipefail
+
+SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
+(
+	cd "$SCRIPT_DIR"
+
+	latest_migration=$(basename "$(find . -maxdepth 1 -name "*.up.sql" | sort -n | tail -n 1)")
+	if [[ -n "${*}" ]]; then
+		name=$*
+		name=${name// /_}
+		num=${latest_migration%%_*}
+		latest_migration="${num}_${name}.up.sql"
+	fi
+
+	filename="$(pwd)/testdata/fixtures/$latest_migration"
+	touch "$filename"
+	echo "$filename"
+	echo "Edit fixture and commit it."
+)

coderd/database/migrations/migrate.go

@@ -5,6 +5,7 @@ import (
 	"database/sql"
 	"embed"
 	"errors"
+	"io/fs"
 	"os"
 
 	"github.com/golang-migrate/migrate/v4"
@@ -160,3 +161,52 @@ func CheckLatestVersion(sourceDriver source.Driver, currentVersion uint) error {
 	}
 	return nil
 }
+
+// Stepper returns a function that runs SQL migrations one step at a time.
+//
+// Stepper cannot be closed pre-emptively, it must be run to completion
+// (or until an error is encountered).
+func Stepper(db *sql.DB) (next func() (version uint, more bool, err error), err error) {
+	_, m, err := setup(db)
+	if err != nil {
+		return nil, xerrors.Errorf("migrate setup: %w", err)
+	}
+
+	return func() (version uint, more bool, err error) {
+		defer func() {
+			if !more {
+				srcErr, dbErr := m.Close()
+				if err != nil {
+					return
+				}
+				if dbErr != nil {
+					err = dbErr
+					return
+				}
+				err = srcErr
+			}
+		}()
+
+		err = m.Steps(1)
+		if err != nil {
+			switch {
+			case errors.Is(err, migrate.ErrNoChange):
+				// It's OK if no changes happened!
+				return 0, false, nil
+			case errors.Is(err, fs.ErrNotExist):
+				// This error is encountered at the of Steps when
+				// reading from embed.FS.
+				return 0, false, nil
+			}
+
+			return 0, false, xerrors.Errorf("Step: %w", err)
+		}
+
+		v, _, err := m.Version()
+		if err != nil {
+			return 0, false, err
+		}
+
+		return v, true, nil
+	}, nil
+}

coderd/database/migrations/migrate_test.go

@@ -3,17 +3,27 @@
 package migrations_test
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
 	"testing"
 
+	"github.com/golang-migrate/migrate/v4"
+	migratepostgres "github.com/golang-migrate/migrate/v4/database/postgres"
 	"github.com/golang-migrate/migrate/v4/source"
+	"github.com/golang-migrate/migrate/v4/source/iofs"
 	"github.com/golang-migrate/migrate/v4/source/stub"
+	"github.com/lib/pq"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
+	"golang.org/x/exp/slices"
 
 	"github.com/coder/coder/coderd/database/migrations"
 	"github.com/coder/coder/coderd/database/postgres"
+	"github.com/coder/coder/testutil"
 )
 
 func TestMain(m *testing.M) {
@@ -129,3 +139,196 @@ func TestCheckLatestVersion(t *testing.T) {
 		})
 	}
 }
+
+func setupMigrate(t *testing.T, db *sql.DB, name, path string) (source.Driver, *migrate.Migrate) {
+	t.Helper()
+
+	ctx := context.Background()
+
+	conn, err := db.Conn(ctx)
+	require.NoError(t, err)
+
+	dbDriver, err := migratepostgres.WithConnection(ctx, conn, &migratepostgres.Config{
+		MigrationsTable: "test_migrate_" + name,
+	})
+	require.NoError(t, err)
+
+	dirFS := os.DirFS(path)
+	d, err := iofs.New(dirFS, ".")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		d.Close()
+	})
+
+	m, err := migrate.NewWithInstance(name, d, "", dbDriver)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		m.Close()
+	})
+
+	return d, m
+}
+
+type tableStats struct {
+	mu sync.Mutex
+	s  map[string]int
+}
+
+func (s *tableStats) Add(table string, n int) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	s.s[table] = s.s[table] + n
+}
+
+func (s *tableStats) Empty() []string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	var m []string
+	for table, n := range s.s {
+		if n == 0 {
+			m = append(m, table)
+		}
+	}
+	return m
+}
+
+func TestMigrateUpWithFixtures(t *testing.T) {
+	t.Parallel()
+
+	if testing.Short() {
+		t.Skip()
+		return
+	}
+
+	type testCase struct {
+		name string
+		path string
+
+		// For determining if test case table stats
+		// are used to determine test coverage.
+		useStats bool
+	}
+	tests := []testCase{
+		{
+			name:     "fixtures",
+			path:     filepath.Join("testdata", "fixtures"),
+			useStats: true,
+		},
+		// More test cases added via glob below.
+	}
+
+	// Folders in testdata/full_dumps represent fixtures for a full
+	// deployment of Coder.
+	matches, err := filepath.Glob(filepath.Join("testdata", "full_dumps", "*"))
+	require.NoError(t, err)
+	for _, match := range matches {
+		tests = append(tests, testCase{
+			name:     filepath.Base(match),
+			path:     match,
+			useStats: true,
+		})
+	}
+
+	// These tables are allowed to have zero rows for now,
+	// but we should eventually add fixtures for them.
+	ignoredTablesForStats := []string{
+		"audit_logs",
+		"git_auth_links",
+		"group_members",
+		"licenses",
+		"replicas",
+	}
+	s := &tableStats{s: make(map[string]int)}
+
+	// This will run after all subtests have run and fail the test if
+	// new tables have been added without covering them with fixtures.
+	t.Cleanup(func() {
+		emptyTables := s.Empty()
+		slices.Sort(emptyTables)
+		for _, table := range ignoredTablesForStats {
+			i := slices.Index(emptyTables, table)
+			if i >= 0 {
+				emptyTables = slices.Delete(emptyTables, i, i+1)
+			}
+		}
+		if len(emptyTables) > 0 {
+			t.Logf("The following tables have zero rows, consider adding fixtures for them or create a full database dump:")
+			t.Errorf("tables have zero rows: %v", emptyTables)
+			t.Logf("See https://github.com/coder/coder/blob/main/docs/CONTRIBUTING.md#database-fixtures-for-testing-migrations for more information")
+		}
+	})
+
+	for _, tt := range tests {
+		tt := tt
+
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			db := testSQLDB(t)
+
+			ctx, _ := testutil.Context(t)
+
+			// Prepare database for stepping up.
+			err := migrations.Down(db)
+			require.NoError(t, err)
+
+			// Initialize migrations for fixtures.
+			fDriver, fMigrate := setupMigrate(t, db, tt.name, tt.path)
+
+			nextStep, err := migrations.Stepper(db)
+			require.NoError(t, err)
+
+			var fixtureVer uint
+			nextFixtureVer, err := fDriver.First()
+			require.NoError(t, err)
+
+			for {
+				version, more, err := nextStep()
+				require.NoError(t, err)
+
+				if !more {
+					// We reached the end of the migrations.
+					break
+				}
+
+				if nextFixtureVer == version {
+					err = fMigrate.Steps(1)
+					require.NoError(t, err)
+					fixtureVer = version
+
+					nv, _ := fDriver.Next(nextFixtureVer)
+					if nv > 0 {
+						nextFixtureVer = nv
+					}
+				}
+
+				t.Logf("migrated to version %d, fixture version %d", version, fixtureVer)
+			}
+
+			// Gather number of rows for all existing tables
+			// at the end of the migrations and fixtures.
+			var tables pq.StringArray
+			err = db.QueryRowContext(ctx, `
+				SELECT array_agg(tablename)
+				FROM pg_catalog.pg_tables
+				WHERE
+					schemaname != 'information_schema'
+					AND schemaname != 'pg_catalog'
+					AND tablename NOT LIKE 'test_migrate_%'
+			`).Scan(&tables)
+			require.NoError(t, err)
+
+			for _, table := range tables {
+				var count int
+				err = db.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+table).Scan(&count)
+				require.NoError(t, err)
+
+				if tt.useStats {
+					s.Add(table, count)
+				}
+			}
+		})
+	}
+}