feat: support declarative schema with db diff

internal/db/diff/diff.go

@@ -5,7 +5,9 @@ import (
 	_ "embed"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
@@ -24,6 +26,7 @@ import (
 	"github.com/supabase/cli/internal/gen/keys"
 	"github.com/supabase/cli/internal/migration/apply"
 	"github.com/supabase/cli/internal/migration/list"
+	"github.com/supabase/cli/internal/migration/repair"
 	"github.com/supabase/cli/internal/utils"
 	"github.com/supabase/cli/internal/utils/parser"
 )
@@ -35,6 +38,19 @@ func Run(ctx context.Context, schema []string, file string, config pgconn.Config
 	if err := utils.LoadConfigFS(fsys); err != nil {
 		return err
 	}
+	if utils.IsLocalDatabase(config) {
+		if container, err := createShadowIfNotExists(ctx, fsys); err != nil {
+			return err
+		} else if len(container) > 0 {
+			defer utils.DockerRemove(container)
+			if !start.WaitForHealthyService(ctx, container, start.HealthTimeout) {
+				return errors.New(start.ErrDatabase)
+			}
+			if err := migrateBaseDatabase(ctx, container, fsys, options...); err != nil {
+				return err
+			}
+		}
+	}
 	// 1. Load all user defined schemas
 	if len(schema) == 0 {
 		schema, err = loadSchema(ctx, config, options...)
@@ -60,6 +76,35 @@ func Run(ctx context.Context, schema []string, file string, config pgconn.Config
 	return nil
 }
 
+func createShadowIfNotExists(ctx context.Context, fsys afero.Fs) (string, error) {
+	if exists, err := afero.DirExists(fsys, utils.SchemasDir); err != nil {
+		return "", errors.Errorf("failed to check schemas: %w", err)
+	} else if !exists {
+		return "", nil
+	}
+	if err := utils.AssertSupabaseDbIsRunning(); !errors.Is(err, utils.ErrNotRunning) {
+		return "", err
+	}
+	fmt.Fprintf(os.Stderr, "Creating local database from %s...\n", utils.Bold(utils.SchemasDir))
+	return CreateShadowDatabase(ctx, utils.Config.Db.Port)
+}
+
+func loadDeclaredSchemas(fsys afero.Fs) ([]string, error) {
+	var declared []string
+	if err := afero.Walk(fsys, utils.SchemasDir, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.Mode().IsRegular() && filepath.Ext(info.Name()) == ".sql" {
+			declared = append(declared, path)
+		}
+		return nil
+	}); err != nil {
+		return nil, errors.Errorf("failed to walk dir: %w", err)
+	}
+	return declared, nil
+}
+
 // https://github.com/djrobstep/migra/blob/master/migra/statements.py#L6
 var dropStatementPattern = regexp.MustCompile(`(?i)drop\s+`)
 
@@ -87,9 +132,9 @@ func loadSchema(ctx context.Context, config pgconn.Config, options ...func(*pgx.
 	return reset.LoadUserSchemas(ctx, conn)
 }
 
-func CreateShadowDatabase(ctx context.Context) (string, error) {
+func CreateShadowDatabase(ctx context.Context, port uint16) (string, error) {
 	config := start.NewContainerConfig()
-	hostPort := strconv.FormatUint(uint64(utils.Config.Db.ShadowPort), 10)
+	hostPort := strconv.FormatUint(uint64(port), 10)
 	hostConfig := container.HostConfig{
 		PortBindings: nat.PortMap{"5432/tcp": []nat.PortBinding{{HostPort: hostPort}}},
 		AutoRemove:   true,
@@ -105,7 +150,7 @@ func CreateShadowDatabase(ctx context.Context) (string, error) {
 func ConnectShadowDatabase(ctx context.Context, timeout time.Duration, options ...func(*pgx.ConnConfig)) (conn *pgx.Conn, err error) {
 	// Retry until connected, cancelled, or timeout
 	policy := backoff.WithMaxRetries(backoff.NewConstantBackOff(time.Second), uint64(timeout.Seconds()))
-	config := pgconn.Config{Port: uint16(utils.Config.Db.ShadowPort)}
+	config := pgconn.Config{Port: utils.Config.Db.ShadowPort}
 	connect := func() (*pgx.Conn, error) {
 		return utils.ConnectLocalPostgres(ctx, config, options...)
 	}
@@ -128,9 +173,35 @@ func MigrateShadowDatabase(ctx context.Context, container string, fsys afero.Fs,
 	return apply.MigrateUp(ctx, conn, migrations, fsys)
 }
 
+func migrateBaseDatabase(ctx context.Context, container string, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {
+	migrations, err := loadDeclaredSchemas(fsys)
+	if err != nil {
+		return err
+	}
+	conn, err := utils.ConnectLocalPostgres(ctx, pgconn.Config{}, options...)
+	if err != nil {
+		return err
+	}
+	defer conn.Close(context.Background())
+	if err := start.SetupDatabase(ctx, conn, container[:12], os.Stderr, fsys); err != nil {
+		return err
+	}
+	for _, path := range migrations {
+		fmt.Fprintln(os.Stderr, "Applying schema "+utils.Bold(path)+"...")
+		migration, err := repair.NewMigrationFromFile(path, fsys)
+		if err != nil {
+			return err
+		}
+		if err := migration.ExecBatch(ctx, conn); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w io.Writer, fsys afero.Fs, differ func(context.Context, string, string, []string) (string, error), options ...func(*pgx.ConnConfig)) (string, error) {
 	fmt.Fprintln(w, "Creating shadow database...")
-	shadow, err := CreateShadowDatabase(ctx)
+	shadow, err := CreateShadowDatabase(ctx, utils.Config.Db.ShadowPort)
 	if err != nil {
 		return "", err
 	}
@@ -144,7 +215,7 @@ func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w
 	fmt.Fprintln(w, "Diffing schemas:", strings.Join(schema, ","))
 	source := utils.ToPostgresURL(pgconn.Config{
 		Host:     utils.Config.Hostname,
-		Port:     uint16(utils.Config.Db.ShadowPort),
+		Port:     utils.Config.Db.ShadowPort,
 		User:     "postgres",
 		Password: utils.Config.Db.Password,
 		Database: "postgres",

internal/db/diff/pgadmin.go

@@ -58,7 +58,7 @@ func run(p utils.Program, ctx context.Context, schema []string, config pgconn.Co
 	p.Send(utils.StatusMsg("Creating shadow database..."))
 
 	// 1. Create shadow db and run migrations
-	shadow, err := CreateShadowDatabase(ctx)
+	shadow, err := CreateShadowDatabase(ctx, utils.Config.Db.ShadowPort)
 	if err != nil {
 		return err
 	}

internal/db/reset/reset_test.go

@@ -29,7 +29,7 @@ func TestResetCommand(t *testing.T) {
 
 	var dbConfig = pgconn.Config{
 		Host:     utils.Config.Hostname,
-		Port:     uint16(utils.Config.Db.Port),
+		Port:     utils.Config.Db.Port,
 		User:     "admin",
 		Password: "password",
 		Database: "postgres",

internal/functions/new/new.go

@@ -51,7 +51,7 @@ func Run(ctx context.Context, slug string, fsys afero.Fs) error {
 			utils.CmdSuggestion = utils.SuggestDebugFlag
 		}
 		config := indexConfig{
-			Port:  uint16(utils.Config.Api.Port),
+			Port:  utils.Config.Api.Port,
 			Slug:  slug,
 			Token: utils.Config.Auth.AnonKey,
 		}

internal/gen/types/typescript/typescript_test.go

@@ -25,7 +25,7 @@ func TestGenLocalCommand(t *testing.T) {
 
 	dbConfig := pgconn.Config{
 		Host:     utils.Config.Hostname,
-		Port:     uint16(utils.Config.Db.Port),
+		Port:     utils.Config.Db.Port,
 		User:     "admin",
 		Password: "password",
 	}

internal/migration/squash/squash.go

@@ -80,7 +80,7 @@ func squashToVersion(ctx context.Context, version string, fsys afero.Fs, options
 
 func squashMigrations(ctx context.Context, migrations []string, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {
 	// 1. Start shadow database
-	shadow, err := diff.CreateShadowDatabase(ctx)
+	shadow, err := diff.CreateShadowDatabase(ctx, utils.Config.Db.ShadowPort)
 	if err != nil {
 		return err
 	}
@@ -100,7 +100,7 @@ func squashMigrations(ctx context.Context, migrations []string, fsys afero.Fs, o
 	schemas := []string{"auth", "storage"}
 	config := pgconn.Config{
 		Host:     utils.Config.Hostname,
-		Port:     uint16(utils.Config.Db.ShadowPort),
+		Port:     utils.Config.Db.ShadowPort,
 		User:     "postgres",
 		Password: utils.Config.Db.Password,
 		Database: "postgres",

internal/start/start.go

@@ -103,7 +103,7 @@ type kongConfig struct {
 	EdgeRuntimeId string
 	LogflareId    string
 	ApiHost       string
-	ApiPort       uint
+	ApiPort       uint16
 }
 
 var (

internal/utils/config.go

@@ -267,16 +267,16 @@ type (
 	api struct {
 		Enabled         bool     `toml:"enabled"`
 		Image           string   `toml:"-"`
-		Port            uint     `toml:"port"`
+		Port            uint16   `toml:"port"`
 		Schemas         []string `toml:"schemas"`
 		ExtraSearchPath []string `toml:"extra_search_path"`
 		MaxRows         uint     `toml:"max_rows"`
 	}
 
 	db struct {
 		Image        string `toml:"-"`
-		Port         uint   `toml:"port"`
-		ShadowPort   uint   `toml:"shadow_port"`
+		Port         uint16 `toml:"port"`
+		ShadowPort   uint16 `toml:"shadow_port"`
 		MajorVersion uint   `toml:"major_version"`
 		Password     string `toml:"-"`
 		RootKey      string `toml:"-" mapstructure:"root_key"`
@@ -303,16 +303,16 @@ type (
 
 	studio struct {
 		Enabled      bool   `toml:"enabled"`
-		Port         uint   `toml:"port"`
+		Port         uint16 `toml:"port"`
 		ApiUrl       string `toml:"api_url"`
 		OpenaiApiKey string `toml:"openai_api_key"`
 	}
 
 	inbucket struct {
-		Enabled  bool `toml:"enabled"`
-		Port     uint `toml:"port"`
-		SmtpPort uint `toml:"smtp_port"`
-		Pop3Port uint `toml:"pop3_port"`
+		Enabled  bool   `toml:"enabled"`
+		Port     uint16 `toml:"port"`
+		SmtpPort uint16 `toml:"smtp_port"`
+		Pop3Port uint16 `toml:"pop3_port"`
 	}
 
 	storage struct {

internal/utils/connect.go

@@ -98,7 +98,7 @@ func ConnectLocalPostgres(ctx context.Context, config pgconn.Config, options ...
 		config.Host = Config.Hostname
 	}
 	if config.Port == 0 {
-		config.Port = uint16(Config.Db.Port)
+		config.Port = Config.Db.Port
 	}
 	if len(config.User) == 0 {
 		config.User = "postgres"
@@ -155,5 +155,5 @@ func ConnectByConfig(ctx context.Context, config pgconn.Config, options ...func(
 }
 
 func IsLocalDatabase(config pgconn.Config) bool {
-	return config.Host == Config.Hostname && config.Port == uint16(Config.Db.Port)
+	return config.Host == Config.Hostname && config.Port == Config.Db.Port
 }

internal/utils/flags/db_url.go

@@ -63,7 +63,7 @@ func ParseDatabaseConfig(flagSet *pflag.FlagSet, fsys afero.Fs) error {
 		}
 		// Ignore other PG settings
 		DbConfig.Host = utils.Config.Hostname
-		DbConfig.Port = uint16(utils.Config.Db.Port)
+		DbConfig.Port = utils.Config.Db.Port
 		DbConfig.User = "postgres"
 		DbConfig.Password = utils.Config.Db.Password
 		DbConfig.Database = "postgres"

internal/utils/misc.go

@@ -188,6 +188,7 @@ var (
 	RestVersionPath       = filepath.Join(TempDir, "rest-version")
 	StorageVersionPath    = filepath.Join(TempDir, "storage-version")
 	CurrBranchPath        = filepath.Join(SupabaseDirPath, ".branches", "_current_branch")
+	SchemasDir            = filepath.Join(SupabaseDirPath, "schemas")
 	MigrationsDir         = filepath.Join(SupabaseDirPath, "migrations")
 	FunctionsDir          = filepath.Join(SupabaseDirPath, "functions")
 	FallbackImportMapPath = filepath.Join(FunctionsDir, "import_map.json")