Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

go/doltcore/migrate: added --drop-conflicts flag to migration, check … #4820

Merged
merged 3 commits into from
Nov 18, 2022
Merged

go/doltcore/migrate: added --drop-conflicts flag to migration, check … #4820

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
69465f0
go/doltcore/migrate: added --drop-conflicts flag to migration, check …
andy-wm-arthur Nov 17, 2022
70ba2d2
fix migration integration tests
andy-wm-arthur Nov 17, 2022
07637a5
Update integration-tests/bats/migrate.bats
andy-wm-arthur Nov 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
27 changes: 14 additions & 13 deletions go/cmd/dolt/commands/migrate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ package commands
import (
"context"

"github.com/fatih/color"
"github.com/dolthub/dolt/go/store/types"

"github.com/dolthub/dolt/go/cmd/dolt/cli"
"github.com/dolthub/dolt/go/cmd/dolt/errhand"
Expand All @@ -31,8 +31,7 @@ const (
migrationPrompt = `Run "dolt migrate" to update this database to the latest data format`
migrationMsg = "Migrating database to the latest data format"

migratePushFlag = "push"
migratePullFlag = "pull"
migrateDropConflictsFlag = "drop-conflicts"
)

var migrateDocs = cli.CommandDocumentationContent{
Expand Down Expand Up @@ -65,8 +64,7 @@ func (cmd MigrateCmd) Docs() *cli.CommandDocumentation {

func (cmd MigrateCmd) ArgParser() *argparser.ArgParser {
ap := argparser.NewArgParser()
ap.SupportsFlag(migratePushFlag, "", "Push all migrated branches to the remote")
ap.SupportsFlag(migratePullFlag, "", "Update all local tracking refs for a migrated remote")
ap.SupportsFlag(migrateDropConflictsFlag, "", "Drop any conflicts visited during the migration")
return ap
}

Expand All @@ -81,31 +79,34 @@ func (cmd MigrateCmd) Exec(ctx context.Context, commandStr string, args []string
help, usage := cli.HelpAndUsagePrinters(cli.CommandDocsForCommandString(commandStr, migrateDocs, ap))
apr := cli.ParseArgsOrDie(ap, args, help)

if apr.Contains(migratePushFlag) && apr.Contains(migratePullFlag) {
cli.PrintErrf(color.RedString("options --%s and --%s are mutually exclusive", migratePushFlag, migratePullFlag))
return 1
}

if err := MigrateDatabase(ctx, dEnv); err != nil {
dropConflicts := apr.Contains(migrateDropConflictsFlag)
if err := MigrateDatabase(ctx, dEnv, dropConflicts); err != nil {
verr := errhand.BuildDError("migration failed").AddCause(err).Build()
return HandleVErrAndExitCode(verr, usage)
}
return 0
}

// MigrateDatabase migrates the NomsBinFormat of |dEnv.DoltDB|.
func MigrateDatabase(ctx context.Context, dEnv *env.DoltEnv) error {
func MigrateDatabase(ctx context.Context, dEnv *env.DoltEnv, dropConflicts bool) error {
menv, err := migrate.NewEnvironment(ctx, dEnv)
if err != nil {
return err
}
menv.DropConflicts = dropConflicts

if curr := menv.Existing.DoltDB.Format(); types.IsFormat_DOLT(curr) {
cli.Println("database is already migrated")
return nil
}

p, err := menv.Migration.FS.Abs(".")
if err != nil {
return err
}
cli.Println("migrating database at tmp dir: ", p)

err = migrate.TraverseDAG(ctx, menv.Existing.DoltDB, menv.Migration.DoltDB)
err = migrate.TraverseDAG(ctx, menv, menv.Existing.DoltDB, menv.Migration.DoltDB)
if err != nil {
return err
}
Expand Down
5 changes: 3 additions & 2 deletions go/libraries/doltcore/migrate/environment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,9 @@ var (

// Environment is a migration environment.
type Environment struct {
Migration *env.DoltEnv
Existing *env.DoltEnv
Migration *env.DoltEnv
Existing *env.DoltEnv
DropConflicts bool
}

// NewEnvironment creates a migration Environment for |existing|.
Expand Down
2 changes: 1 addition & 1 deletion go/libraries/doltcore/migrate/integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ func runMigration(t *testing.T, ctx context.Context, preEnv *env.DoltEnv) (postE
DoltDB: ddb,
}

err = migrate.TraverseDAG(ctx, preEnv.DoltDB, postEnv.DoltDB)
err = migrate.TraverseDAG(ctx, migrate.Environment{}, preEnv.DoltDB, postEnv.DoltDB)
assert.NoError(t, err)
return
}
Expand Down
14 changes: 7 additions & 7 deletions go/libraries/doltcore/migrate/transform.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ var (
flushRef = ref.NewInternalRef("migration-flush")
)

func migrateWorkingSet(ctx context.Context, brRef ref.BranchRef, wsRef ref.WorkingSetRef, old, new *doltdb.DoltDB) error {
func migrateWorkingSet(ctx context.Context, menv Environment, brRef ref.BranchRef, wsRef ref.WorkingSetRef, old, new *doltdb.DoltDB) error {
oldWs, err := old.ResolveWorkingSet(ctx, wsRef)
if err != nil {
return err
Expand All @@ -65,12 +65,12 @@ func migrateWorkingSet(ctx context.Context, brRef ref.BranchRef, wsRef ref.Worki
return err
}

wr, err := migrateRoot(ctx, oldHeadRoot, oldWs.WorkingRoot(), newHeadRoot)
wr, err := migrateRoot(ctx, menv, oldHeadRoot, oldWs.WorkingRoot(), newHeadRoot)
if err != nil {
return err
}

sr, err := migrateRoot(ctx, oldHeadRoot, oldWs.StagedRoot(), newHeadRoot)
sr, err := migrateRoot(ctx, menv, oldHeadRoot, oldWs.StagedRoot(), newHeadRoot)
if err != nil {
return err
}
Expand All @@ -90,7 +90,7 @@ func migrateWorkingSet(ctx context.Context, brRef ref.BranchRef, wsRef ref.Worki
return new.UpdateWorkingSet(ctx, wsRef, newWs, hash.Hash{}, oldWs.Meta())
}

func migrateCommit(ctx context.Context, oldCm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error {
func migrateCommit(ctx context.Context, menv Environment, oldCm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error {
oldHash, err := oldCm.HashOf()
if err != nil {
return err
Expand Down Expand Up @@ -147,7 +147,7 @@ func migrateCommit(ctx context.Context, oldCm *doltdb.Commit, new *doltdb.DoltDB
return err
}

mRoot, err := migrateRoot(ctx, oldParentRoot, oldRoot, newParentRoot)
mRoot, err := migrateRoot(ctx, menv, oldParentRoot, oldRoot, newParentRoot)
if err != nil {
return err
}
Expand Down Expand Up @@ -265,7 +265,7 @@ func migrateCommitOptions(ctx context.Context, oldCm *doltdb.Commit, prog Progre
}, nil
}

func migrateRoot(ctx context.Context, oldParent, oldRoot, newParent *doltdb.RootValue) (*doltdb.RootValue, error) {
func migrateRoot(ctx context.Context, menv Environment, oldParent, oldRoot, newParent *doltdb.RootValue) (*doltdb.RootValue, error) {
migrated := newParent

fkc, err := oldRoot.GetForeignKeyCollection(ctx)
Expand All @@ -292,7 +292,7 @@ func migrateRoot(ctx context.Context, oldParent, oldRoot, newParent *doltdb.Root
ok, err := oldTbl.HasConflicts(ctx)
if err != nil {
return true, err
} else if ok {
} else if ok && !menv.DropConflicts {
return true, fmt.Errorf("cannot migrate table with conflicts (%s)", name)
}

Expand Down
26 changes: 13 additions & 13 deletions go/libraries/doltcore/migrate/traverse.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (
)

// TraverseDAG traverses |old|, migrating values to |new|.
func TraverseDAG(ctx context.Context, old, new *doltdb.DoltDB) (err error) {
func TraverseDAG(ctx context.Context, menv Environment, old, new *doltdb.DoltDB) (err error) {
var heads []ref.DoltRef
var prog Progress

Expand All @@ -50,7 +50,7 @@ func TraverseDAG(ctx context.Context, old, new *doltdb.DoltDB) (err error) {
}()

for i := range heads {
if err = traverseRefHistory(ctx, heads[i], old, new, prog); err != nil {
if err = traverseRefHistory(ctx, menv, heads[i], old, new, prog); err != nil {
return err
}
}
Expand All @@ -61,23 +61,23 @@ func TraverseDAG(ctx context.Context, old, new *doltdb.DoltDB) (err error) {
return nil
}

func traverseRefHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb.DoltDB, prog Progress) error {
func traverseRefHistory(ctx context.Context, menv Environment, r ref.DoltRef, old, new *doltdb.DoltDB, prog Progress) error {
switch r.GetType() {
case ref.BranchRefType:
if err := traverseBranchHistory(ctx, r, old, new, prog); err != nil {
if err := traverseBranchHistory(ctx, menv, r, old, new, prog); err != nil {
return err
}
wsRef, err := ref.WorkingSetRefForHead(r)
if err != nil {
return err
}
return migrateWorkingSet(ctx, r.(ref.BranchRef), wsRef, old, new)
return migrateWorkingSet(ctx, menv, r.(ref.BranchRef), wsRef, old, new)

case ref.TagRefType:
return traverseTagHistory(ctx, r.(ref.TagRef), old, new, prog)
return traverseTagHistory(ctx, menv, r.(ref.TagRef), old, new, prog)

case ref.RemoteRefType:
return traverseBranchHistory(ctx, r, old, new, prog)
return traverseBranchHistory(ctx, menv, r, old, new, prog)

case ref.WorkspaceRefType, ref.InternalRefType:
return nil
Expand All @@ -87,12 +87,12 @@ func traverseRefHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb.Dol
}
}

func traverseBranchHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb.DoltDB, prog Progress) error {
func traverseBranchHistory(ctx context.Context, menv Environment, r ref.DoltRef, old, new *doltdb.DoltDB, prog Progress) error {
cm, err := old.ResolveCommitRef(ctx, r)
if err != nil {
return err
}
if err = traverseCommitHistory(ctx, cm, new, prog); err != nil {
if err = traverseCommitHistory(ctx, menv, cm, new, prog); err != nil {
return err
}

Expand All @@ -108,13 +108,13 @@ func traverseBranchHistory(ctx context.Context, r ref.DoltRef, old, new *doltdb.
return new.SetHead(ctx, r, newHash)
}

func traverseTagHistory(ctx context.Context, r ref.TagRef, old, new *doltdb.DoltDB, prog Progress) error {
func traverseTagHistory(ctx context.Context, menv Environment, r ref.TagRef, old, new *doltdb.DoltDB, prog Progress) error {
t, err := old.ResolveTag(ctx, r)
if err != nil {
return err
}

if err = traverseCommitHistory(ctx, t.Commit, new, prog); err != nil {
if err = traverseCommitHistory(ctx, menv, t.Commit, new, prog); err != nil {
return err
}

Expand All @@ -133,7 +133,7 @@ func traverseTagHistory(ctx context.Context, r ref.TagRef, old, new *doltdb.Dolt
return new.NewTagAtCommit(ctx, r, cm, t.Meta)
}

func traverseCommitHistory(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error {
func traverseCommitHistory(ctx context.Context, menv Environment, cm *doltdb.Commit, new *doltdb.DoltDB, prog Progress) error {
ch, err := cm.HashOf()
if err != nil {
return err
Expand All @@ -155,7 +155,7 @@ func traverseCommitHistory(ctx context.Context, cm *doltdb.Commit, new *doltdb.D
}
if idx < 0 {
// parents for |cm| are done, migrate |cm|
if err = migrateCommit(ctx, cm, new, prog); err != nil {
if err = migrateCommit(ctx, menv, cm, new, prog); err != nil {
return err
}
// pop the stack, traverse upwards
Expand Down
30 changes: 30 additions & 0 deletions integration-tests/bats/migrate.bats
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,3 +266,33 @@ SQL
[[ ! "$output" =~ "alpha" ]] || false
[[ ! "$output" =~ "beta" ]] || false
}

@test "migrate: --drop-conflicts drops conflicts on migrate" {
dolt sql <<SQL
CREATE TABLE test (pk int primary key, c0 int, c1 int);
INSERT INTO test VALUES (0,0,0);
CALL dcommit('-Am', 'added table test');
CALL dcheckout('-b', 'other');
INSERT INTO test VALUES (1, 2, 3);
CALL dcommit('-am', 'added row on branch other');
CALL dcheckout('main');
INSERT INTO test VALUES (1, -2, -3);
CALL dcommit('-am', 'added row on branch main');
SET @@dolt_allow_commit_conflicts = 1;
CALL dmerge('other');
SQL
# dolt migrate --drop-conflicts
andy-wm-arthur marked this conversation as resolved.
Show resolved Hide resolved
}

@test "migrate: no panic for migration on migrated database" {
dolt sql <<SQL
CREATE TABLE test (pk int primary key, c0 int, c1 int);
INSERT INTO test VALUES (0,0,0);
CALL dadd('-A');
CALL dcommit('-am', 'added table test');
SQL
dolt migrate
run dolt migrate
[ $status -eq 0 ]
[[ "$output" =~ "already migrated" ]] || false
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Other cases to write bats for:

  • Committed conflicts
  • Conflicts on a non-checked out working sets. (Different branch)