From a1fae365000d31fb54b7bc80ec4ff4bc79ffe98e Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Thu, 30 Jun 2022 17:12:23 -0700 Subject: [PATCH 01/31] Trying something --- go/cmd/dolt/cli/arg_parser_helpers.go | 13 ++ go/cmd/dolt/commands/branch.go | 2 +- go/cmd/dolt/commands/clone.go | 18 +- .../doltcore/sqle/dprocedures/dolt_clone.go | 174 ++++++++++++++++++ 4 files changed, 191 insertions(+), 16 deletions(-) create mode 100755 go/libraries/doltcore/sqle/dprocedures/dolt_clone.go diff --git a/go/cmd/dolt/cli/arg_parser_helpers.go b/go/cmd/dolt/cli/arg_parser_helpers.go index c6961990907..1ba81710006 100644 --- a/go/cmd/dolt/cli/arg_parser_helpers.go +++ b/go/cmd/dolt/cli/arg_parser_helpers.go @@ -95,6 +95,8 @@ const ( DeleteFlag = "delete" DeleteForceFlag = "D" OutputOnlyFlag = "output-only" + RemoteParam = "remote" + BranchParam = "branch" ) const ( @@ -146,6 +148,17 @@ func CreateAddArgParser() *argparser.ArgParser { return ap } +func CreateCloneArgParser() *argparser.ArgParser { + ap := argparser.NewArgParser() + ap.SupportsString(RemoteParam, "", "name", "Name of the remote to be added. Default will be 'origin'.") + ap.SupportsString(BranchParam, "b", "branch", "The branch to be cloned. If not specified all branches will be cloned.") + ap.SupportsString(dbfactory.AWSRegionParam, "", "region", "") + ap.SupportsValidatedString(dbfactory.AWSCredsTypeParam, "", "creds-type", "", argparser.ValidatorFromStrList(dbfactory.AWSCredsTypeParam, credTypes)) + ap.SupportsString(dbfactory.AWSCredsFileParam, "", "file", "AWS credentials file.") + ap.SupportsString(dbfactory.AWSCredsProfile, "", "profile", "AWS profile to use.") + return ap +} + func CreateResetArgParser() *argparser.ArgParser { ap := argparser.NewArgParser() ap.SupportsFlag(HardResetParam, "", "Resets the working tables and staged tables. Any changes to tracked tables in the working tree since {{.LessThan}}commit{{.GreaterThan}} are discarded.") diff --git a/go/cmd/dolt/commands/branch.go b/go/cmd/dolt/commands/branch.go index e872b2d0d68..e544a75d8e0 100644 --- a/go/cmd/dolt/commands/branch.go +++ b/go/cmd/dolt/commands/branch.go @@ -138,7 +138,7 @@ func printBranches(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgPar branchSet := set.NewStrSet(apr.Args) verbose := apr.Contains(verboseFlag) - printRemote := apr.Contains(remoteParam) + printRemote := apr.Contains(RemoteParam) printAll := apr.Contains(allFlag) branches, err := dEnv.DoltDB.GetHeadRefs(ctx) diff --git a/go/cmd/dolt/commands/clone.go b/go/cmd/dolt/commands/clone.go index a7f42647ec7..ce46e3363aa 100644 --- a/go/cmd/dolt/commands/clone.go +++ b/go/cmd/dolt/commands/clone.go @@ -33,11 +33,6 @@ import ( "github.com/dolthub/dolt/go/store/types" ) -const ( - remoteParam = "remote" - branchParam = "branch" -) - var cloneDocs = cli.CommandDocumentationContent{ ShortDesc: "Clone a data repository into a new directory", LongDesc: `Clones a repository into a newly created directory, creates remote-tracking branches for each branch in the cloned repository (visible using {{.LessThan}}dolt branch -a{{.GreaterThan}}), and creates and checks out an initial branch that is forked from the cloned repository's currently active branch. @@ -75,14 +70,7 @@ func (cmd CloneCmd) Docs() *cli.CommandDocumentation { } func (cmd CloneCmd) ArgParser() *argparser.ArgParser { - ap := argparser.NewArgParser() - ap.SupportsString(remoteParam, "", "name", "Name of the remote to be added. Default will be 'origin'.") - ap.SupportsString(branchParam, "b", "branch", "The branch to be cloned. If not specified all branches will be cloned.") - ap.SupportsString(dbfactory.AWSRegionParam, "", "region", "") - ap.SupportsValidatedString(dbfactory.AWSCredsTypeParam, "", "creds-type", "", argparser.ValidatorFromStrList(dbfactory.AWSCredsTypeParam, credTypes)) - ap.SupportsString(dbfactory.AWSCredsFileParam, "", "file", "AWS credentials file.") - ap.SupportsString(dbfactory.AWSCredsProfile, "", "profile", "AWS profile to use.") - return ap + return cli.CreateCloneArgParser() } // EventType returns the type of the event to log @@ -105,8 +93,8 @@ func (cmd CloneCmd) Exec(ctx context.Context, commandStr string, args []string, } func clone(ctx context.Context, apr *argparser.ArgParseResults, dEnv *env.DoltEnv) errhand.VerboseError { - remoteName := apr.GetValueOrDefault(remoteParam, "origin") - branch := apr.GetValueOrDefault(branchParam, "") + remoteName := apr.GetValueOrDefault(cli.RemoteParam, "origin") + branch := apr.GetValueOrDefault(cli.BranchParam, "") dir, urlStr, verr := parseArgs(apr) if verr != nil { return verr diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go new file mode 100755 index 00000000000..e11303260e6 --- /dev/null +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go @@ -0,0 +1,174 @@ +// Copyright 2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dprocedures + +import ( + "os" + "path" + + "github.com/dolthub/dolt/go/cmd/dolt/cli" + "github.com/dolthub/dolt/go/cmd/dolt/errhand" + "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/doltcore/env/actions" + "github.com/dolthub/dolt/go/libraries/doltcore/remotestorage" + "github.com/dolthub/dolt/go/libraries/utils/argparser" + "github.com/dolthub/dolt/go/libraries/utils/earl" + "github.com/dolthub/dolt/go/store/types" + "github.com/dolthub/go-mysql-server/sql" +) + +// doltClone is a stored procedure to clone a database from a remote +func doltClone(ctx *sql.Context, args ...string) (sql.RowIter, error) { + ap := cli.CreateCloneArgParser() + apr, err := ap.Parse(args) + if err != nil { + return nil, err + } + + remoteName := apr.GetValueOrDefault(cli.RemoteParam, "origin") + branch := apr.GetValueOrDefault(cli.BranchParam, "") + dir, urlStr, err := parseArgs(apr) + if err != nil { + return nil, err + } + + // TODO: fill in from database provider + userDirExists := false + + var dEnv *env.DoltEnv + + scheme, remoteUrl, err := env.GetAbsRemoteUrl(dEnv.FS, dEnv.Config, urlStr) + + if err != nil { + return nil, errhand.BuildDError("error: '%s' is not valid.", urlStr).Build() + } + var params map[string]string + params, err = parseRemoteArgs(apr, scheme, remoteUrl) + if err != nil { + return nil, err + } + + var r env.Remote + var srcDB *doltdb.DoltDB + r, srcDB, err = createRemote(ctx, remoteName, remoteUrl, params, dEnv) + if err != nil { + return nil, err + } + + dEnv, err = actions.EnvForClone(ctx, srcDB.ValueReadWriter().Format(), r, dir, dEnv.FS, dEnv.Version, env.GetCurrentUserHomeDir) + if err != nil { + return nil, errhand.VerboseErrorFromError(err) + } + + err = actions.CloneRemote(ctx, srcDB, remoteName, branch, dEnv) + if err != nil { + // If we're cloning into a directory that already exists do not erase it. Otherwise + // make best effort to delete the directory we created. + if userDirExists { + // Set the working dir to the parent of the .dolt folder so we can delete .dolt + _ = os.Chdir(dir) + _ = dEnv.FS.Delete(dbfactory.DoltDir, true) + } else { + _ = os.Chdir("../") + _ = dEnv.FS.Delete(dir, true) + } + return nil, err + } + + err = dEnv.RepoStateWriter().UpdateBranch(dEnv.RepoState.CWBHeadRef().GetPath(), env.BranchConfig{ + Merge: dEnv.RepoState.Head, + Remote: remoteName, + }) + if err != nil { + return nil, err + } + + err = dEnv.RepoState.Save(dEnv.FS) + if err != nil { + return nil, err + } + + return rowToIter(int64(0)), nil +} + +// TODO: lifted from clone cmd +func parseArgs(apr *argparser.ArgParseResults) (string, string, error) { + if apr.NArg() < 1 || apr.NArg() > 2 { + return "", "", errhand.BuildDError("").SetPrintUsage().Build() + } + + urlStr := apr.Arg(0) + _, err := earl.Parse(urlStr) + + if err != nil { + return "", "", errhand.BuildDError("error: invalid remote url: " + urlStr).Build() + } + + var dir string + if apr.NArg() == 2 { + dir = apr.Arg(1) + } else { + dir = path.Base(urlStr) + if dir == "." { + dir = path.Dir(urlStr) + } else if dir == "/" { + return "", "", errhand.BuildDError("Could not infer repo name. Please explicitily define a directory for this url").Build() + } + } + + return dir, urlStr, nil +} + +// TODO: lifted from clone cmd +func parseRemoteArgs(apr *argparser.ArgParseResults, scheme, remoteUrl string) (map[string]string, errhand.VerboseError) { + params := map[string]string{} + + var err error + if scheme == dbfactory.AWSScheme { + err = cli.AddAWSParams(remoteUrl, apr, params) + } else { + err = cli.VerifyNoAwsParams(apr) + } + + if err != nil { + return nil, errhand.VerboseErrorFromError(err) + } + + return params, nil +} + +// TODO: lifted from clone cmd +func createRemote(ctx *sql.Context, remoteName, remoteUrl string, params map[string]string, dEnv *env.DoltEnv) (env.Remote, *doltdb.DoltDB, errhand.VerboseError) { + cli.Printf("cloning %s\n", remoteUrl) + + r := env.NewRemote(remoteName, remoteUrl, params, dEnv) + + ddb, err := r.GetRemoteDB(ctx, types.Format_Default) + + if err != nil { + bdr := errhand.BuildDError("error: failed to get remote db").AddCause(err) + + if err == remotestorage.ErrInvalidDoltSpecPath { + urlObj, _ := earl.Parse(remoteUrl) + bdr.AddDetails("'%s' should be in the format 'organization/repo'", urlObj.Path) + } + + return env.NoRemote, nil, bdr.Build() + } + + return r, ddb, nil +} From 134e9d50d8d51510dbb7b8eab46b5865dd6de948 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 1 Jul 2022 09:24:53 -0700 Subject: [PATCH 02/31] Inlined credtypes --- go/cmd/dolt/cli/arg_parser_helpers.go | 2 +- go/cmd/dolt/commands/remote.go | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/go/cmd/dolt/cli/arg_parser_helpers.go b/go/cmd/dolt/cli/arg_parser_helpers.go index 1ba81710006..a455d8efb88 100644 --- a/go/cmd/dolt/cli/arg_parser_helpers.go +++ b/go/cmd/dolt/cli/arg_parser_helpers.go @@ -153,7 +153,7 @@ func CreateCloneArgParser() *argparser.ArgParser { ap.SupportsString(RemoteParam, "", "name", "Name of the remote to be added. Default will be 'origin'.") ap.SupportsString(BranchParam, "b", "branch", "The branch to be cloned. If not specified all branches will be cloned.") ap.SupportsString(dbfactory.AWSRegionParam, "", "region", "") - ap.SupportsValidatedString(dbfactory.AWSCredsTypeParam, "", "creds-type", "", argparser.ValidatorFromStrList(dbfactory.AWSCredsTypeParam, credTypes)) + ap.SupportsValidatedString(dbfactory.AWSCredsTypeParam, "", "creds-type", "", argparser.ValidatorFromStrList(dbfactory.AWSCredsTypeParam, dbfactory.AWSCredTypes)) ap.SupportsString(dbfactory.AWSCredsFileParam, "", "file", "AWS credentials file.") ap.SupportsString(dbfactory.AWSCredsProfile, "", "profile", "AWS profile to use.") return ap diff --git a/go/cmd/dolt/commands/remote.go b/go/cmd/dolt/commands/remote.go index 09668f9c631..a9ac925ddb1 100644 --- a/go/cmd/dolt/commands/remote.go +++ b/go/cmd/dolt/commands/remote.go @@ -67,8 +67,6 @@ const ( removeRemoteShortId = "rm" ) -var credTypes = dbfactory.AWSCredTypes - type RemoteCmd struct{} // Name is returns the name of the Dolt cli command. This is what is used on the command line to invoke the command @@ -93,7 +91,7 @@ func (cmd RemoteCmd) ArgParser() *argparser.ArgParser { ap.ArgListHelp = append(ap.ArgListHelp, [2]string{"profile", "AWS profile to use."}) ap.SupportsFlag(verboseFlag, "v", "When printing the list of remotes adds additional details.") ap.SupportsString(dbfactory.AWSRegionParam, "", "region", "") - ap.SupportsValidatedString(dbfactory.AWSCredsTypeParam, "", "creds-type", "", argparser.ValidatorFromStrList(dbfactory.AWSCredsTypeParam, credTypes)) + ap.SupportsValidatedString(dbfactory.AWSCredsTypeParam, "", "creds-type", "", argparser.ValidatorFromStrList(dbfactory.AWSCredsTypeParam, dbfactory.AWSCredTypes)) ap.SupportsString(dbfactory.AWSCredsFileParam, "", "file", "AWS credentials file") ap.SupportsString(dbfactory.AWSCredsProfile, "", "profile", "AWS profile to use") return ap From 6cf1d2d0e56f4ac68f3e2bdbbc6fc26950c20376 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 1 Jul 2022 13:01:15 -0700 Subject: [PATCH 03/31] working dolt_clone --- go/cmd/dolt/commands/branch.go | 2 +- go/cmd/dolt/commands/engine/sqlengine.go | 2 +- go/libraries/doltcore/env/multi_repo_env.go | 7 ++ go/libraries/doltcore/env/remotes.go | 2 +- .../doltcore/sqle/database_provider.go | 98 +++++++++++++++++++ .../doltcore/sqle/dprocedures/dolt_clone.go | 63 +++--------- .../doltcore/sqle/dprocedures/init.go | 1 + .../sqle/dsess/revision_db_provider.go | 6 ++ 8 files changed, 130 insertions(+), 51 deletions(-) diff --git a/go/cmd/dolt/commands/branch.go b/go/cmd/dolt/commands/branch.go index e544a75d8e0..33ea62ecb1f 100644 --- a/go/cmd/dolt/commands/branch.go +++ b/go/cmd/dolt/commands/branch.go @@ -138,7 +138,7 @@ func printBranches(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgPar branchSet := set.NewStrSet(apr.Args) verbose := apr.Contains(verboseFlag) - printRemote := apr.Contains(RemoteParam) + printRemote := apr.Contains(cli.RemoteParam) printAll := apr.Contains(allFlag) branches, err := dEnv.DoltDB.GetHeadRefs(ctx) diff --git a/go/cmd/dolt/commands/engine/sqlengine.go b/go/cmd/dolt/commands/engine/sqlengine.go index 601831ecafd..dd20fc3e827 100644 --- a/go/cmd/dolt/commands/engine/sqlengine.go +++ b/go/cmd/dolt/commands/engine/sqlengine.go @@ -82,7 +82,7 @@ func NewSqlEngine( all := append(dsqleDBsAsSqlDBs(dbs), infoDB) b := env.GetDefaultInitBranch(mrEnv.Config()) - pro := dsqle.NewDoltDatabaseProvider(b, mrEnv.FileSystem(), all...) + pro := dsqle.NewDoltDatabaseProvider(b, mrEnv.FileSystem(), all...).WithRemoteDialer(mrEnv.RemoteDialProvider()) // Load in privileges from file, if it exists persister := mysql_file_handler.NewPersister(config.PrivFilePath) diff --git a/go/libraries/doltcore/env/multi_repo_env.go b/go/libraries/doltcore/env/multi_repo_env.go index ccab2d02b6d..da3340aa343 100644 --- a/go/libraries/doltcore/env/multi_repo_env.go +++ b/go/libraries/doltcore/env/multi_repo_env.go @@ -56,6 +56,13 @@ func (mrEnv *MultiRepoEnv) FileSystem() filesys.Filesys { return mrEnv.fs } +func (mrEnv *MultiRepoEnv) RemoteDialProvider() dbfactory.GRPCDialProvider { + for _, env := range mrEnv.envs { + return env.env + } + return nil +} + func (mrEnv *MultiRepoEnv) Config() config.ReadWriteConfig { return mrEnv.cfg } diff --git a/go/libraries/doltcore/env/remotes.go b/go/libraries/doltcore/env/remotes.go index 6c6d43aeb46..24d60d2b7f2 100644 --- a/go/libraries/doltcore/env/remotes.go +++ b/go/libraries/doltcore/env/remotes.go @@ -396,7 +396,7 @@ func GetAbsRemoteUrl(fs filesys2.Filesys, cfg config.ReadableConfig, urlArg stri return "", "", err } - if u.Scheme != "" { + if u.Scheme != "" && fs != nil { if u.Scheme == dbfactory.FileScheme || u.Scheme == dbfactory.LocalBSScheme { absUrl, err := getAbsFileRemoteUrl(u.Host+u.Path, u.Scheme, fs) diff --git a/go/libraries/doltcore/sqle/database_provider.go b/go/libraries/doltcore/sqle/database_provider.go index 6a6e6fe9cda..d5b441425bb 100644 --- a/go/libraries/doltcore/sqle/database_provider.go +++ b/go/libraries/doltcore/sqle/database_provider.go @@ -20,6 +20,10 @@ import ( "strings" "sync" + "github.com/dolthub/dolt/go/cmd/dolt/errhand" + "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" + "github.com/dolthub/dolt/go/libraries/doltcore/remotestorage" + "github.com/dolthub/dolt/go/libraries/utils/earl" "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" @@ -45,6 +49,7 @@ type DoltDatabaseProvider struct { defaultBranch string dataRootDir string fs filesys.Filesys + remoteDialer dbfactory.GRPCDialProvider dbFactoryUrl string } @@ -95,6 +100,11 @@ func (p DoltDatabaseProvider) WithDbFactoryUrl(url string) DoltDatabaseProvider return p } +func (p DoltDatabaseProvider) WithRemoteDialer(provider dbfactory.GRPCDialProvider) DoltDatabaseProvider { + p.remoteDialer = provider + return p +} + func (p DoltDatabaseProvider) Database(ctx *sql.Context, name string) (db sql.Database, err error) { var ok bool p.mu.RLock() @@ -193,6 +203,94 @@ func (p DoltDatabaseProvider) CreateDatabase(ctx *sql.Context, name string) erro return dsess.AddDB(ctx, dbstate) } +func (p DoltDatabaseProvider) CloneDatabaseFromRemote(ctx *sql.Context, dbName, branch, remoteName, remoteUrl string, remoteParams map[string]string) error { + p.mu.Lock() + defer p.mu.Unlock() + + exists, isDir := p.fs.Exists(dbName) + if exists && isDir { + return sql.ErrDatabaseExists.New(dbName) + } else if exists { + return fmt.Errorf("Cannot create DB, file exists at %s", dbName) + } + + // err := p.fs.MkDirs(dbName) + // if err != nil { + // return err + // } + + var r env.Remote + var srcDB *doltdb.DoltDB + r, srcDB, err := createRemote(ctx, remoteName, remoteUrl, remoteParams, p.remoteDialer) + if err != nil { + return err + } + + dEnv, err := actions.EnvForClone(ctx, srcDB.ValueReadWriter().Format(), r, dbName, p.fs, "VERSION", env.GetCurrentUserHomeDir) + if err != nil { + return err + } + + err = actions.CloneRemote(ctx, srcDB, remoteName, branch, dEnv) + if err != nil { + return err + } + + err = dEnv.RepoStateWriter().UpdateBranch(dEnv.RepoState.CWBHeadRef().GetPath(), env.BranchConfig{ + Merge: dEnv.RepoState.Head, + Remote: remoteName, + }) + + // // TODO: fill in version appropriately + // dsess := dsess.DSessFromSess(ctx.Session) + // newEnv := env.Load(ctx, env.GetCurrentUserHomeDir, newFs, p.dbFactoryUrl, "TODO") + // err = newEnv.InitRepo(ctx, types.Format_Default, dsess.Username(), dsess.Email(), p.defaultBranch) + // if err != nil { + // return err + // } + // + // fkChecks, err := ctx.GetSessionVariable(ctx, "foreign_key_checks") + // if err != nil { + // return err + // } + // + // opts := editor.Options{ + // Deaf: newEnv.DbEaFactory(), + // // TODO: this doesn't seem right, why is this getting set in the constructor to the DB + // ForeignKeyChecksDisabled: fkChecks.(int8) == 0, + // } + // + // db := NewDatabase(name, newEnv.DbData(), opts) + // p.databases[formatDbMapKeyName(db.Name())] = db + // + // dbstate, err := GetInitialDBState(ctx, db) + // if err != nil { + // return err + // } + // + // return dsess.AddDB(ctx, dbstate) + return nil +} + +func createRemote(ctx *sql.Context, remoteName, remoteUrl string, params map[string]string, dialer dbfactory.GRPCDialProvider) (env.Remote, *doltdb.DoltDB, error) { + r := env.NewRemote(remoteName, remoteUrl, params, dialer) + + ddb, err := r.GetRemoteDB(ctx, types.Format_Default) + + if err != nil { + bdr := errhand.BuildDError("error: failed to get remote db").AddCause(err) + + if err == remotestorage.ErrInvalidDoltSpecPath { + urlObj, _ := earl.Parse(remoteUrl) + bdr.AddDetails("'%s' should be in the format 'organization/repo'", urlObj.Path) + } + + return env.NoRemote, nil, bdr.Build() + } + + return r, ddb, nil +} + func (p DoltDatabaseProvider) DropDatabase(ctx *sql.Context, name string) error { p.mu.Lock() defer p.mu.Unlock() diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go index e11303260e6..39c0204f38b 100755 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go @@ -15,7 +15,6 @@ package dprocedures import ( - "os" "path" "github.com/dolthub/dolt/go/cmd/dolt/cli" @@ -23,9 +22,10 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" - "github.com/dolthub/dolt/go/libraries/doltcore/env/actions" "github.com/dolthub/dolt/go/libraries/doltcore/remotestorage" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" "github.com/dolthub/dolt/go/libraries/utils/argparser" + "github.com/dolthub/dolt/go/libraries/utils/config" "github.com/dolthub/dolt/go/libraries/utils/earl" "github.com/dolthub/dolt/go/store/types" "github.com/dolthub/go-mysql-server/sql" @@ -46,12 +46,8 @@ func doltClone(ctx *sql.Context, args ...string) (sql.RowIter, error) { return nil, err } - // TODO: fill in from database provider - userDirExists := false - - var dEnv *env.DoltEnv - - scheme, remoteUrl, err := env.GetAbsRemoteUrl(dEnv.FS, dEnv.Config, urlStr) + // TODO: empty config here + scheme, remoteUrl, err := env.GetAbsRemoteUrl(nil, emptyConfig(), urlStr) if err != nil { return nil, errhand.BuildDError("error: '%s' is not valid.", urlStr).Build() @@ -62,49 +58,22 @@ func doltClone(ctx *sql.Context, args ...string) (sql.RowIter, error) { return nil, err } - var r env.Remote - var srcDB *doltdb.DoltDB - r, srcDB, err = createRemote(ctx, remoteName, remoteUrl, params, dEnv) - if err != nil { - return nil, err - } - - dEnv, err = actions.EnvForClone(ctx, srcDB.ValueReadWriter().Format(), r, dir, dEnv.FS, dEnv.Version, env.GetCurrentUserHomeDir) - if err != nil { - return nil, errhand.VerboseErrorFromError(err) - } - - err = actions.CloneRemote(ctx, srcDB, remoteName, branch, dEnv) - if err != nil { - // If we're cloning into a directory that already exists do not erase it. Otherwise - // make best effort to delete the directory we created. - if userDirExists { - // Set the working dir to the parent of the .dolt folder so we can delete .dolt - _ = os.Chdir(dir) - _ = dEnv.FS.Delete(dbfactory.DoltDir, true) - } else { - _ = os.Chdir("../") - _ = dEnv.FS.Delete(dir, true) - } - return nil, err - } - - err = dEnv.RepoStateWriter().UpdateBranch(dEnv.RepoState.CWBHeadRef().GetPath(), env.BranchConfig{ - Merge: dEnv.RepoState.Head, - Remote: remoteName, - }) - if err != nil { - return nil, err - } + sess := dsess.DSessFromSess(ctx.Session) + sess.Provider().CloneDatabaseFromRemote(ctx, dir, branch, remoteName, remoteUrl, params) - err = dEnv.RepoState.Save(dEnv.FS) - if err != nil { - return nil, err - } + // + // err = dEnv.RepoState.Save(dEnv.FS) + // if err != nil { + // return nil, err + // } return rowToIter(int64(0)), nil } +func emptyConfig() config.ReadableConfig { + return &config.MapConfig{} +} + // TODO: lifted from clone cmd func parseArgs(apr *argparser.ArgParseResults) (string, string, error) { if apr.NArg() < 1 || apr.NArg() > 2 { @@ -153,8 +122,6 @@ func parseRemoteArgs(apr *argparser.ArgParseResults, scheme, remoteUrl string) ( // TODO: lifted from clone cmd func createRemote(ctx *sql.Context, remoteName, remoteUrl string, params map[string]string, dEnv *env.DoltEnv) (env.Remote, *doltdb.DoltDB, errhand.VerboseError) { - cli.Printf("cloning %s\n", remoteUrl) - r := env.NewRemote(remoteName, remoteUrl, params, dEnv) ddb, err := r.GetRemoteDB(ctx, types.Format_Default) diff --git a/go/libraries/doltcore/sqle/dprocedures/init.go b/go/libraries/doltcore/sqle/dprocedures/init.go index e0dc0e37042..63cbdf662e9 100644 --- a/go/libraries/doltcore/sqle/dprocedures/init.go +++ b/go/libraries/doltcore/sqle/dprocedures/init.go @@ -22,6 +22,7 @@ var DoltProcedures = []sql.ExternalStoredProcedureDetails{ {Name: "dolt_branch", Schema: int64Schema("status"), Function: doltBranch}, {Name: "dolt_checkout", Schema: int64Schema("status"), Function: doltCheckout}, {Name: "dolt_clean", Schema: int64Schema("status"), Function: doltClean}, + {Name: "dolt_clone", Schema: int64Schema("status"), Function: doltClone}, {Name: "dolt_commit", Schema: stringSchema("hash"), Function: doltCommit}, {Name: "dolt_fetch", Schema: int64Schema("success"), Function: doltFetch}, {Name: "dolt_merge", Schema: int64Schema("fast_forward", "conflicts"), Function: doltMerge}, diff --git a/go/libraries/doltcore/sqle/dsess/revision_db_provider.go b/go/libraries/doltcore/sqle/dsess/revision_db_provider.go index fd67f16f4ea..902ba373939 100644 --- a/go/libraries/doltcore/sqle/dsess/revision_db_provider.go +++ b/go/libraries/doltcore/sqle/dsess/revision_db_provider.go @@ -32,6 +32,8 @@ type RevisionDatabaseProvider interface { RevisionDbState(ctx *sql.Context, revDB string) (InitialDbState, error) // DropRevisionDb removes the specified revision database from the databases this provider is tracking. DropRevisionDb(ctx *sql.Context, revDB string) error + // CloneDatabaseFromRemote clones the database from the remote given as a new database in this provider + CloneDatabaseFromRemote(ctx *sql.Context, dbName, branch, remoteName, remoteUrl string, remoteParams map[string]string) error } func EmptyDatabaseProvider() RevisionDatabaseProvider { @@ -40,6 +42,10 @@ func EmptyDatabaseProvider() RevisionDatabaseProvider { type emptyRevisionDatabaseProvider struct{} +func (e emptyRevisionDatabaseProvider) CloneDatabaseFromRemote(ctx *sql.Context, dbName, branch, remoteName, remoteUrl string, remoteParams map[string]string) error { + return nil +} + func (e emptyRevisionDatabaseProvider) DropRevisionDb(ctx *sql.Context, revDB string) error { return nil } From 3797be16257578072d2b186833bd2200552152fb Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 1 Jul 2022 13:30:24 -0700 Subject: [PATCH 04/31] Added new db to session after clone --- .../doltcore/sqle/database_provider.go | 50 ++++++++----------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/go/libraries/doltcore/sqle/database_provider.go b/go/libraries/doltcore/sqle/database_provider.go index d5b441425bb..f94d493c480 100644 --- a/go/libraries/doltcore/sqle/database_provider.go +++ b/go/libraries/doltcore/sqle/database_provider.go @@ -241,35 +241,27 @@ func (p DoltDatabaseProvider) CloneDatabaseFromRemote(ctx *sql.Context, dbName, Remote: remoteName, }) - // // TODO: fill in version appropriately - // dsess := dsess.DSessFromSess(ctx.Session) - // newEnv := env.Load(ctx, env.GetCurrentUserHomeDir, newFs, p.dbFactoryUrl, "TODO") - // err = newEnv.InitRepo(ctx, types.Format_Default, dsess.Username(), dsess.Email(), p.defaultBranch) - // if err != nil { - // return err - // } - // - // fkChecks, err := ctx.GetSessionVariable(ctx, "foreign_key_checks") - // if err != nil { - // return err - // } - // - // opts := editor.Options{ - // Deaf: newEnv.DbEaFactory(), - // // TODO: this doesn't seem right, why is this getting set in the constructor to the DB - // ForeignKeyChecksDisabled: fkChecks.(int8) == 0, - // } - // - // db := NewDatabase(name, newEnv.DbData(), opts) - // p.databases[formatDbMapKeyName(db.Name())] = db - // - // dbstate, err := GetInitialDBState(ctx, db) - // if err != nil { - // return err - // } - // - // return dsess.AddDB(ctx, dbstate) - return nil + dsess := dsess.DSessFromSess(ctx.Session) + fkChecks, err := ctx.GetSessionVariable(ctx, "foreign_key_checks") + if err != nil { + return err + } + + opts := editor.Options{ + Deaf: dEnv.DbEaFactory(), + // TODO: this doesn't seem right, why is this getting set in the constructor to the DB + ForeignKeyChecksDisabled: fkChecks.(int8) == 0, + } + + db := NewDatabase(dbName, dEnv.DbData(), opts) + p.databases[formatDbMapKeyName(db.Name())] = db + + dbstate, err := GetInitialDBState(ctx, db) + if err != nil { + return err + } + + return dsess.AddDB(ctx, dbstate) } func createRemote(ctx *sql.Context, remoteName, remoteUrl string, params map[string]string, dialer dbfactory.GRPCDialProvider) (env.Remote, *doltdb.DoltDB, error) { From 3f52707f209c95eff1ec7698b27ac9ebb151f66b Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 1 Jul 2022 14:59:29 -0700 Subject: [PATCH 05/31] Cleaned up clone --- .../doltcore/sqle/database_provider.go | 24 +++++++++++-------- .../doltcore/sqle/dprocedures/dolt_clone.go | 17 ++++++------- .../doltcore/sqle/dsess/dolt_session.go | 2 +- .../sqle/dsess/revision_db_provider.go | 13 +++++++++- go/libraries/doltcore/sqle/dsess/session.go | 6 ++--- go/libraries/doltcore/sqle/testutil.go | 2 +- 6 files changed, 38 insertions(+), 26 deletions(-) diff --git a/go/libraries/doltcore/sqle/database_provider.go b/go/libraries/doltcore/sqle/database_provider.go index f94d493c480..dc88d69d11c 100644 --- a/go/libraries/doltcore/sqle/database_provider.go +++ b/go/libraries/doltcore/sqle/database_provider.go @@ -57,7 +57,7 @@ type DoltDatabaseProvider struct { var _ sql.DatabaseProvider = (*DoltDatabaseProvider)(nil) var _ sql.FunctionProvider = (*DoltDatabaseProvider)(nil) var _ sql.MutableDatabaseProvider = (*DoltDatabaseProvider)(nil) -var _ dsess.RevisionDatabaseProvider = (*DoltDatabaseProvider)(nil) +var _ dsess.DoltDatabaseProvider = (*DoltDatabaseProvider)(nil) // NewDoltDatabaseProvider returns a provider for the databases given func NewDoltDatabaseProvider(defaultBranch string, fs filesys.Filesys, databases ...sql.Database) DoltDatabaseProvider { @@ -100,11 +100,16 @@ func (p DoltDatabaseProvider) WithDbFactoryUrl(url string) DoltDatabaseProvider return p } +// WithRemoteDialer returns a copy of this provider with the dialer provided func (p DoltDatabaseProvider) WithRemoteDialer(provider dbfactory.GRPCDialProvider) DoltDatabaseProvider { p.remoteDialer = provider return p } +func (p DoltDatabaseProvider) FileSystem() filesys.Filesys { + return p.fs +} + func (p DoltDatabaseProvider) Database(ctx *sql.Context, name string) (db sql.Database, err error) { var ok bool p.mu.RLock() @@ -211,17 +216,16 @@ func (p DoltDatabaseProvider) CloneDatabaseFromRemote(ctx *sql.Context, dbName, if exists && isDir { return sql.ErrDatabaseExists.New(dbName) } else if exists { - return fmt.Errorf("Cannot create DB, file exists at %s", dbName) + return fmt.Errorf("cannot create DB, file exists at %s", dbName) } - // err := p.fs.MkDirs(dbName) - // if err != nil { - // return err - // } - var r env.Remote var srcDB *doltdb.DoltDB - r, srcDB, err := createRemote(ctx, remoteName, remoteUrl, remoteParams, p.remoteDialer) + dialer := p.remoteDialer + if dialer == nil { + + } + r, srcDB, err := createRemote(ctx, remoteName, remoteUrl, remoteParams, dialer) if err != nil { return err } @@ -241,7 +245,7 @@ func (p DoltDatabaseProvider) CloneDatabaseFromRemote(ctx *sql.Context, dbName, Remote: remoteName, }) - dsess := dsess.DSessFromSess(ctx.Session) + sess := dsess.DSessFromSess(ctx.Session) fkChecks, err := ctx.GetSessionVariable(ctx, "foreign_key_checks") if err != nil { return err @@ -261,7 +265,7 @@ func (p DoltDatabaseProvider) CloneDatabaseFromRemote(ctx *sql.Context, dbName, return err } - return dsess.AddDB(ctx, dbstate) + return sess.AddDB(ctx, dbstate) } func createRemote(ctx *sql.Context, remoteName, remoteUrl string, params map[string]string, dialer dbfactory.GRPCDialProvider) (env.Remote, *doltdb.DoltDB, error) { diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go index 39c0204f38b..d814941a345 100755 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go @@ -39,6 +39,7 @@ func doltClone(ctx *sql.Context, args ...string) (sql.RowIter, error) { return nil, err } + remoteName := apr.GetValueOrDefault(cli.RemoteParam, "origin") branch := apr.GetValueOrDefault(cli.BranchParam, "") dir, urlStr, err := parseArgs(apr) @@ -46,8 +47,8 @@ func doltClone(ctx *sql.Context, args ...string) (sql.RowIter, error) { return nil, err } - // TODO: empty config here - scheme, remoteUrl, err := env.GetAbsRemoteUrl(nil, emptyConfig(), urlStr) + sess := dsess.DSessFromSess(ctx.Session) + scheme, remoteUrl, err := env.GetAbsRemoteUrl(sess.Provider().FileSystem(), emptyConfig(), urlStr) if err != nil { return nil, errhand.BuildDError("error: '%s' is not valid.", urlStr).Build() @@ -58,14 +59,10 @@ func doltClone(ctx *sql.Context, args ...string) (sql.RowIter, error) { return nil, err } - sess := dsess.DSessFromSess(ctx.Session) - sess.Provider().CloneDatabaseFromRemote(ctx, dir, branch, remoteName, remoteUrl, params) - - // - // err = dEnv.RepoState.Save(dEnv.FS) - // if err != nil { - // return nil, err - // } + err = sess.Provider().CloneDatabaseFromRemote(ctx, dir, branch, remoteName, remoteUrl, params) + if err != nil { + return nil, err + } return rowToIter(int64(0)), nil } diff --git a/go/libraries/doltcore/sqle/dsess/dolt_session.go b/go/libraries/doltcore/sqle/dsess/dolt_session.go index d8c3e800916..aac96267297 100644 --- a/go/libraries/doltcore/sqle/dsess/dolt_session.go +++ b/go/libraries/doltcore/sqle/dsess/dolt_session.go @@ -39,7 +39,7 @@ var _ sql.Session = (*DoltSession)(nil) var _ sql.PersistableSession = (*DoltSession)(nil) // NewDoltSession creates a DoltSession object from a standard sql.Session and 0 or more Database objects. -func NewDoltSession(ctx *sql.Context, sqlSess *sql.BaseSession, pro RevisionDatabaseProvider, conf config.ReadWriteConfig, dbs ...InitialDbState) (*DoltSession, error) { +func NewDoltSession(ctx *sql.Context, sqlSess *sql.BaseSession, pro DoltDatabaseProvider, conf config.ReadWriteConfig, dbs ...InitialDbState) (*DoltSession, error) { sess, err := NewSession(ctx, sqlSess, pro, conf, dbs...) if err != nil { return nil, err diff --git a/go/libraries/doltcore/sqle/dsess/revision_db_provider.go b/go/libraries/doltcore/sqle/dsess/revision_db_provider.go index 902ba373939..e55b58abf0b 100644 --- a/go/libraries/doltcore/sqle/dsess/revision_db_provider.go +++ b/go/libraries/doltcore/sqle/dsess/revision_db_provider.go @@ -15,6 +15,7 @@ package dsess import ( + "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/dolthub/go-mysql-server/sql" "gopkg.in/src-d/go-errors.v1" ) @@ -32,16 +33,26 @@ type RevisionDatabaseProvider interface { RevisionDbState(ctx *sql.Context, revDB string) (InitialDbState, error) // DropRevisionDb removes the specified revision database from the databases this provider is tracking. DropRevisionDb(ctx *sql.Context, revDB string) error +} + +type DoltDatabaseProvider interface { + RevisionDatabaseProvider // CloneDatabaseFromRemote clones the database from the remote given as a new database in this provider CloneDatabaseFromRemote(ctx *sql.Context, dbName, branch, remoteName, remoteUrl string, remoteParams map[string]string) error + // FileSystem returns the filesystem used by this provider, rooted at the data directory for all databases + FileSystem() filesys.Filesys } -func EmptyDatabaseProvider() RevisionDatabaseProvider { +func EmptyDatabaseProvider() DoltDatabaseProvider { return emptyRevisionDatabaseProvider{} } type emptyRevisionDatabaseProvider struct{} +func (e emptyRevisionDatabaseProvider) FileSystem() filesys.Filesys { + return nil +} + func (e emptyRevisionDatabaseProvider) CloneDatabaseFromRemote(ctx *sql.Context, dbName, branch, remoteName, remoteUrl string, remoteParams map[string]string) error { return nil } diff --git a/go/libraries/doltcore/sqle/dsess/session.go b/go/libraries/doltcore/sqle/dsess/session.go index 6455bea9e65..159e3ba0006 100644 --- a/go/libraries/doltcore/sqle/dsess/session.go +++ b/go/libraries/doltcore/sqle/dsess/session.go @@ -62,7 +62,7 @@ type Session struct { username string email string dbStates map[string]*DatabaseSessionState - provider RevisionDatabaseProvider + provider DoltDatabaseProvider tempTables map[string][]sql.Table } @@ -82,7 +82,7 @@ func DefaultSession() *Session { } // NewSession creates a Session object from a standard sql.Session and 0 or more Database objects. -func NewSession(ctx *sql.Context, sqlSess *sql.BaseSession, pro RevisionDatabaseProvider, conf config.ReadableConfig, dbs ...InitialDbState) (*Session, error) { +func NewSession(ctx *sql.Context, sqlSess *sql.BaseSession, pro DoltDatabaseProvider, conf config.ReadableConfig, dbs ...InitialDbState) (*Session, error) { username := conf.GetStringOrDefault(env.UserNameKey, "") email := conf.GetStringOrDefault(env.UserEmailKey, "") sess := &Session{ @@ -106,7 +106,7 @@ func NewSession(ctx *sql.Context, sqlSess *sql.BaseSession, pro RevisionDatabase } // Provider returns the RevisionDatabaseProvider for this session. -func (sess *Session) Provider() RevisionDatabaseProvider { +func (sess *Session) Provider() DoltDatabaseProvider { return sess.provider } diff --git a/go/libraries/doltcore/sqle/testutil.go b/go/libraries/doltcore/sqle/testutil.go index 536c068ef3f..b91d86e51f5 100644 --- a/go/libraries/doltcore/sqle/testutil.go +++ b/go/libraries/doltcore/sqle/testutil.go @@ -104,7 +104,7 @@ func NewTestSQLCtx(ctx context.Context) *sql.Context { return NewTestSQLCtxWithProvider(ctx, dsess.EmptyDatabaseProvider()) } -func NewTestSQLCtxWithProvider(ctx context.Context, pro dsess.RevisionDatabaseProvider) *sql.Context { +func NewTestSQLCtxWithProvider(ctx context.Context, pro dsess.DoltDatabaseProvider) *sql.Context { s, err := dsess.NewDoltSession( sql.NewEmptyContext(), sql.NewBaseSession(), From 58d115c9a0180cd71c963defc51caab42928f1e7 Mon Sep 17 00:00:00 2001 From: zachmu Date: Fri, 1 Jul 2022 22:01:20 +0000 Subject: [PATCH 06/31] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/cmd/dolt/cli/arg_parser_helpers.go | 4 ++-- go/libraries/doltcore/env/remotes.go | 2 +- go/libraries/doltcore/sqle/database_provider.go | 8 ++++---- go/libraries/doltcore/sqle/dprocedures/dolt_clone.go | 4 ++-- go/libraries/doltcore/sqle/dsess/revision_db_provider.go | 3 ++- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/go/cmd/dolt/cli/arg_parser_helpers.go b/go/cmd/dolt/cli/arg_parser_helpers.go index a455d8efb88..f044e8a6140 100644 --- a/go/cmd/dolt/cli/arg_parser_helpers.go +++ b/go/cmd/dolt/cli/arg_parser_helpers.go @@ -95,8 +95,8 @@ const ( DeleteFlag = "delete" DeleteForceFlag = "D" OutputOnlyFlag = "output-only" - RemoteParam = "remote" - BranchParam = "branch" + RemoteParam = "remote" + BranchParam = "branch" ) const ( diff --git a/go/libraries/doltcore/env/remotes.go b/go/libraries/doltcore/env/remotes.go index 24d60d2b7f2..eefa2d79a09 100644 --- a/go/libraries/doltcore/env/remotes.go +++ b/go/libraries/doltcore/env/remotes.go @@ -396,7 +396,7 @@ func GetAbsRemoteUrl(fs filesys2.Filesys, cfg config.ReadableConfig, urlArg stri return "", "", err } - if u.Scheme != "" && fs != nil { + if u.Scheme != "" && fs != nil { if u.Scheme == dbfactory.FileScheme || u.Scheme == dbfactory.LocalBSScheme { absUrl, err := getAbsFileRemoteUrl(u.Host+u.Path, u.Scheme, fs) diff --git a/go/libraries/doltcore/sqle/database_provider.go b/go/libraries/doltcore/sqle/database_provider.go index dc88d69d11c..3135a91fb69 100644 --- a/go/libraries/doltcore/sqle/database_provider.go +++ b/go/libraries/doltcore/sqle/database_provider.go @@ -20,19 +20,19 @@ import ( "strings" "sync" - "github.com/dolthub/dolt/go/cmd/dolt/errhand" - "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" - "github.com/dolthub/dolt/go/libraries/doltcore/remotestorage" - "github.com/dolthub/dolt/go/libraries/utils/earl" "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/dolt/go/cmd/dolt/errhand" + "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/env/actions" "github.com/dolthub/dolt/go/libraries/doltcore/ref" + "github.com/dolthub/dolt/go/libraries/doltcore/remotestorage" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dfunctions" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" "github.com/dolthub/dolt/go/libraries/doltcore/table/editor" + "github.com/dolthub/dolt/go/libraries/utils/earl" "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/dolthub/dolt/go/store/types" ) diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go index d814941a345..4cbbe98eee4 100755 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go @@ -17,6 +17,8 @@ package dprocedures import ( "path" + "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/cmd/dolt/errhand" "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" @@ -28,7 +30,6 @@ import ( "github.com/dolthub/dolt/go/libraries/utils/config" "github.com/dolthub/dolt/go/libraries/utils/earl" "github.com/dolthub/dolt/go/store/types" - "github.com/dolthub/go-mysql-server/sql" ) // doltClone is a stored procedure to clone a database from a remote @@ -39,7 +40,6 @@ func doltClone(ctx *sql.Context, args ...string) (sql.RowIter, error) { return nil, err } - remoteName := apr.GetValueOrDefault(cli.RemoteParam, "origin") branch := apr.GetValueOrDefault(cli.BranchParam, "") dir, urlStr, err := parseArgs(apr) diff --git a/go/libraries/doltcore/sqle/dsess/revision_db_provider.go b/go/libraries/doltcore/sqle/dsess/revision_db_provider.go index e55b58abf0b..b42894846c1 100644 --- a/go/libraries/doltcore/sqle/dsess/revision_db_provider.go +++ b/go/libraries/doltcore/sqle/dsess/revision_db_provider.go @@ -15,9 +15,10 @@ package dsess import ( - "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/dolthub/go-mysql-server/sql" "gopkg.in/src-d/go-errors.v1" + + "github.com/dolthub/dolt/go/libraries/utils/filesys" ) // ErrRevisionDbNotFound is thrown when a RevisionDatabaseProvider cannot find a specified revision database. From d5fa232353bc16534927c4442a211cc9edd32a70 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 1 Jul 2022 15:23:39 -0700 Subject: [PATCH 07/31] Fixed compile error --- go/libraries/doltcore/sqle/enginetest/dolt_harness.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_harness.go b/go/libraries/doltcore/sqle/enginetest/dolt_harness.go index 84373c6d3e3..73ff206d932 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_harness.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_harness.go @@ -256,7 +256,7 @@ func (d *DoltHarness) newSessionWithClient(client sql.Client) *dsess.DoltSession dSession, err := dsess.NewDoltSession( enginetest.NewContext(d), sql.NewBaseSessionWithClientServer("address", client, 1), - pro.(dsess.RevisionDatabaseProvider), + pro.(dsess.DoltDatabaseProvider), localConfig, states..., ) From a65c3a7de2c62a63eba3f68fe8deb3b96f37a74e Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Thu, 14 Jul 2022 14:50:44 -0700 Subject: [PATCH 08/31] Small cleanup --- .../doltcore/sqle/dprocedures/dolt_clone.go | 35 +++---------------- 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go index 4cbbe98eee4..d48da301a5e 100755 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go @@ -22,14 +22,11 @@ import ( "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/cmd/dolt/errhand" "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" - "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" - "github.com/dolthub/dolt/go/libraries/doltcore/remotestorage" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" "github.com/dolthub/dolt/go/libraries/utils/argparser" "github.com/dolthub/dolt/go/libraries/utils/config" "github.com/dolthub/dolt/go/libraries/utils/earl" - "github.com/dolthub/dolt/go/store/types" ) // doltClone is a stored procedure to clone a database from a remote @@ -42,7 +39,7 @@ func doltClone(ctx *sql.Context, args ...string) (sql.RowIter, error) { remoteName := apr.GetValueOrDefault(cli.RemoteParam, "origin") branch := apr.GetValueOrDefault(cli.BranchParam, "") - dir, urlStr, err := parseArgs(apr) + dir, urlStr, err := getDirectoryAndUrlString(apr) if err != nil { return nil, err } @@ -54,7 +51,7 @@ func doltClone(ctx *sql.Context, args ...string) (sql.RowIter, error) { return nil, errhand.BuildDError("error: '%s' is not valid.", urlStr).Build() } var params map[string]string - params, err = parseRemoteArgs(apr, scheme, remoteUrl) + params, err = remoteParams(apr, scheme, remoteUrl) if err != nil { return nil, err } @@ -71,8 +68,7 @@ func emptyConfig() config.ReadableConfig { return &config.MapConfig{} } -// TODO: lifted from clone cmd -func parseArgs(apr *argparser.ArgParseResults) (string, string, error) { +func getDirectoryAndUrlString(apr *argparser.ArgParseResults) (string, string, error) { if apr.NArg() < 1 || apr.NArg() > 2 { return "", "", errhand.BuildDError("").SetPrintUsage().Build() } @@ -99,8 +95,7 @@ func parseArgs(apr *argparser.ArgParseResults) (string, string, error) { return dir, urlStr, nil } -// TODO: lifted from clone cmd -func parseRemoteArgs(apr *argparser.ArgParseResults, scheme, remoteUrl string) (map[string]string, errhand.VerboseError) { +func remoteParams(apr *argparser.ArgParseResults, scheme, remoteUrl string) (map[string]string, errhand.VerboseError) { params := map[string]string{} var err error @@ -115,24 +110,4 @@ func parseRemoteArgs(apr *argparser.ArgParseResults, scheme, remoteUrl string) ( } return params, nil -} - -// TODO: lifted from clone cmd -func createRemote(ctx *sql.Context, remoteName, remoteUrl string, params map[string]string, dEnv *env.DoltEnv) (env.Remote, *doltdb.DoltDB, errhand.VerboseError) { - r := env.NewRemote(remoteName, remoteUrl, params, dEnv) - - ddb, err := r.GetRemoteDB(ctx, types.Format_Default) - - if err != nil { - bdr := errhand.BuildDError("error: failed to get remote db").AddCause(err) - - if err == remotestorage.ErrInvalidDoltSpecPath { - urlObj, _ := earl.Parse(remoteUrl) - bdr.AddDetails("'%s' should be in the format 'organization/repo'", urlObj.Path) - } - - return env.NoRemote, nil, bdr.Build() - } - - return r, ddb, nil -} +} \ No newline at end of file From 83347ed36fc6123ddd2a06d168d00406bcf2eebf Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Thu, 14 Jul 2022 14:59:02 -0700 Subject: [PATCH 09/31] Couple typos and comments --- go/libraries/doltcore/sqle/database_provider.go | 1 + go/libraries/doltcore/sqle/dprocedures/dolt_clone.go | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/database_provider.go b/go/libraries/doltcore/sqle/database_provider.go index 8e55903eb55..f8b72ee1738 100644 --- a/go/libraries/doltcore/sqle/database_provider.go +++ b/go/libraries/doltcore/sqle/database_provider.go @@ -268,6 +268,7 @@ func (p DoltDatabaseProvider) CloneDatabaseFromRemote(ctx *sql.Context, dbName, return sess.AddDB(ctx, dbstate) } +// TODO: extract a shared library for this functionality func createRemote(ctx *sql.Context, remoteName, remoteUrl string, params map[string]string, dialer dbfactory.GRPCDialProvider) (env.Remote, *doltdb.DoltDB, error) { r := env.NewRemote(remoteName, remoteUrl, params, dialer) diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go index d48da301a5e..13b9dc169e2 100755 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go @@ -88,7 +88,7 @@ func getDirectoryAndUrlString(apr *argparser.ArgParseResults) (string, string, e if dir == "." { dir = path.Dir(urlStr) } else if dir == "/" { - return "", "", errhand.BuildDError("Could not infer repo name. Please explicitily define a directory for this url").Build() + return "", "", errhand.BuildDError("Could not infer repo name. Please explicitly define a directory for this url").Build() } } @@ -100,6 +100,7 @@ func remoteParams(apr *argparser.ArgParseResults, scheme, remoteUrl string) (map var err error if scheme == dbfactory.AWSScheme { + // TODO: get AWS params from session err = cli.AddAWSParams(remoteUrl, apr, params) } else { err = cli.VerifyNoAwsParams(apr) From 8bd4a0fd9068ea799009d625bd5ee7abb03ffedf Mon Sep 17 00:00:00 2001 From: zachmu Date: Thu, 14 Jul 2022 22:00:13 +0000 Subject: [PATCH 10/31] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/libraries/doltcore/sqle/dprocedures/dolt_clone.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go index 13b9dc169e2..458dfe32f80 100755 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go @@ -111,4 +111,4 @@ func remoteParams(apr *argparser.ArgParseResults, scheme, remoteUrl string) (map } return params, nil -} \ No newline at end of file +} From a597bebfb44267a6d4d7628e03b53a10240fd697 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Thu, 14 Jul 2022 15:48:44 -0700 Subject: [PATCH 11/31] Added bats test for dolt_clone procedure --- integration-tests/bats/remotes.bats | 36 +++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/integration-tests/bats/remotes.bats b/integration-tests/bats/remotes.bats index d30e79a965c..ab804118421 100644 --- a/integration-tests/bats/remotes.bats +++ b/integration-tests/bats/remotes.bats @@ -1941,3 +1941,39 @@ setup_ref_test() { [ "$status" -eq 0 ] [[ "$output" =~ "Fast-forward" ]] || false } + +@test "remotes: dolt_clone procedure" { + mkdir remote + mkdir repo1 + + cd repo1 + dolt init + dolt remote add origin file://../remote + dolt push origin main + dolt checkout -b other + dolt push --set-upstream origin other + + cd .. + + run dolt sql < Date: Thu, 14 Jul 2022 15:55:22 -0700 Subject: [PATCH 12/31] Restoring changes after merge conflict --- go/libraries/doltcore/sqle/database_provider.go | 2 +- go/libraries/doltcore/sqle/dsess/session.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go/libraries/doltcore/sqle/database_provider.go b/go/libraries/doltcore/sqle/database_provider.go index f8b72ee1738..a49c6486c3c 100644 --- a/go/libraries/doltcore/sqle/database_provider.go +++ b/go/libraries/doltcore/sqle/database_provider.go @@ -223,7 +223,7 @@ func (p DoltDatabaseProvider) CloneDatabaseFromRemote(ctx *sql.Context, dbName, var srcDB *doltdb.DoltDB dialer := p.remoteDialer if dialer == nil { - + // TODO: fill this in or error out } r, srcDB, err := createRemote(ctx, remoteName, remoteUrl, remoteParams, dialer) if err != nil { diff --git a/go/libraries/doltcore/sqle/dsess/session.go b/go/libraries/doltcore/sqle/dsess/session.go index 4de45092f4c..4993253fefb 100644 --- a/go/libraries/doltcore/sqle/dsess/session.go +++ b/go/libraries/doltcore/sqle/dsess/session.go @@ -66,7 +66,7 @@ type DoltSession struct { username string email string dbStates map[string]*DatabaseSessionState - provider RevisionDatabaseProvider + provider DoltDatabaseProvider tempTables map[string][]sql.Table globalsConf config.ReadWriteConfig mu *sync.Mutex @@ -76,7 +76,7 @@ var _ sql.Session = (*DoltSession)(nil) var _ sql.PersistableSession = (*DoltSession)(nil) // DefaultSession creates a DoltSession with default values -func DefaultSession(pro RevisionDatabaseProvider) *DoltSession { +func DefaultSession(pro DoltDatabaseProvider) *DoltSession { return &DoltSession{ Session: sql.NewBaseSession(), username: "", @@ -90,7 +90,7 @@ func DefaultSession(pro RevisionDatabaseProvider) *DoltSession { } // NewDoltSession creates a DoltSession object from a standard sql.Session and 0 or more Database objects. -func NewDoltSession(ctx *sql.Context, sqlSess *sql.BaseSession, pro RevisionDatabaseProvider, conf config.ReadWriteConfig, dbs ...InitialDbState) (*DoltSession, error) { +func NewDoltSession(ctx *sql.Context, sqlSess *sql.BaseSession, pro DoltDatabaseProvider, conf config.ReadWriteConfig, dbs ...InitialDbState) (*DoltSession, error) { username := conf.GetStringOrDefault(env.UserNameKey, "") email := conf.GetStringOrDefault(env.UserEmailKey, "") globals := config.NewPrefixConfig(conf, env.SqlServerGlobalsPrefix) @@ -117,7 +117,7 @@ func NewDoltSession(ctx *sql.Context, sqlSess *sql.BaseSession, pro RevisionData } // Provider returns the RevisionDatabaseProvider for this session. -func (d *DoltSession) Provider() RevisionDatabaseProvider { +func (d *DoltSession) Provider() DoltDatabaseProvider { return d.provider } From 9281576975c4ec74ee88ff0cfb078854bd2cc7e8 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Tue, 19 Jul 2022 11:11:33 -0700 Subject: [PATCH 13/31] Adding some godocs, error handling, and minor cleanup. --- go/libraries/doltcore/sqle/database_provider.go | 3 ++- go/libraries/doltcore/sqle/dprocedures/dolt_clone.go | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/libraries/doltcore/sqle/database_provider.go b/go/libraries/doltcore/sqle/database_provider.go index a49c6486c3c..711d1630032 100644 --- a/go/libraries/doltcore/sqle/database_provider.go +++ b/go/libraries/doltcore/sqle/database_provider.go @@ -208,6 +208,7 @@ func (p DoltDatabaseProvider) CreateDatabase(ctx *sql.Context, name string) erro return dsess.AddDB(ctx, dbstate) } +// CloneDatabaseFromRemote implements DoltDatabaseProvider interface func (p DoltDatabaseProvider) CloneDatabaseFromRemote(ctx *sql.Context, dbName, branch, remoteName, remoteUrl string, remoteParams map[string]string) error { p.mu.Lock() defer p.mu.Unlock() @@ -223,7 +224,7 @@ func (p DoltDatabaseProvider) CloneDatabaseFromRemote(ctx *sql.Context, dbName, var srcDB *doltdb.DoltDB dialer := p.remoteDialer if dialer == nil { - // TODO: fill this in or error out + return fmt.Errorf("unable to clone remote database; no remote dialer configured") } r, srcDB, err := createRemote(ctx, remoteName, remoteUrl, remoteParams, dialer) if err != nil { diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go index 458dfe32f80..bdaf7e7ddcf 100755 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go @@ -46,12 +46,11 @@ func doltClone(ctx *sql.Context, args ...string) (sql.RowIter, error) { sess := dsess.DSessFromSess(ctx.Session) scheme, remoteUrl, err := env.GetAbsRemoteUrl(sess.Provider().FileSystem(), emptyConfig(), urlStr) - if err != nil { return nil, errhand.BuildDError("error: '%s' is not valid.", urlStr).Build() } - var params map[string]string - params, err = remoteParams(apr, scheme, remoteUrl) + + params, err := remoteParams(apr, scheme, remoteUrl) if err != nil { return nil, err } From 9e7fbc0296e32df8301500f395ccfe119cb62a66 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Tue, 19 Jul 2022 14:23:44 -0700 Subject: [PATCH 14/31] whitespace cleanup --- go/libraries/doltcore/env/actions/clone.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/go/libraries/doltcore/env/actions/clone.go b/go/libraries/doltcore/env/actions/clone.go index ee7b0c459f6..6bcd625c45d 100644 --- a/go/libraries/doltcore/env/actions/clone.go +++ b/go/libraries/doltcore/env/actions/clone.go @@ -64,20 +64,17 @@ func EnvForClone(ctx context.Context, nbf *types.NomsBinFormat, r env.Remote, di } err := fs.MkDirs(dir) - if err != nil { return nil, fmt.Errorf("%w: %s; %s", ErrFailedToCreateDirectory, dir, err.Error()) } err = os.Chdir(dir) - if err != nil { return nil, fmt.Errorf("%w: %s; %s", ErrFailedToAccessDir, dir, err.Error()) } dEnv := env.Load(ctx, homeProvider, fs, doltdb.LocalDirDoltDB, version) err = dEnv.InitRepoWithNoData(ctx, nbf) - if err != nil { return nil, fmt.Errorf("%w; %s", ErrFailedToInitRepo, err.Error()) } @@ -85,7 +82,6 @@ func EnvForClone(ctx context.Context, nbf *types.NomsBinFormat, r env.Remote, di dEnv.RSLoadErr = nil if !env.IsEmptyRemote(r) { dEnv.RepoState, err = env.CloneRepoState(dEnv.FS, r) - if err != nil { return nil, fmt.Errorf("%w: %s; %s", ErrFailedToCreateRepoStateWithRemote, r.Name, err.Error()) } From 92db69c6438d0faac012b5a72dfade70c8e60b18 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Tue, 19 Jul 2022 14:28:45 -0700 Subject: [PATCH 15/31] Filling in empty error message; disabling SetPrintUsage, since this isn't a direct CLI command (and would cause SQL command usage to print); cleaning up whitespace. --- go/libraries/doltcore/sqle/dprocedures/dolt_clone.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go index bdaf7e7ddcf..713659980d7 100755 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go @@ -69,12 +69,11 @@ func emptyConfig() config.ReadableConfig { func getDirectoryAndUrlString(apr *argparser.ArgParseResults) (string, string, error) { if apr.NArg() < 1 || apr.NArg() > 2 { - return "", "", errhand.BuildDError("").SetPrintUsage().Build() + return "", "", errhand.BuildDError("error: invalid number of arguments: database URL must be specified and database name is optional").Build() } urlStr := apr.Arg(0) _, err := earl.Parse(urlStr) - if err != nil { return "", "", errhand.BuildDError("error: invalid remote url: " + urlStr).Build() } @@ -87,7 +86,7 @@ func getDirectoryAndUrlString(apr *argparser.ArgParseResults) (string, string, e if dir == "." { dir = path.Dir(urlStr) } else if dir == "/" { - return "", "", errhand.BuildDError("Could not infer repo name. Please explicitly define a directory for this url").Build() + return "", "", errhand.BuildDError("Could not infer repo name. Please explicitly define a directory for this url").Build() } } From 329f02b5791e16fb497e25ee901767ad54d3df0f Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Tue, 19 Jul 2022 14:30:18 -0700 Subject: [PATCH 16/31] Changing FileSystem.WithWorkingDir implementations to actually return a copy instead of modifying the original struct, like their docs describe. --- go/libraries/utils/filesys/inmemfs.go | 5 +++-- go/libraries/utils/filesys/localfs.go | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/go/libraries/utils/filesys/inmemfs.go b/go/libraries/utils/filesys/inmemfs.go index 9a4a2fea5ae..49c19b33f1e 100644 --- a/go/libraries/utils/filesys/inmemfs.go +++ b/go/libraries/utils/filesys/inmemfs.go @@ -144,8 +144,9 @@ func (fs InMemFS) WithWorkingDir(path string) (Filesys, error) { return nil, err } - fs.cwd = abs - return &fs, nil + newFs := fs + newFs.cwd = abs + return &newFs, nil } func (fs *InMemFS) getAbsPath(path string) string { diff --git a/go/libraries/utils/filesys/localfs.go b/go/libraries/utils/filesys/localfs.go index 551a0056b31..2fb14add254 100644 --- a/go/libraries/utils/filesys/localfs.go +++ b/go/libraries/utils/filesys/localfs.go @@ -79,8 +79,9 @@ func (fs localFS) WithWorkingDir(path string) (Filesys, error) { return nil, err } - fs.cwd = abs - return &fs, nil + newFs := fs + newFs.cwd = abs + return &newFs, nil } var errStopMarker = errors.New("stop") From b3d5592fb76ac06e4ac5712c3a36888eefc71953 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Tue, 19 Jul 2022 14:33:44 -0700 Subject: [PATCH 17/31] Hack to fix a bug where clone logic was changing the process' current working directory and causing incorrect behavior after running dolt_clone(). --- go/libraries/doltcore/sqle/database_provider.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/go/libraries/doltcore/sqle/database_provider.go b/go/libraries/doltcore/sqle/database_provider.go index 711d1630032..3b6e89570ac 100644 --- a/go/libraries/doltcore/sqle/database_provider.go +++ b/go/libraries/doltcore/sqle/database_provider.go @@ -17,6 +17,7 @@ package sqle import ( "context" "fmt" + "os" "strings" "sync" @@ -231,6 +232,16 @@ func (p DoltDatabaseProvider) CloneDatabaseFromRemote(ctx *sql.Context, dbName, return err } + // TODO: EnvForClone changes the working directory for the current process – this hack + // undoes that. We should audit the code and find a cleaner way to fix this, such + // as: 1) see if we really need to change the working directory, or 2) if that + // function should reset it afterwards, or 3) if it should be hidden behind a flag. + cwd, err := os.Getwd() + if err != nil { + return err + } + defer os.Chdir(cwd) + dEnv, err := actions.EnvForClone(ctx, srcDB.ValueReadWriter().Format(), r, dbName, p.fs, "VERSION", env.GetCurrentUserHomeDir) if err != nil { return err From 9526c9bd24c7874f152781fd56ccf0adcb699935 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Tue, 19 Jul 2022 16:01:01 -0700 Subject: [PATCH 18/31] Expanding on interface documentation --- go/libraries/doltcore/sqle/dsess/revision_db_provider.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/libraries/doltcore/sqle/dsess/revision_db_provider.go b/go/libraries/doltcore/sqle/dsess/revision_db_provider.go index b42894846c1..10643f2fc7c 100644 --- a/go/libraries/doltcore/sqle/dsess/revision_db_provider.go +++ b/go/libraries/doltcore/sqle/dsess/revision_db_provider.go @@ -38,9 +38,9 @@ type RevisionDatabaseProvider interface { type DoltDatabaseProvider interface { RevisionDatabaseProvider - // CloneDatabaseFromRemote clones the database from the remote given as a new database in this provider + // CloneDatabaseFromRemote clones the database from the specified URL as a new database in this provider. dbName is the name for the new database, branch is an optional parameter indicating which branch to clone (otherwise all branches are cloned), remoteName is the name for the remote created in the new database, and remoteUrl is a URL (e.g. "file:///dbs/db1") or an / path indicating a database hosted on DoltHub. CloneDatabaseFromRemote(ctx *sql.Context, dbName, branch, remoteName, remoteUrl string, remoteParams map[string]string) error - // FileSystem returns the filesystem used by this provider, rooted at the data directory for all databases + // FileSystem returns the filesystem used by this provider, rooted at the data directory for all databases. FileSystem() filesys.Filesys } From a0ea4ca760d61a0b2f7262447130aa7365fa0bf7 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Tue, 19 Jul 2022 19:52:24 -0700 Subject: [PATCH 19/31] Updated BATS tests --- integration-tests/bats/remotes.bats | 34 ++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/integration-tests/bats/remotes.bats b/integration-tests/bats/remotes.bats index ab804118421..4865be9edc6 100644 --- a/integration-tests/bats/remotes.bats +++ b/integration-tests/bats/remotes.bats @@ -1954,19 +1954,41 @@ setup_ref_test() { dolt push --set-upstream origin other cd .. - - run dolt sql < Date: Tue, 19 Jul 2022 19:55:27 -0700 Subject: [PATCH 20/31] Reverting extra copy --- go/libraries/utils/filesys/inmemfs.go | 5 ++--- go/libraries/utils/filesys/localfs.go | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/go/libraries/utils/filesys/inmemfs.go b/go/libraries/utils/filesys/inmemfs.go index 49c19b33f1e..9a4a2fea5ae 100644 --- a/go/libraries/utils/filesys/inmemfs.go +++ b/go/libraries/utils/filesys/inmemfs.go @@ -144,9 +144,8 @@ func (fs InMemFS) WithWorkingDir(path string) (Filesys, error) { return nil, err } - newFs := fs - newFs.cwd = abs - return &newFs, nil + fs.cwd = abs + return &fs, nil } func (fs *InMemFS) getAbsPath(path string) string { diff --git a/go/libraries/utils/filesys/localfs.go b/go/libraries/utils/filesys/localfs.go index 2fb14add254..551a0056b31 100644 --- a/go/libraries/utils/filesys/localfs.go +++ b/go/libraries/utils/filesys/localfs.go @@ -79,9 +79,8 @@ func (fs localFS) WithWorkingDir(path string) (Filesys, error) { return nil, err } - newFs := fs - newFs.cwd = abs - return &newFs, nil + fs.cwd = abs + return &fs, nil } var errStopMarker = errors.New("stop") From d122e0b5ec2889b4a436a43ae220f11235d45403 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Wed, 20 Jul 2022 13:50:50 -0700 Subject: [PATCH 21/31] Updating BATS tests to test remaining cases (single branch, overriding remote name, and cloning from DoltHub) --- integration-tests/bats/remotes.bats | 63 ++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 15 deletions(-) diff --git a/integration-tests/bats/remotes.bats b/integration-tests/bats/remotes.bats index 4865be9edc6..28c7df03dc2 100644 --- a/integration-tests/bats/remotes.bats +++ b/integration-tests/bats/remotes.bats @@ -1943,6 +1943,10 @@ setup_ref_test() { } @test "remotes: dolt_clone procedure" { + repoDir="$BATS_TMPDIR/dolt-repo-$$" + tempDir=$(mktemp -d) + + cd $tempDir mkdir remote mkdir repo1 @@ -1953,31 +1957,40 @@ setup_ref_test() { dolt checkout -b other dolt push --set-upstream origin other - cd .. + cd $repoDir run dolt sql -q "call dolt_clone()" [ "$status" -eq 1 ] [[ "$output" =~ "error: invalid number of arguments" ]] || false - run dolt sql -q "call dolt_clone('file://./remote', 'foo', 'bar')" + run dolt sql -q "call dolt_clone('file://$tempDir/remote', 'foo', 'bar')" [ "$status" -eq 1 ] [[ "$output" =~ "error: invalid number of arguments" ]] || false - # Clone a local database - run dolt sql -q "call dolt_clone('file://./remote', 'repo2');" + # Clone a local database and check for all the branches + run dolt sql -q "call dolt_clone('file://$tempDir/remote');" + [ "$status" -eq 0 ] + cd remote + run dolt branch [ "$status" -eq 0 ] + [[ ! "$output" =~ "other" ]] || false + [[ "$output" =~ "main" ]] || false + run dolt branch --remote + [[ "$output" =~ "origin/other" ]] || false + [[ "$output" =~ "origin/main" ]] || false + cd .. # Ensure we can't clone it again - run dolt sql -q "call dolt_clone('file://./remote', 'repo2');" + run dolt sql -q "call dolt_clone('file://$tempDir/remote');" [ "$status" -eq 1 ] - [[ "$output" =~ "can't create database repo2; database exists" ]] || false + [[ "$output" =~ "can't create database remote; database exists" ]] || false - # Drop the new database and re-clone it - dolt sql -q "drop database repo2" + # Drop the new database and re-clone it with a different name + dolt sql -q "drop database remote" run dolt sql -q "show databases" [ "$status" -eq 0 ] [[ ! "$output" =~ "repo2" ]] || false - dolt sql -q "call dolt_clone('file://./remote', 'repo2');" + dolt sql -q "call dolt_clone('file://$tempDir/remote', 'repo2');" # Sanity check that we can use the new database dolt sql << SQL @@ -1988,14 +2001,34 @@ SQL cd repo2 dolt commit -am "a commit for main from repo2" dolt push origin main + cd .. - run dolt branch - [[ ! "$output" =~ "other" ]] || false + # Test -remote option to customize the origin remote name for the cloned DB + run dolt sql -q "call dolt_clone('-remote', 'custom', 'file://$tempDir/remote', 'custom_remote');" + [ "$status" -eq 0 ] + run dolt sql -q "use custom_remote; select name from dolt_remotes;" + [ "$status" -eq 0 ] + [[ "$output" =~ "custom" ]] || false - run dolt checkout other + # Test -branch option to only clone a single branch + run dolt sql -q "call dolt_clone('-branch', 'other', 'file://$tempDir/remote', 'single_branch');" [ "$status" -eq 0 ] - [[ "$output" =~ "branch 'other' set up to track 'origin/other'." ]] || false + run dolt sql -q "use single_branch; select name from dolt_branches;" + [ "$status" -eq 0 ] + [[ "$output" =~ "other" ]] || false + [[ ! "$output" =~ "main" ]] || false + run dolt sql -q "use single_branch; select active_branch();" + [ "$status" -eq 0 ] + [[ "$output" =~ "other" ]] || false + # TODO: to match Git's semantics, clone for a single branch should NOT create any other remote tracking branches + # TODO: Issue link + # run dolt checkout main + # [ "$status" -eq 1 ] - run dolt status - [[ "$output" =~ "Your branch is up to date with 'origin/other'." ]] || false + # Test cloning from a DoltHub remote + run dolt sql -q "call dolt_clone('dolthub/VEID');" + [ "$status" -eq 0 ] + run dolt sql -q "use VEID; show tables;" + [ "$status" -eq 0 ] + [[ "$output" =~ "Stg_product2" ]] || false } From b6e6dd734a04cd6dd1e5398ec8382714161ff8b1 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Wed, 20 Jul 2022 14:00:39 -0700 Subject: [PATCH 22/31] Switching to use remotesrv instead of DoltHub for tests --- integration-tests/bats/remotes.bats | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/integration-tests/bats/remotes.bats b/integration-tests/bats/remotes.bats index 28c7df03dc2..a12d198f99b 100644 --- a/integration-tests/bats/remotes.bats +++ b/integration-tests/bats/remotes.bats @@ -2025,10 +2025,18 @@ SQL # run dolt checkout main # [ "$status" -eq 1 ] - # Test cloning from a DoltHub remote - run dolt sql -q "call dolt_clone('dolthub/VEID');" + # Set up a test repo in the remote server + cd repo2 + dolt remote add test-remote http://localhost:50051/test-org/test-repo + dolt sql -q "CREATE TABLE test_table (pk INT)" + dolt commit -am "main commit" + dolt push test-remote main + cd .. + + # Test cloning from a server remote + run dolt sql -q "call dolt_clone('http://localhost:50051/test-org/test-repo');" [ "$status" -eq 0 ] - run dolt sql -q "use VEID; show tables;" + run dolt sql -q "use test_repo; show tables;" [ "$status" -eq 0 ] - [[ "$output" =~ "Stg_product2" ]] || false + [[ "$output" =~ "test_table" ]] || false } From 5ea361e3c425c33ed4c1edcbe87354f06bc6c807 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Wed, 20 Jul 2022 14:26:00 -0700 Subject: [PATCH 23/31] Adding issue URL for not creating remote tracking branches when cloning a single branch. --- integration-tests/bats/remotes.bats | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-tests/bats/remotes.bats b/integration-tests/bats/remotes.bats index a12d198f99b..44c6eb72709 100644 --- a/integration-tests/bats/remotes.bats +++ b/integration-tests/bats/remotes.bats @@ -2020,8 +2020,8 @@ SQL run dolt sql -q "use single_branch; select active_branch();" [ "$status" -eq 0 ] [[ "$output" =~ "other" ]] || false - # TODO: to match Git's semantics, clone for a single branch should NOT create any other remote tracking branches - # TODO: Issue link + # TODO: To match Git's semantics, clone for a single branch should NOT create any other + # remote tracking branches (https://github.com/dolthub/dolt/issues/3873) # run dolt checkout main # [ "$status" -eq 1 ] From b6d48d09d432d737c21733590b4cc5d90292661c Mon Sep 17 00:00:00 2001 From: fulghum Date: Wed, 20 Jul 2022 22:57:54 +0000 Subject: [PATCH 24/31] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/libraries/doltcore/sqle/dsess/revision_db_provider.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/dsess/revision_db_provider.go b/go/libraries/doltcore/sqle/dsess/revision_db_provider.go index 960eaf3f723..a447968acc0 100644 --- a/go/libraries/doltcore/sqle/dsess/revision_db_provider.go +++ b/go/libraries/doltcore/sqle/dsess/revision_db_provider.go @@ -53,7 +53,6 @@ type DoltDatabaseProvider interface { CloneDatabaseFromRemote(ctx *sql.Context, dbName, branch, remoteName, remoteUrl string, remoteParams map[string]string) error } - func EmptyDatabaseProvider() DoltDatabaseProvider { return emptyRevisionDatabaseProvider{} } From abda8af763c8b3b1b765f306eb78eef66a68b253 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Wed, 20 Jul 2022 16:03:54 -0700 Subject: [PATCH 25/31] Fixing from dialer refactoring --- go/libraries/doltcore/sqle/database_provider.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/libraries/doltcore/sqle/database_provider.go b/go/libraries/doltcore/sqle/database_provider.go index 6526b481fed..a6be15fb196 100644 --- a/go/libraries/doltcore/sqle/database_provider.go +++ b/go/libraries/doltcore/sqle/database_provider.go @@ -289,9 +289,9 @@ func (p DoltDatabaseProvider) CloneDatabaseFromRemote(ctx *sql.Context, dbName, // TODO: extract a shared library for this functionality func createRemote(ctx *sql.Context, remoteName, remoteUrl string, params map[string]string, dialer dbfactory.GRPCDialProvider) (env.Remote, *doltdb.DoltDB, error) { - r := env.NewRemote(remoteName, remoteUrl, params, dialer) + r := env.NewRemote(remoteName, remoteUrl, params) - ddb, err := r.GetRemoteDB(ctx, types.Format_Default) + ddb, err := r.GetRemoteDB(ctx, types.Format_Default, dialer) if err != nil { bdr := errhand.BuildDError("error: failed to get remote db").AddCause(err) From e31d52f7437ac0d45d7c8b6171d7e984da9c7b6a Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Wed, 20 Jul 2022 16:04:06 -0700 Subject: [PATCH 26/31] Tweaking argument descriptions. --- go/cmd/dolt/cli/arg_parser_helpers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/cmd/dolt/cli/arg_parser_helpers.go b/go/cmd/dolt/cli/arg_parser_helpers.go index 9772249aedf..02e049473dc 100644 --- a/go/cmd/dolt/cli/arg_parser_helpers.go +++ b/go/cmd/dolt/cli/arg_parser_helpers.go @@ -151,8 +151,8 @@ func CreateAddArgParser() *argparser.ArgParser { func CreateCloneArgParser() *argparser.ArgParser { ap := argparser.NewArgParser() - ap.SupportsString(RemoteParam, "", "name", "Name of the remote to be added. Default will be 'origin'.") - ap.SupportsString(BranchParam, "b", "branch", "The branch to be cloned. If not specified all branches will be cloned.") + ap.SupportsString(RemoteParam, "", "name", "Name of the remote to be added to the cloned database. The default is 'origin'.") + ap.SupportsString(BranchParam, "b", "branch", "The branch to be cloned. If not specified all branches will be cloned.") ap.SupportsString(dbfactory.AWSRegionParam, "", "region", "") ap.SupportsValidatedString(dbfactory.AWSCredsTypeParam, "", "creds-type", "", argparser.ValidatorFromStrList(dbfactory.AWSCredsTypeParam, dbfactory.AWSCredTypes)) ap.SupportsString(dbfactory.AWSCredsFileParam, "", "file", "AWS credentials file.") From 24b2afb44a7033d9d14ef90ffae0c863597127d1 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Wed, 20 Jul 2022 16:05:15 -0700 Subject: [PATCH 27/31] Removing duplicate function definition (from merge) and cleaning up imports. --- .../doltcore/sqle/dprocedures/dolt_clone.go | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go index 713659980d7..2c965b74f79 100755 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go @@ -21,7 +21,6 @@ import ( "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/cmd/dolt/errhand" - "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" "github.com/dolthub/dolt/go/libraries/utils/argparser" @@ -92,21 +91,3 @@ func getDirectoryAndUrlString(apr *argparser.ArgParseResults) (string, string, e return dir, urlStr, nil } - -func remoteParams(apr *argparser.ArgParseResults, scheme, remoteUrl string) (map[string]string, errhand.VerboseError) { - params := map[string]string{} - - var err error - if scheme == dbfactory.AWSScheme { - // TODO: get AWS params from session - err = cli.AddAWSParams(remoteUrl, apr, params) - } else { - err = cli.VerifyNoAwsParams(apr) - } - - if err != nil { - return nil, errhand.VerboseErrorFromError(err) - } - - return params, nil -} From 44e4afa617c5ba5a510cfa0a82abd986e81595e3 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Thu, 21 Jul 2022 12:26:08 -0700 Subject: [PATCH 28/31] Removing os.Chdir from actions.Clone since this isn't safe in a server env. --- go/libraries/doltcore/env/actions/clone.go | 5 ++--- go/libraries/doltcore/env/environment.go | 2 +- go/libraries/doltcore/sqle/database_provider.go | 11 ----------- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/go/libraries/doltcore/env/actions/clone.go b/go/libraries/doltcore/env/actions/clone.go index 809d373cf6a..198788de4aa 100644 --- a/go/libraries/doltcore/env/actions/clone.go +++ b/go/libraries/doltcore/env/actions/clone.go @@ -18,7 +18,6 @@ import ( "context" "errors" "fmt" - "os" "path/filepath" "sort" "sync" @@ -68,12 +67,12 @@ func EnvForClone(ctx context.Context, nbf *types.NomsBinFormat, r env.Remote, di return nil, fmt.Errorf("%w: %s; %s", ErrFailedToCreateDirectory, dir, err.Error()) } - err = os.Chdir(dir) + newFs, err := fs.WithWorkingDir(dir) if err != nil { return nil, fmt.Errorf("%w: %s; %s", ErrFailedToAccessDir, dir, err.Error()) } - dEnv := env.Load(ctx, homeProvider, fs, doltdb.LocalDirDoltDB, version) + dEnv := env.Load(ctx, homeProvider, newFs, doltdb.LocalDirDoltDB, version) err = dEnv.InitRepoWithNoData(ctx, nbf) if err != nil { return nil, fmt.Errorf("%w; %s", ErrFailedToInitRepo, err.Error()) diff --git a/go/libraries/doltcore/env/environment.go b/go/libraries/doltcore/env/environment.go index 8f5f52da8a8..1172390a257 100644 --- a/go/libraries/doltcore/env/environment.go +++ b/go/libraries/doltcore/env/environment.go @@ -361,7 +361,7 @@ func (dEnv *DoltEnv) InitRepoWithNoData(ctx context.Context, nbf *types.NomsBinF return err } - dEnv.DoltDB, err = doltdb.LoadDoltDB(ctx, nbf, dEnv.urlStr, filesys.LocalFS) + dEnv.DoltDB, err = doltdb.LoadDoltDB(ctx, nbf, dEnv.urlStr, dEnv.FS) return err } diff --git a/go/libraries/doltcore/sqle/database_provider.go b/go/libraries/doltcore/sqle/database_provider.go index a6be15fb196..ded18ef172f 100644 --- a/go/libraries/doltcore/sqle/database_provider.go +++ b/go/libraries/doltcore/sqle/database_provider.go @@ -17,7 +17,6 @@ package sqle import ( "context" "fmt" - "os" "strings" "sync" @@ -239,16 +238,6 @@ func (p DoltDatabaseProvider) CloneDatabaseFromRemote(ctx *sql.Context, dbName, return err } - // TODO: EnvForClone changes the working directory for the current process – this hack - // undoes that. We should audit the code and find a cleaner way to fix this, such - // as: 1) see if we really need to change the working directory, or 2) if that - // function should reset it afterwards, or 3) if it should be hidden behind a flag. - cwd, err := os.Getwd() - if err != nil { - return err - } - defer os.Chdir(cwd) - dEnv, err := actions.EnvForClone(ctx, srcDB.ValueReadWriter().Format(), r, dbName, p.fs, "VERSION", env.GetCurrentUserHomeDir) if err != nil { return err From 26cfe9d454588e618461fb883990845386eb331e Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Thu, 21 Jul 2022 13:57:51 -0700 Subject: [PATCH 29/31] Removing more usage of os.Chdir --- go/cmd/dolt/commands/backup.go | 8 ++------ go/cmd/dolt/commands/clone.go | 9 +++------ 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/go/cmd/dolt/commands/backup.go b/go/cmd/dolt/commands/backup.go index b22998b797e..baff4b7c785 100644 --- a/go/cmd/dolt/commands/backup.go +++ b/go/cmd/dolt/commands/backup.go @@ -17,7 +17,6 @@ package commands import ( "context" "encoding/json" - "os" "strings" "github.com/dolthub/dolt/go/libraries/doltcore/env/actions" @@ -314,7 +313,7 @@ func restoreBackup(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgPar return errhand.VerboseErrorFromError(err) } - // make .dolt dir whith env.NoRemote to avoid origin upstream + // Create a new Dolt env for the clone and assign it to dEnv; use env.NoRemote to avoid origin upstream dEnv, err = actions.EnvForClone(ctx, srcDb.ValueReadWriter().Format(), env.NoRemote, dir, dEnv.FS, dEnv.Version, env.GetCurrentUserHomeDir) if err != nil { return errhand.VerboseErrorFromError(err) @@ -331,12 +330,9 @@ func restoreBackup(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgPar // If we're cloning into a directory that already exists do not erase it. Otherwise // make best effort to delete the directory we created. if userDirExists { - // Set the working dir to the parent of the .dolt folder so we can delete .dolt - _ = os.Chdir(dir) _ = dEnv.FS.Delete(dbfactory.DoltDir, true) } else { - _ = os.Chdir("../") - _ = dEnv.FS.Delete(dir, true) + _ = dEnv.FS.Delete(".", true) } return errhand.VerboseErrorFromError(err) } diff --git a/go/cmd/dolt/commands/clone.go b/go/cmd/dolt/commands/clone.go index b8ec45f6251..dd24c882a1b 100644 --- a/go/cmd/dolt/commands/clone.go +++ b/go/cmd/dolt/commands/clone.go @@ -16,7 +16,6 @@ package commands import ( "context" - "os" "path" "github.com/dolthub/dolt/go/cmd/dolt/cli" @@ -120,6 +119,7 @@ func clone(ctx context.Context, apr *argparser.ArgParseResults, dEnv *env.DoltEn return verr } + // Create a new Dolt env for the clone and assign it to dEnv dEnv, err = actions.EnvForClone(ctx, srcDB.ValueReadWriter().Format(), r, dir, dEnv.FS, dEnv.Version, env.GetCurrentUserHomeDir) if err != nil { return errhand.VerboseErrorFromError(err) @@ -130,12 +130,9 @@ func clone(ctx context.Context, apr *argparser.ArgParseResults, dEnv *env.DoltEn // If we're cloning into a directory that already exists do not erase it. Otherwise // make best effort to delete the directory we created. if userDirExists { - // Set the working dir to the parent of the .dolt folder so we can delete .dolt - _ = os.Chdir(dir) - _ = dEnv.FS.Delete(dbfactory.DoltDir, true) + dEnv.FS.Delete(dbfactory.DoltDir, true) } else { - _ = os.Chdir("../") - _ = dEnv.FS.Delete(dir, true) + dEnv.FS.Delete(".", true) } return errhand.VerboseErrorFromError(err) } From 0be183b3151b0d08c5c74b773b5c937f38d8565a Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Thu, 21 Jul 2022 14:52:17 -0700 Subject: [PATCH 30/31] Renaming DoltEnv variables to be more explicit and clear --- go/cmd/dolt/commands/backup.go | 15 +++++++++------ go/cmd/dolt/commands/clone.go | 19 +++++++++++-------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/go/cmd/dolt/commands/backup.go b/go/cmd/dolt/commands/backup.go index baff4b7c785..98b732f86d4 100644 --- a/go/cmd/dolt/commands/backup.go +++ b/go/cmd/dolt/commands/backup.go @@ -313,26 +313,29 @@ func restoreBackup(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgPar return errhand.VerboseErrorFromError(err) } - // Create a new Dolt env for the clone and assign it to dEnv; use env.NoRemote to avoid origin upstream - dEnv, err = actions.EnvForClone(ctx, srcDb.ValueReadWriter().Format(), env.NoRemote, dir, dEnv.FS, dEnv.Version, env.GetCurrentUserHomeDir) + // Create a new Dolt env for the clone; use env.NoRemote to avoid origin upstream + clonedEnv, err := actions.EnvForClone(ctx, srcDb.ValueReadWriter().Format(), env.NoRemote, dir, dEnv.FS, dEnv.Version, env.GetCurrentUserHomeDir) if err != nil { return errhand.VerboseErrorFromError(err) } + // Nil out the old Dolt env so we don't accidentally use the wrong database + dEnv = nil + // still make empty repo state - _, err = env.CreateRepoState(dEnv.FS, env.DefaultInitBranch) + _, err = env.CreateRepoState(clonedEnv.FS, env.DefaultInitBranch) if err != nil { return errhand.VerboseErrorFromError(err) } - err = actions.SyncRoots(ctx, srcDb, dEnv.DoltDB, dEnv.TempTableFilesDir(), buildProgStarter(downloadLanguage), stopProgFuncs) + err = actions.SyncRoots(ctx, srcDb, clonedEnv.DoltDB, clonedEnv.TempTableFilesDir(), buildProgStarter(downloadLanguage), stopProgFuncs) if err != nil { // If we're cloning into a directory that already exists do not erase it. Otherwise // make best effort to delete the directory we created. if userDirExists { - _ = dEnv.FS.Delete(dbfactory.DoltDir, true) + _ = clonedEnv.FS.Delete(dbfactory.DoltDir, true) } else { - _ = dEnv.FS.Delete(".", true) + _ = clonedEnv.FS.Delete(".", true) } return errhand.VerboseErrorFromError(err) } diff --git a/go/cmd/dolt/commands/clone.go b/go/cmd/dolt/commands/clone.go index dd24c882a1b..19bca6fe816 100644 --- a/go/cmd/dolt/commands/clone.go +++ b/go/cmd/dolt/commands/clone.go @@ -119,20 +119,23 @@ func clone(ctx context.Context, apr *argparser.ArgParseResults, dEnv *env.DoltEn return verr } - // Create a new Dolt env for the clone and assign it to dEnv - dEnv, err = actions.EnvForClone(ctx, srcDB.ValueReadWriter().Format(), r, dir, dEnv.FS, dEnv.Version, env.GetCurrentUserHomeDir) + // Create a new Dolt env for the clone + clonedEnv, err := actions.EnvForClone(ctx, srcDB.ValueReadWriter().Format(), r, dir, dEnv.FS, dEnv.Version, env.GetCurrentUserHomeDir) if err != nil { return errhand.VerboseErrorFromError(err) } - err = actions.CloneRemote(ctx, srcDB, remoteName, branch, dEnv) + // Nil out the old Dolt env so we don't accidentally operate on the wrong database + dEnv = nil + + err = actions.CloneRemote(ctx, srcDB, remoteName, branch, clonedEnv) if err != nil { // If we're cloning into a directory that already exists do not erase it. Otherwise // make best effort to delete the directory we created. if userDirExists { - dEnv.FS.Delete(dbfactory.DoltDir, true) + clonedEnv.FS.Delete(dbfactory.DoltDir, true) } else { - dEnv.FS.Delete(".", true) + clonedEnv.FS.Delete(".", true) } return errhand.VerboseErrorFromError(err) } @@ -145,15 +148,15 @@ func clone(ctx context.Context, apr *argparser.ArgParseResults, dEnv *env.DoltEn } } - err = dEnv.RepoStateWriter().UpdateBranch(dEnv.RepoState.CWBHeadRef().GetPath(), env.BranchConfig{ - Merge: dEnv.RepoState.Head, + err = clonedEnv.RepoStateWriter().UpdateBranch(clonedEnv.RepoState.CWBHeadRef().GetPath(), env.BranchConfig{ + Merge: clonedEnv.RepoState.Head, Remote: remoteName, }) if err != nil { return errhand.VerboseErrorFromError(err) } - err = dEnv.RepoState.Save(dEnv.FS) + err = clonedEnv.RepoState.Save(clonedEnv.FS) if err != nil { return errhand.VerboseErrorFromError(err) } From b1613683f71c105c357497c178472a16f0301c97 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Thu, 21 Jul 2022 15:03:51 -0700 Subject: [PATCH 31/31] Updating godocs to better explain functions --- go/libraries/doltcore/env/actions/clone.go | 1 + go/libraries/doltcore/env/environment.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/go/libraries/doltcore/env/actions/clone.go b/go/libraries/doltcore/env/actions/clone.go index 198788de4aa..7dfc76be190 100644 --- a/go/libraries/doltcore/env/actions/clone.go +++ b/go/libraries/doltcore/env/actions/clone.go @@ -55,6 +55,7 @@ var ErrUserNotFound = errors.New("could not determine user name. run dolt config var ErrEmailNotFound = errors.New("could not determine email. run dolt config --global --add user.email") var ErrCloneFailed = errors.New("clone failed") +// EnvForClone creates a new DoltEnv and configures it with repo state from the specified remote. The returned DoltEnv is ready for content to be cloned into it. The directory used for the new DoltEnv is determined by resolving the specified dir against the specified Filesys. func EnvForClone(ctx context.Context, nbf *types.NomsBinFormat, r env.Remote, dir string, fs filesys.Filesys, version string, homeProvider env.HomeDirProvider) (*env.DoltEnv, error) { exists, _ := fs.Exists(filepath.Join(dir, dbfactory.DoltDir)) diff --git a/go/libraries/doltcore/env/environment.go b/go/libraries/doltcore/env/environment.go index 1172390a257..eb76a013915 100644 --- a/go/libraries/doltcore/env/environment.go +++ b/go/libraries/doltcore/env/environment.go @@ -98,7 +98,7 @@ type DoltEnv struct { IgnoreLockFile bool } -// Load loads the DoltEnv for the current directory of the cli +// Load loads the DoltEnv for the .dolt directory determined by resolving the specified urlStr with the specified Filesys. func Load(ctx context.Context, hdp HomeDirProvider, fs filesys.Filesys, urlStr, version string) *DoltEnv { config, cfgErr := LoadDoltCliConfig(hdp, fs) repoState, rsErr := LoadRepoState(fs)