From 621f6ceb6ba41b4fd72acf923c63e6f510911007 Mon Sep 17 00:00:00 2001 From: Pascal Breuninger Date: Sun, 12 May 2024 17:21:46 +0200 Subject: [PATCH] feat(cli): add different git cloning strategies and respective CLI flag --git-clone-strategy for git based workspaces. Possible values are: - "" (full clone, default behaviour) - blobless (full commit history and trees, no blobs) - treeless (full commit history, no trees nor blobs) - shallow (current HEAD, no history) --- cmd/agent/workspace/install_dotfiles.go | 2 +- cmd/agent/workspace/up.go | 7 +- cmd/build.go | 1 + cmd/helper/get_workspace_config.go | 2 +- cmd/up.go | 1 + pkg/git/clone.go | 111 ++++++++++++++++++++++++ pkg/git/git.go | 21 ++--- pkg/provider/workspace.go | 37 ++++---- 8 files changed, 149 insertions(+), 33 deletions(-) create mode 100644 pkg/git/clone.go diff --git a/cmd/agent/workspace/install_dotfiles.go b/cmd/agent/workspace/install_dotfiles.go index 5f4267b14..a6e3cc04a 100644 --- a/cmd/agent/workspace/install_dotfiles.go +++ b/cmd/agent/workspace/install_dotfiles.go @@ -51,7 +51,7 @@ func (cmd *InstallDotfilesCmd) Run(ctx context.Context) error { logger.Infof("Cloning dotfiles %s", cmd.Repository) gitInfo := git.NormalizeRepositoryGitInfo(cmd.Repository) - err = git.CloneRepository(ctx, gitInfo, targetDir, "", false, logger.Writer(logrus.DebugLevel, false), logger) + err = git.CloneRepository(ctx, gitInfo, targetDir, "", false, nil, logger.Writer(logrus.DebugLevel, false), logger) if err != nil { return err } diff --git a/cmd/agent/workspace/up.go b/cmd/agent/workspace/up.go index c3d1f0f5b..d5cc72774 100644 --- a/cmd/agent/workspace/up.go +++ b/cmd/agent/workspace/up.go @@ -201,7 +201,8 @@ func prepareWorkspace(ctx context.Context, workspaceInfo *provider2.AgentWorkspa } log.Debugf("Clone Repository") - err = CloneRepository(ctx, workspaceInfo.Agent.Local == "true", workspaceInfo.ContentFolder, workspaceInfo.Workspace.Source, helper, log) + gitCloner := git.NewCloner(workspaceInfo.CLIOptions.GitCloneStrategy) + err = CloneRepository(ctx, workspaceInfo.Agent.Local == "true", workspaceInfo.ContentFolder, workspaceInfo.Workspace.Source, helper, gitCloner, log) if err != nil { // fallback log.Errorf("Cloning failed: %v. Trying cloning on local machine and uploading folder", err) @@ -406,7 +407,7 @@ func (cmd *UpCmd) devPodUp(ctx context.Context, workspaceInfo *provider2.AgentWo return result, nil } -func CloneRepository(ctx context.Context, local bool, workspaceDir string, source provider2.WorkspaceSource, helper string, log log.Logger) error { +func CloneRepository(ctx context.Context, local bool, workspaceDir string, source provider2.WorkspaceSource, helper string, cloner git.Cloner, log log.Logger) error { // remove the credential helper or otherwise we will receive strange errors within the container defer func() { if helper != "" { @@ -478,7 +479,7 @@ func CloneRepository(ctx context.Context, local bool, workspaceDir string, sourc // run git command gitInfo := git.NewGitInfo(source.GitRepository, source.GitBranch, source.GitCommit, source.GitPRReference, source.GitSubPath) - err := git.CloneRepository(ctx, gitInfo, workspaceDir, helper, false, writer, log) + err := git.CloneRepository(ctx, gitInfo, workspaceDir, helper, false, cloner, writer, log) if err != nil { return errors.Wrap(err, "clone repository") } diff --git a/cmd/build.go b/cmd/build.go index 64d57fcb5..f58d82021 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -117,6 +117,7 @@ func NewBuildCmd(flags *flags.GlobalFlags) *cobra.Command { buildCmd.Flags().BoolVar(&cmd.SkipPush, "skip-push", false, "If true will not push the image to the repository, useful for testing") buildCmd.Flags().StringVar(&cmd.GitBranch, "git-branch", "", "The git branch to use") buildCmd.Flags().StringVar(&cmd.GitCommit, "git-commit", "", "The git commit SHA to use") + buildCmd.Flags().Var(&cmd.GitCloneStrategy, "git-clone-strategy", "The git clone strategy DevPod uses to checkout git based workspaces. Can be full (default), blobless, treeless or shallow") // TESTING buildCmd.Flags().BoolVar(&cmd.ForceBuild, "force-build", false, "TESTING ONLY") diff --git a/cmd/helper/get_workspace_config.go b/cmd/helper/get_workspace_config.go index 8a568f91b..c1a3becc1 100644 --- a/cmd/helper/get_workspace_config.go +++ b/cmd/helper/get_workspace_config.go @@ -153,7 +153,7 @@ func findDevcontainerFiles(ctx context.Context, rawSource, tmpDirPath string, ma gitInfo := git.NewGitInfo(gitRepository, gitBranch, gitCommit, gitPRReference, gitSubDir) log.Debugf("Cloning git repository into %s", tmpDirPath) - err := git.CloneRepository(ctx, gitInfo, tmpDirPath, "", true, log.Writer(logrus.DebugLevel, false), log) + err := git.CloneRepository(ctx, gitInfo, tmpDirPath, "", true, git.NewCloner(git.ShallowCloneStrategy), log.Writer(logrus.DebugLevel, false), log) if err != nil { return nil, err } diff --git a/cmd/up.go b/cmd/up.go index 12f8d8fb2..9f85010f2 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -149,6 +149,7 @@ func NewUpCmd(flags *flags.GlobalFlags) *cobra.Command { upCmd.Flags().BoolVar(&cmd.OpenIDE, "open-ide", true, "If this is false and an IDE is configured, DevPod will only install the IDE server backend, but not open it") upCmd.Flags().StringVar(&cmd.GitBranch, "git-branch", "", "The git branch to use") upCmd.Flags().StringVar(&cmd.GitCommit, "git-commit", "", "The git commit SHA to use") + upCmd.Flags().Var(&cmd.GitCloneStrategy, "git-clone-strategy", "The git clone strategy DevPod uses to checkout git based workspaces. Can be full (default), blobless, treeless or shallow") upCmd.Flags().StringVar(&cmd.FallbackImage, "fallback-image", "", "The fallback image to use if no devcontainer configuration has been detected") upCmd.Flags().BoolVar(&cmd.DisableDaemon, "disable-daemon", false, "If enabled, will not install a daemon into the target machine to track activity") diff --git a/pkg/git/clone.go b/pkg/git/clone.go new file mode 100644 index 000000000..93228c9a7 --- /dev/null +++ b/pkg/git/clone.go @@ -0,0 +1,111 @@ +package git + +import ( + "context" + "fmt" + "io" + + "github.com/spf13/pflag" +) + +type CloneStrategy string + +const ( + FullCloneStrategy CloneStrategy = "" + BloblessCloneStrategy CloneStrategy = "blobless" + TreelessCloneStrategy CloneStrategy = "treeless" + ShallowCloneStrategy CloneStrategy = "shallow" +) + +type Cloner interface { + Clone(ctx context.Context, repository string, targetDir string, extraArgs []string, stdout, stderr io.Writer) error +} + +func NewCloner(strategy CloneStrategy) Cloner { + switch strategy { + case BloblessCloneStrategy: + return &bloblessClone{} + case TreelessCloneStrategy: + return &treelessClone{} + case ShallowCloneStrategy: + return &shallowClone{} + case FullCloneStrategy: + return &fullClone{} + default: + return &fullClone{} + } +} + +var _ pflag.Value = (*CloneStrategy)(nil) + +func (s *CloneStrategy) Set(v string) error { + switch v { + case string(FullCloneStrategy), + string(BloblessCloneStrategy), + string(TreelessCloneStrategy), + string(ShallowCloneStrategy): + { + *s = CloneStrategy(v) + return nil + } + default: + return fmt.Errorf("CloneStrategy %s not supported", v) + } +} +func (s *CloneStrategy) Type() string { + return "cloneStrategy" +} +func (s *CloneStrategy) String() string { + return string(*s) +} + +type fullClone struct{} + +var _ Cloner = &fullClone{} + +func (c *fullClone) Clone(ctx context.Context, repository string, targetDir string, extraArgs []string, stdout, stderr io.Writer) error { + args := []string{"clone"} + args = append(args, extraArgs...) + args = append(args, repository, targetDir) + return run(ctx, args, stdout, stderr) +} + +type bloblessClone struct{} + +var _ Cloner = &bloblessClone{} + +func (c *bloblessClone) Clone(ctx context.Context, repository string, targetDir string, extraArgs []string, stdout, stderr io.Writer) error { + args := []string{"clone", "--filter=blob:none"} + args = append(args, extraArgs...) + args = append(args, repository, targetDir) + return run(ctx, args, stdout, stderr) +} + +type treelessClone struct{} + +var _ Cloner = treelessClone{} + +func (c treelessClone) Clone(ctx context.Context, repository string, targetDir string, extraArgs []string, stdout, stderr io.Writer) error { + args := []string{"clone", "--filter=tree:0"} + args = append(args, extraArgs...) + args = append(args, repository, targetDir) + return run(ctx, args, stdout, stderr) +} + +type shallowClone struct{} + +var _ Cloner = shallowClone{} + +func (c shallowClone) Clone(ctx context.Context, repository string, targetDir string, extraArgs []string, stdout, stderr io.Writer) error { + args := []string{"clone", "--depth=1"} + args = append(args, extraArgs...) + args = append(args, repository, targetDir) + return run(ctx, args, stdout, stderr) +} + +func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { + gitCommand := CommandContext(ctx, args...) + gitCommand.Stdout = stdout + gitCommand.Stderr = stderr + return gitCommand.Run() +} diff --git a/pkg/git/git.go b/pkg/git/git.go index ba27bee9a..ca4547ff7 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -114,22 +114,23 @@ func NormalizeRepositoryGitInfo(str string) *GitInfo { return NewGitInfo(repository, branch, commit, pr, subpath) } -func CloneRepository(ctx context.Context, gitInfo *GitInfo, targetDir string, helper string, bare bool, writer io.Writer, log log.Logger) error { - args := []string{"clone"} +func CloneRepository(ctx context.Context, gitInfo *GitInfo, targetDir string, helper string, bare bool, cloner Cloner, writer io.Writer, log log.Logger) error { + if cloner == nil { + cloner = NewCloner(FullCloneStrategy) + } + + extraArgs := []string{} if bare && gitInfo.Commit == "" { - args = append(args, "--bare", "--depth=1") + extraArgs = append(extraArgs, "--bare", "--depth=1") } if helper != "" { - args = append(args, "--config", "credential.helper="+helper) + extraArgs = append(extraArgs, "--config", "credential.helper="+helper) } if gitInfo.Branch != "" { - args = append(args, "--branch", gitInfo.Branch) + extraArgs = append(extraArgs, "--branch", gitInfo.Branch) } - args = append(args, gitInfo.Repository, targetDir) - gitCommand := CommandContext(ctx, args...) - gitCommand.Stdout = writer - gitCommand.Stderr = writer - err := gitCommand.Run() + + err := cloner.Clone(ctx, gitInfo.Repository, targetDir, extraArgs, writer, writer) if err != nil { return errors.Wrap(err, "error cloning repository") } diff --git a/pkg/provider/workspace.go b/pkg/provider/workspace.go index cc155ba81..ebc200193 100644 --- a/pkg/provider/workspace.go +++ b/pkg/provider/workspace.go @@ -164,24 +164,25 @@ type AgentWorkspaceInfo struct { type CLIOptions struct { // up options - ID string `json:"id,omitempty"` - Source string `json:"source,omitempty"` - IDE string `json:"ide,omitempty"` - IDEOptions []string `json:"ideOptions,omitempty"` - PrebuildRepositories []string `json:"prebuildRepositories,omitempty"` - DevContainerImage string `json:"devContainerImage,omitempty"` - DevContainerPath string `json:"devContainerPath,omitempty"` - WorkspaceEnv []string `json:"workspaceEnv,omitempty"` - WorkspaceEnvFile []string `json:"workspaceEnvFile,omitempty"` - Recreate bool `json:"recreate,omitempty"` - Reset bool `json:"reset,omitempty"` - Proxy bool `json:"proxy,omitempty"` - DisableDaemon bool `json:"disableDaemon,omitempty"` - DaemonInterval string `json:"daemonInterval,omitempty"` - ForceCredentials bool `json:"forceCredentials,omitempty"` - GitBranch string `json:"gitBranch,omitempty"` - GitCommit string `json:"gitCommit,omitempty"` - FallbackImage string `json:"fallbackImage,omitempty"` + ID string `json:"id,omitempty"` + Source string `json:"source,omitempty"` + IDE string `json:"ide,omitempty"` + IDEOptions []string `json:"ideOptions,omitempty"` + PrebuildRepositories []string `json:"prebuildRepositories,omitempty"` + DevContainerImage string `json:"devContainerImage,omitempty"` + DevContainerPath string `json:"devContainerPath,omitempty"` + WorkspaceEnv []string `json:"workspaceEnv,omitempty"` + WorkspaceEnvFile []string `json:"workspaceEnvFile,omitempty"` + Recreate bool `json:"recreate,omitempty"` + Reset bool `json:"reset,omitempty"` + Proxy bool `json:"proxy,omitempty"` + DisableDaemon bool `json:"disableDaemon,omitempty"` + DaemonInterval string `json:"daemonInterval,omitempty"` + ForceCredentials bool `json:"forceCredentials,omitempty"` + GitBranch string `json:"gitBranch,omitempty"` + GitCommit string `json:"gitCommit,omitempty"` + GitCloneStrategy git.CloneStrategy `json:"gitCloneStrategy,omitempty"` + FallbackImage string `json:"fallbackImage,omitempty"` // build options Repository string `json:"repository,omitempty"`