Skip to content

Commit

Permalink
feat(cli): add different git cloning strategies and respective CLI flag
Browse files Browse the repository at this point in the history
--git-clone-strategy for git based workspaces.
Possible values are:
- "" (full clone, default behaviour)
- blobless (full commit history and trees, no blobs)
- treeless (full commit history, no trees nor blobs)
- shallow (current HEAD, no history)
  • Loading branch information
pascalbreuninger committed May 13, 2024
1 parent d73cd70 commit 621f6ce
Show file tree
Hide file tree
Showing 8 changed files with 149 additions and 33 deletions.
2 changes: 1 addition & 1 deletion cmd/agent/workspace/install_dotfiles.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func (cmd *InstallDotfilesCmd) Run(ctx context.Context) error {
logger.Infof("Cloning dotfiles %s", cmd.Repository)

gitInfo := git.NormalizeRepositoryGitInfo(cmd.Repository)
err = git.CloneRepository(ctx, gitInfo, targetDir, "", false, logger.Writer(logrus.DebugLevel, false), logger)
err = git.CloneRepository(ctx, gitInfo, targetDir, "", false, nil, logger.Writer(logrus.DebugLevel, false), logger)
if err != nil {
return err
}
Expand Down
7 changes: 4 additions & 3 deletions cmd/agent/workspace/up.go
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,8 @@ func prepareWorkspace(ctx context.Context, workspaceInfo *provider2.AgentWorkspa
}

log.Debugf("Clone Repository")
err = CloneRepository(ctx, workspaceInfo.Agent.Local == "true", workspaceInfo.ContentFolder, workspaceInfo.Workspace.Source, helper, log)
gitCloner := git.NewCloner(workspaceInfo.CLIOptions.GitCloneStrategy)
err = CloneRepository(ctx, workspaceInfo.Agent.Local == "true", workspaceInfo.ContentFolder, workspaceInfo.Workspace.Source, helper, gitCloner, log)
if err != nil {
// fallback
log.Errorf("Cloning failed: %v. Trying cloning on local machine and uploading folder", err)
Expand Down Expand Up @@ -406,7 +407,7 @@ func (cmd *UpCmd) devPodUp(ctx context.Context, workspaceInfo *provider2.AgentWo
return result, nil
}

func CloneRepository(ctx context.Context, local bool, workspaceDir string, source provider2.WorkspaceSource, helper string, log log.Logger) error {
func CloneRepository(ctx context.Context, local bool, workspaceDir string, source provider2.WorkspaceSource, helper string, cloner git.Cloner, log log.Logger) error {
// remove the credential helper or otherwise we will receive strange errors within the container
defer func() {
if helper != "" {
Expand Down Expand Up @@ -478,7 +479,7 @@ func CloneRepository(ctx context.Context, local bool, workspaceDir string, sourc

// run git command
gitInfo := git.NewGitInfo(source.GitRepository, source.GitBranch, source.GitCommit, source.GitPRReference, source.GitSubPath)
err := git.CloneRepository(ctx, gitInfo, workspaceDir, helper, false, writer, log)
err := git.CloneRepository(ctx, gitInfo, workspaceDir, helper, false, cloner, writer, log)
if err != nil {
return errors.Wrap(err, "clone repository")
}
Expand Down
1 change: 1 addition & 0 deletions cmd/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ func NewBuildCmd(flags *flags.GlobalFlags) *cobra.Command {
buildCmd.Flags().BoolVar(&cmd.SkipPush, "skip-push", false, "If true will not push the image to the repository, useful for testing")
buildCmd.Flags().StringVar(&cmd.GitBranch, "git-branch", "", "The git branch to use")
buildCmd.Flags().StringVar(&cmd.GitCommit, "git-commit", "", "The git commit SHA to use")
buildCmd.Flags().Var(&cmd.GitCloneStrategy, "git-clone-strategy", "The git clone strategy DevPod uses to checkout git based workspaces. Can be full (default), blobless, treeless or shallow")

// TESTING
buildCmd.Flags().BoolVar(&cmd.ForceBuild, "force-build", false, "TESTING ONLY")
Expand Down
2 changes: 1 addition & 1 deletion cmd/helper/get_workspace_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ func findDevcontainerFiles(ctx context.Context, rawSource, tmpDirPath string, ma

gitInfo := git.NewGitInfo(gitRepository, gitBranch, gitCommit, gitPRReference, gitSubDir)
log.Debugf("Cloning git repository into %s", tmpDirPath)
err := git.CloneRepository(ctx, gitInfo, tmpDirPath, "", true, log.Writer(logrus.DebugLevel, false), log)
err := git.CloneRepository(ctx, gitInfo, tmpDirPath, "", true, git.NewCloner(git.ShallowCloneStrategy), log.Writer(logrus.DebugLevel, false), log)
if err != nil {
return nil, err
}
Expand Down
1 change: 1 addition & 0 deletions cmd/up.go
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ func NewUpCmd(flags *flags.GlobalFlags) *cobra.Command {
upCmd.Flags().BoolVar(&cmd.OpenIDE, "open-ide", true, "If this is false and an IDE is configured, DevPod will only install the IDE server backend, but not open it")
upCmd.Flags().StringVar(&cmd.GitBranch, "git-branch", "", "The git branch to use")
upCmd.Flags().StringVar(&cmd.GitCommit, "git-commit", "", "The git commit SHA to use")
upCmd.Flags().Var(&cmd.GitCloneStrategy, "git-clone-strategy", "The git clone strategy DevPod uses to checkout git based workspaces. Can be full (default), blobless, treeless or shallow")
upCmd.Flags().StringVar(&cmd.FallbackImage, "fallback-image", "", "The fallback image to use if no devcontainer configuration has been detected")

upCmd.Flags().BoolVar(&cmd.DisableDaemon, "disable-daemon", false, "If enabled, will not install a daemon into the target machine to track activity")
Expand Down
111 changes: 111 additions & 0 deletions pkg/git/clone.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
package git

import (
"context"
"fmt"
"io"

"github.com/spf13/pflag"
)

type CloneStrategy string

const (
FullCloneStrategy CloneStrategy = ""
BloblessCloneStrategy CloneStrategy = "blobless"
TreelessCloneStrategy CloneStrategy = "treeless"
ShallowCloneStrategy CloneStrategy = "shallow"
)

type Cloner interface {
Clone(ctx context.Context, repository string, targetDir string, extraArgs []string, stdout, stderr io.Writer) error
}

func NewCloner(strategy CloneStrategy) Cloner {
switch strategy {
case BloblessCloneStrategy:
return &bloblessClone{}
case TreelessCloneStrategy:
return &treelessClone{}
case ShallowCloneStrategy:
return &shallowClone{}
case FullCloneStrategy:
return &fullClone{}
default:
return &fullClone{}
}
}

var _ pflag.Value = (*CloneStrategy)(nil)

func (s *CloneStrategy) Set(v string) error {
switch v {
case string(FullCloneStrategy),
string(BloblessCloneStrategy),
string(TreelessCloneStrategy),
string(ShallowCloneStrategy):
{
*s = CloneStrategy(v)
return nil
}
default:
return fmt.Errorf("CloneStrategy %s not supported", v)
}
}
func (s *CloneStrategy) Type() string {
return "cloneStrategy"
}
func (s *CloneStrategy) String() string {
return string(*s)
}

type fullClone struct{}

var _ Cloner = &fullClone{}

func (c *fullClone) Clone(ctx context.Context, repository string, targetDir string, extraArgs []string, stdout, stderr io.Writer) error {
args := []string{"clone"}
args = append(args, extraArgs...)
args = append(args, repository, targetDir)
return run(ctx, args, stdout, stderr)
}

type bloblessClone struct{}

var _ Cloner = &bloblessClone{}

func (c *bloblessClone) Clone(ctx context.Context, repository string, targetDir string, extraArgs []string, stdout, stderr io.Writer) error {
args := []string{"clone", "--filter=blob:none"}
args = append(args, extraArgs...)
args = append(args, repository, targetDir)
return run(ctx, args, stdout, stderr)
}

type treelessClone struct{}

var _ Cloner = treelessClone{}

func (c treelessClone) Clone(ctx context.Context, repository string, targetDir string, extraArgs []string, stdout, stderr io.Writer) error {
args := []string{"clone", "--filter=tree:0"}
args = append(args, extraArgs...)
args = append(args, repository, targetDir)
return run(ctx, args, stdout, stderr)
}

type shallowClone struct{}

var _ Cloner = shallowClone{}

func (c shallowClone) Clone(ctx context.Context, repository string, targetDir string, extraArgs []string, stdout, stderr io.Writer) error {
args := []string{"clone", "--depth=1"}
args = append(args, extraArgs...)
args = append(args, repository, targetDir)
return run(ctx, args, stdout, stderr)
}

func run(ctx context.Context, args []string, stdout, stderr io.Writer) error {
gitCommand := CommandContext(ctx, args...)
gitCommand.Stdout = stdout
gitCommand.Stderr = stderr
return gitCommand.Run()
}
21 changes: 11 additions & 10 deletions pkg/git/git.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,22 +114,23 @@ func NormalizeRepositoryGitInfo(str string) *GitInfo {
return NewGitInfo(repository, branch, commit, pr, subpath)
}

func CloneRepository(ctx context.Context, gitInfo *GitInfo, targetDir string, helper string, bare bool, writer io.Writer, log log.Logger) error {
args := []string{"clone"}
func CloneRepository(ctx context.Context, gitInfo *GitInfo, targetDir string, helper string, bare bool, cloner Cloner, writer io.Writer, log log.Logger) error {
if cloner == nil {
cloner = NewCloner(FullCloneStrategy)
}

extraArgs := []string{}
if bare && gitInfo.Commit == "" {
args = append(args, "--bare", "--depth=1")
extraArgs = append(extraArgs, "--bare", "--depth=1")
}
if helper != "" {
args = append(args, "--config", "credential.helper="+helper)
extraArgs = append(extraArgs, "--config", "credential.helper="+helper)
}
if gitInfo.Branch != "" {
args = append(args, "--branch", gitInfo.Branch)
extraArgs = append(extraArgs, "--branch", gitInfo.Branch)
}
args = append(args, gitInfo.Repository, targetDir)
gitCommand := CommandContext(ctx, args...)
gitCommand.Stdout = writer
gitCommand.Stderr = writer
err := gitCommand.Run()

err := cloner.Clone(ctx, gitInfo.Repository, targetDir, extraArgs, writer, writer)
if err != nil {
return errors.Wrap(err, "error cloning repository")
}
Expand Down
37 changes: 19 additions & 18 deletions pkg/provider/workspace.go
Original file line number Diff line number Diff line change
Expand Up @@ -164,24 +164,25 @@ type AgentWorkspaceInfo struct {

type CLIOptions struct {
// up options
ID string `json:"id,omitempty"`
Source string `json:"source,omitempty"`
IDE string `json:"ide,omitempty"`
IDEOptions []string `json:"ideOptions,omitempty"`
PrebuildRepositories []string `json:"prebuildRepositories,omitempty"`
DevContainerImage string `json:"devContainerImage,omitempty"`
DevContainerPath string `json:"devContainerPath,omitempty"`
WorkspaceEnv []string `json:"workspaceEnv,omitempty"`
WorkspaceEnvFile []string `json:"workspaceEnvFile,omitempty"`
Recreate bool `json:"recreate,omitempty"`
Reset bool `json:"reset,omitempty"`
Proxy bool `json:"proxy,omitempty"`
DisableDaemon bool `json:"disableDaemon,omitempty"`
DaemonInterval string `json:"daemonInterval,omitempty"`
ForceCredentials bool `json:"forceCredentials,omitempty"`
GitBranch string `json:"gitBranch,omitempty"`
GitCommit string `json:"gitCommit,omitempty"`
FallbackImage string `json:"fallbackImage,omitempty"`
ID string `json:"id,omitempty"`
Source string `json:"source,omitempty"`
IDE string `json:"ide,omitempty"`
IDEOptions []string `json:"ideOptions,omitempty"`
PrebuildRepositories []string `json:"prebuildRepositories,omitempty"`
DevContainerImage string `json:"devContainerImage,omitempty"`
DevContainerPath string `json:"devContainerPath,omitempty"`
WorkspaceEnv []string `json:"workspaceEnv,omitempty"`
WorkspaceEnvFile []string `json:"workspaceEnvFile,omitempty"`
Recreate bool `json:"recreate,omitempty"`
Reset bool `json:"reset,omitempty"`
Proxy bool `json:"proxy,omitempty"`
DisableDaemon bool `json:"disableDaemon,omitempty"`
DaemonInterval string `json:"daemonInterval,omitempty"`
ForceCredentials bool `json:"forceCredentials,omitempty"`
GitBranch string `json:"gitBranch,omitempty"`
GitCommit string `json:"gitCommit,omitempty"`
GitCloneStrategy git.CloneStrategy `json:"gitCloneStrategy,omitempty"`
FallbackImage string `json:"fallbackImage,omitempty"`

// build options
Repository string `json:"repository,omitempty"`
Expand Down

0 comments on commit 621f6ce

Please sign in to comment.