feat(dev-mode): less rebuilds due to better cache handling

Single persistent dev-mode service branch instead of multiple. This
branch now constantly merges user working branch in itself, which allows
for better utilization of cached stages and less rebuilds.

`--dev-branch-prefix` flag replaced with `--dev-branch` flag.

Signed-off-by: Ilya Lesikov <ilya@lesikov.com>

Author: ilya-lesikov

cmd/werf/common/common.go

@@ -93,7 +93,7 @@ type CmdData struct {
 	LooseGiterminism *bool
 	Dev              *bool
 	DevIgnore        *[]string
-	DevBranchPrefix  *string
+	DevBranch        *string
 
 	IntrospectBeforeError *bool
 	IntrospectAfterError  *bool
@@ -178,7 +178,7 @@ func SetupGiterminismOptions(cmdData *CmdData, cmd *cobra.Command) {
 	setupLooseGiterminism(cmdData, cmd)
 	setupDev(cmdData, cmd)
 	setupDevIgnore(cmdData, cmd)
-	setupDevBranchPrefix(cmdData, cmd)
+	setupDevBranch(cmdData, cmd)
 }
 
 func setupLooseGiterminism(cmdData *CmdData, cmd *cobra.Command) {
@@ -198,16 +198,16 @@ func setupDevIgnore(cmdData *CmdData, cmd *cobra.Command) {
 Also, can be specified with $WERF_DEV_IGNORE_* (e.g. $WERF_DEV_IGNORE_TESTS=*_test.go, $WERF_DEV_IGNORE_DOCS=path/to/docs)`)
 }
 
-func setupDevBranchPrefix(cmdData *CmdData, cmd *cobra.Command) {
-	cmdData.DevBranchPrefix = new(string)
+func setupDevBranch(cmdData *CmdData, cmd *cobra.Command) {
+	cmdData.DevBranch = new(string)
 
-	defaultValue := "werf-dev-"
-	envValue := os.Getenv("WERF_DEV_BRANCH_PREFIX")
+	defaultValue := "_werf-dev"
+	envValue := os.Getenv("WERF_DEV_BRANCH")
 	if envValue != "" {
 		defaultValue = envValue
 	}
 
-	cmd.Flags().StringVarP(cmdData.DevBranchPrefix, "dev-branch-prefix", "", defaultValue, `Set dev git branch prefix (default $WERF_DEV_BRANCH_PREFIX or werf-dev-)`)
+	cmd.Flags().StringVarP(cmdData.DevBranch, "dev-branch", "", defaultValue, fmt.Sprintf("Set dev git branch name (default $WERF_DEV_BRANCH or %q)", defaultValue))
 }
 
 func SetupHomeDir(cmdData *CmdData, cmd *cobra.Command) {
@@ -1179,7 +1179,7 @@ func GetGiterminismManager(ctx context.Context, cmdData *CmdData) (giterminism_m
 	var openLocalRepoOptions git_repo.OpenLocalRepoOptions
 	if *cmdData.Dev {
 		openLocalRepoOptions.WithServiceHeadCommit = true
-		openLocalRepoOptions.ServiceBranchOptions.Prefix = *cmdData.DevBranchPrefix
+		openLocalRepoOptions.ServiceBranchOptions.Name = *cmdData.DevBranch
 		openLocalRepoOptions.ServiceBranchOptions.GlobExcludeList = GetDevIgnore(cmdData)
 	}
 

pkg/git_repo/local.go

@@ -40,7 +40,7 @@ type OpenLocalRepoOptions struct {
 }
 
 type ServiceBranchOptions struct {
-	Prefix          string
+	Name            string
 	GlobExcludeList []string
 }
 
@@ -71,29 +71,22 @@ func OpenLocalRepo(ctx context.Context, name, workTreeDir string, opts OpenLocal
 			defer werf.ReleaseHostLock(lock)
 		}
 
-		gitStatusResult, err := l.status(ctx)
+		devHeadCommit, err := true_git.SyncSourceWorktreeWithServiceBranch(
+			context.Background(),
+			l.GitDir,
+			l.WorkTreeDir,
+			l.getRepoWorkTreeCacheDir(l.getRepoID()),
+			l.headCommitHash,
+			true_git.SyncSourceWorktreeWithServiceBranchOptions{
+				ServiceBranch:   opts.ServiceBranchOptions.Name,
+				GlobExcludeList: opts.ServiceBranchOptions.GlobExcludeList,
+			},
+		)
 		if err != nil {
-			return nil, fmt.Errorf("unable to get git status: %s", err)
+			return l, err
 		}
 
-		if len(gitStatusResult.PathListWithSubmodules()) != 0 {
-			devHeadCommit, err := true_git.SyncSourceWorktreeWithServiceBranch(
-				context.Background(),
-				l.GitDir,
-				l.WorkTreeDir,
-				l.getRepoWorkTreeCacheDir(l.getRepoID()),
-				l.headCommitHash,
-				true_git.SyncSourceWorktreeWithServiceBranchOptions{
-					ServiceBranchPrefix: opts.ServiceBranchOptions.Prefix,
-					GlobExcludeList:     opts.ServiceBranchOptions.GlobExcludeList,
-				},
-			)
-			if err != nil {
-				return l, err
-			}
-
-			l.headCommitHash = devHeadCommit
-		}
+		l.headCommitHash = devHeadCommit
 	}
 
 	return l, nil

pkg/true_git/repository.go

@@ -57,6 +57,15 @@ func Fetch(ctx context.Context, path string, options FetchOptions) error {
 	return gitCmd.Run(ctx)
 }
 
+func GetLastBranchCommitSHA(ctx context.Context, repoPath, branch string) (string, error) {
+	revParseCmd := NewGitCmd(ctx, &GitCmdOptions{RepoDir: repoPath}, "rev-parse", branch)
+	if err := revParseCmd.Run(ctx); err != nil {
+		return "", fmt.Errorf("git rev parse branch command failed: %s", err)
+	}
+
+	return strings.TrimSpace(revParseCmd.OutBuf.String()), nil
+}
+
 func IsShallowClone(ctx context.Context, path string) (bool, error) {
 	if gitVersion.LessThan(semver.MustParse("2.15.0")) {
 		exist, err := util.FileExists(filepath.Join(path, ".git", "shallow"))

pkg/true_git/service_branch.go

@@ -13,8 +13,8 @@ import (
 )
 
 type SyncSourceWorktreeWithServiceBranchOptions struct {
-	ServiceBranchPrefix string
-	GlobExcludeList     []string
+	ServiceBranch   string
+	GlobExcludeList []string
 }
 
 func SyncSourceWorktreeWithServiceBranch(ctx context.Context, gitDir, sourceWorktreeDir, worktreeCacheDir, commit string, opts SyncSourceWorktreeWithServiceBranchOptions) (string, error) {
@@ -39,10 +39,9 @@ func SyncSourceWorktreeWithServiceBranch(ctx context.Context, gitDir, sourceWork
 			return fmt.Errorf("unable to remove %s: %s", currentCommitPath, err)
 		}
 
-		branchName := fmt.Sprintf("%s%s", opts.ServiceBranchPrefix, commit)
-		resultCommit, err = syncWorktreeWithServiceWorktreeBranch(ctx, sourceWorktreeDir, serviceWorktreeDir, commit, branchName, opts.GlobExcludeList)
+		resultCommit, err = syncWorktreeWithServiceWorktreeBranch(ctx, sourceWorktreeDir, serviceWorktreeDir, commit, opts.ServiceBranch, opts.GlobExcludeList)
 		if err != nil {
-			return fmt.Errorf("unable to sync worktree with service branch %q: %s", branchName, err)
+			return fmt.Errorf("unable to sync worktree with service branch %q: %s", opts.ServiceBranch, err)
 		}
 
 		return nil
@@ -54,14 +53,13 @@ func SyncSourceWorktreeWithServiceBranch(ctx context.Context, gitDir, sourceWork
 }
 
 func syncWorktreeWithServiceWorktreeBranch(ctx context.Context, sourceWorktreeDir, serviceWorktreeDir, sourceCommit, branchName string, globExcludeList []string) (string, error) {
-	serviceBranchHeadCommit, err := getOrPrepareServiceBranchHeadCommit(ctx, serviceWorktreeDir, sourceCommit, branchName)
-	if err != nil {
+	if err := prepareAndCheckoutServiceBranch(ctx, serviceWorktreeDir, sourceCommit, branchName); err != nil {
 		return "", fmt.Errorf("unable to get or prepare service branch head commit: %s", err)
 	}
 
-	checkoutCmd := NewGitCmd(ctx, &GitCmdOptions{RepoDir: serviceWorktreeDir}, "checkout", branchName)
-	if err = checkoutCmd.Run(ctx); err != nil {
-		return "", fmt.Errorf("git checkout command failed: %s", err)
+	serviceBranchHeadCommit, err := GetLastBranchCommitSHA(ctx, serviceWorktreeDir, branchName)
+	if err != nil {
+		return "", fmt.Errorf("unable to get service worktree commit SHA: %s", err)
 	}
 
 	revertedChangesExist, err := revertExcludedChangesInServiceWorktreeIndex(ctx, sourceWorktreeDir, serviceWorktreeDir, sourceCommit, serviceBranchHeadCommit, globExcludeList)
@@ -90,31 +88,44 @@ func syncWorktreeWithServiceWorktreeBranch(ctx context.Context, sourceWorktreeDi
 	return newCommit, nil
 }
 
-func getOrPrepareServiceBranchHeadCommit(ctx context.Context, serviceWorktreeDir string, sourceCommit string, branchName string) (string, error) {
+func prepareAndCheckoutServiceBranch(ctx context.Context, serviceWorktreeDir string, sourceCommit string, branchName string) error {
 	branchListCmd := NewGitCmd(ctx, &GitCmdOptions{RepoDir: serviceWorktreeDir}, "branch", "--list", branchName)
 	if err := branchListCmd.Run(ctx); err != nil {
-		return "", fmt.Errorf("git branch list command failed: %s", err)
+		return fmt.Errorf("git branch list command failed: %s", err)
 	}
 
-	var isServiceBranchExist bool
-	isServiceBranchExist = branchListCmd.OutBuf.Len() != 0
-
-	if !isServiceBranchExist {
+	if branchListCmd.OutBuf.Len() == 0 {
 		checkoutCmd := NewGitCmd(ctx, &GitCmdOptions{RepoDir: serviceWorktreeDir}, "checkout", "-b", branchName, sourceCommit)
 		if err := checkoutCmd.Run(ctx); err != nil {
-			return "", fmt.Errorf("git checkout command failed: %s", err)
+			return fmt.Errorf("git checkout command failed: %s", err)
 		}
 
-		return sourceCommit, nil
+		return nil
 	}
 
-	revParseCmd := NewGitCmd(ctx, &GitCmdOptions{RepoDir: serviceWorktreeDir}, "rev-parse", branchName)
-	if err := revParseCmd.Run(ctx); err != nil {
-		return "", fmt.Errorf("git rev parse branch command failed: %s", err)
+	checkoutCmd := NewGitCmd(ctx, &GitCmdOptions{RepoDir: serviceWorktreeDir}, "checkout", branchName)
+	if err := checkoutCmd.Run(ctx); err != nil {
+		return fmt.Errorf("git checkout command failed: %s", err)
+	}
+
+	isSourceCommitInServiceBranch, err := IsAncestor(ctx, sourceCommit, branchName, serviceWorktreeDir)
+	if err != nil {
+		return fmt.Errorf("unable to detect whether sourceCommit %q is in service branch: %s", sourceCommit, err)
+	}
+	if isSourceCommitInServiceBranch {
+		return nil
+	}
+
+	mergeCmd := NewGitCmd(
+		ctx, &GitCmdOptions{RepoDir: serviceWorktreeDir},
+		"-c", "user.email=werf@werf.io", "-c", "user.name=werf",
+		"merge", "--no-edit", "--no-ff", "--allow-unrelated-histories", "-s", "recursive", "-X", "theirs", sourceCommit,
+	)
+	if err = mergeCmd.Run(ctx); err != nil {
+		return fmt.Errorf("git merge of source commit %q into service branch failed: %s", sourceCommit, err)
 	}
 
-	serviceBranchHeadCommit := strings.TrimSpace(revParseCmd.OutBuf.String())
-	return serviceBranchHeadCommit, nil
+	return nil
 }
 
 func revertExcludedChangesInServiceWorktreeIndex(ctx context.Context, sourceWorktreeDir string, serviceWorktreeDir string, sourceCommit string, serviceBranchHeadCommit string, globExcludeList []string) (bool, error) {