fix(git): try to prevent unshallow error 'shallow file has changed since we read it'

1. Added file lock for a fetch-with-unshallow operation.
2. Catch possible error with telemetry event.

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>

Author: distorhead

pkg/build/conveyor.go

@@ -907,8 +907,8 @@ func generateGitMappings(ctx context.Context, imageBaseConfig *config.StapelImag
 
 			if isShallowClone {
 				if c.werfConfig.Meta.GitWorktree.GetAllowUnshallow() {
-					if err := localGitRepo.FetchOrigin(ctx); err != nil {
-						return nil, err
+					if err := localGitRepo.Unshallow(ctx); err != nil {
+						return nil, fmt.Errorf("unable to fetch local git repo: %w", err)
 					}
 				} else {
 					logboek.Context(ctx).Warn().LogLn("The usage of shallow git clone may break reproducibility and slow down incremental rebuilds.")

pkg/git_repo/git_repo.go

@@ -43,7 +43,8 @@ type GitRepo interface {
 	GetWorkTreeDir() string
 	RemoteOriginUrl(_ context.Context) (string, error)
 	IsShallowClone(ctx context.Context) (bool, error)
-	FetchOrigin(ctx context.Context) error
+	FetchOrigin(ctx context.Context, opts FetchOptions) error
+	Unshallow(ctx context.Context) error
 	SyncWithOrigin(ctx context.Context) error
 
 	CreateDetachedMergeCommit(ctx context.Context, fromCommit, toCommit string) (string, error)
@@ -75,6 +76,10 @@ type GitRepo interface {
 	ValidateStatusResult(ctx context.Context, pathMatcher path_matcher.PathMatcher) error
 }
 
+type FetchOptions struct {
+	Unshallow bool
+}
+
 type gitRepo interface {
 	GitRepo
 

pkg/git_repo/local.go

@@ -10,10 +10,12 @@ import (
 
 	"github.com/go-git/go-git/v5"
 
+	"github.com/werf/lockgate"
 	"github.com/werf/logboek"
 	"github.com/werf/logboek/pkg/types"
 	"github.com/werf/werf/pkg/git_repo/repo_handle"
 	"github.com/werf/werf/pkg/path_matcher"
+	"github.com/werf/werf/pkg/telemetry"
 	"github.com/werf/werf/pkg/true_git"
 	"github.com/werf/werf/pkg/true_git/status"
 	"github.com/werf/werf/pkg/util"
@@ -156,29 +158,74 @@ func (repo *Local) SyncWithOrigin(ctx context.Context) error {
 	})
 }
 
-func (repo *Local) FetchOrigin(ctx context.Context) error {
+func (repo *Local) acquireFetchLock(ctx context.Context) (lockgate.LockHandle, error) {
+	_, lock, err := werf.AcquireHostLock(ctx, fmt.Sprintf("local_git_repo.fetch.%s", repo.GitDir), lockgate.AcquireOptions{})
+	return lock, err
+}
+
+func (repo *Local) Unshallow(ctx context.Context) error {
+	if lock, err := repo.acquireFetchLock(ctx); err != nil {
+		return fmt.Errorf("unable to acquire fetch lock: %w", err)
+	} else {
+		defer werf.ReleaseHostLock(lock)
+	}
+
 	isShallow, err := repo.IsShallowClone(ctx)
 	if err != nil {
 		return fmt.Errorf("check shallow clone failed: %w", err)
 	}
+	if !isShallow {
+		return nil
+	}
 
-	remoteOriginUrl, err := repo.RemoteOriginUrl(ctx)
+	err = repo.doFetchOrigin(ctx, true)
 	if err != nil {
-		return fmt.Errorf("get remote origin failed: %w", err)
+		return fmt.Errorf("unable to fetch origin: %w", err)
 	}
 
-	if remoteOriginUrl == "" {
-		return fmt.Errorf("git remote origin was not detected")
+	return nil
+}
+
+func (repo *Local) FetchOrigin(ctx context.Context, opts FetchOptions) error {
+	if lock, err := repo.acquireFetchLock(ctx); err != nil {
+		return fmt.Errorf("unable to acquire fetch lock: %w", err)
+	} else {
+		defer werf.ReleaseHostLock(lock)
 	}
 
+	var unshallow bool
+	if opts.Unshallow {
+		isShallow, err := repo.IsShallowClone(ctx)
+		if err != nil {
+			return fmt.Errorf("check shallow clone failed: %w", err)
+		}
+		unshallow = isShallow
+	}
+
+	return repo.doFetchOrigin(ctx, unshallow)
+}
+
+func (repo *Local) doFetchOrigin(ctx context.Context, unshallow bool) error {
 	return logboek.Context(ctx).Default().LogProcess("Fetching origin").DoError(func() error {
+		remoteOriginUrl, err := repo.RemoteOriginUrl(ctx)
+		if err != nil {
+			return fmt.Errorf("get remote origin failed: %w", err)
+		}
+
+		if remoteOriginUrl == "" {
+			return fmt.Errorf("git remote origin was not detected")
+		}
+
 		fetchOptions := true_git.FetchOptions{
-			Unshallow: isShallow,
+			Unshallow: unshallow,
 			RefSpecs:  map[string]string{"origin": "+refs/heads/*:refs/remotes/origin/*"},
 		}
 
 		if err := true_git.Fetch(ctx, repo.WorkTreeDir, fetchOptions); err != nil {
-			return fmt.Errorf("fetch failed: %w", err)
+			if true_git.IsShallowFileChangedSinceWeReadIt(err) {
+				telemetry.GetTelemetryWerfIO().UnshallowFailed(ctx, err)
+			}
+			return err
 		}
 
 		return nil

pkg/git_repo/remote.go

@@ -107,7 +107,7 @@ func (repo *Remote) CloneAndFetch(ctx context.Context) error {
 		return nil
 	}
 
-	return repo.FetchOrigin(ctx)
+	return repo.FetchOrigin(ctx, FetchOptions{})
 }
 
 func (repo *Remote) isCloneExists() (bool, error) {
@@ -210,7 +210,11 @@ func (repo *Remote) SyncWithOrigin(ctx context.Context) error {
 	panic("not implemented")
 }
 
-func (repo *Remote) FetchOrigin(ctx context.Context) error {
+func (repo *Remote) Unshallow(ctx context.Context) error {
+	panic("not implemented")
+}
+
+func (repo *Remote) FetchOrigin(ctx context.Context, opts FetchOptions) error {
 	if repo.IsDryRun {
 		return nil
 	}

pkg/telemetry/event.go

@@ -1,10 +1,13 @@
 package telemetry
 
+import "os"
+
 type EventType string
 
 const (
-	CommandStartedEvent EventType = "CommandStarted"
-	CommandExitedEvent  EventType = "CommandExited"
+	CommandStartedEvent  EventType = "CommandStarted"
+	CommandExitedEvent   EventType = "CommandExited"
+	UnshallowFailedEvent EventType = "UnshallowFailed"
 )
 
 type Event interface {
@@ -38,3 +41,19 @@ type CommandExited struct {
 }
 
 func (e *CommandExited) GetType() EventType { return CommandExitedEvent }
+
+func NewUnshallowFailed(errorMessage string) *UnshallowFailed {
+	return &UnshallowFailed{
+		ErrorMessage:        errorMessage,
+		GitlabRunnerVersion: os.Getenv("CI_RUNNER_VERSION"),
+		GitlabServerVersion: os.Getenv("CI_SERVER_VERSION"),
+	}
+}
+
+type UnshallowFailed struct {
+	ErrorMessage        string `json:"errorMessage"`
+	GitlabRunnerVersion string `json:"gitlabRunnerVersion"`
+	GitlabServerVersion string `json:"gitlabServerVersion"`
+}
+
+func (*UnshallowFailed) GetType() EventType { return UnshallowFailedEvent }

pkg/telemetry/helpers.go

@@ -25,7 +25,7 @@ func NewTraceExporter(url string) (*otlptrace.Exporter, error) {
 		otlptracehttp.WithEndpoint(urlObj.Host),
 		otlptracehttp.WithURLPath(urlObj.Path),
 		otlptracehttp.WithRetry(otlptracehttp.RetryConfig{Enabled: false}),
-		otlptracehttp.WithTimeout(800*time.Millisecond),
+		otlptracehttp.WithTimeout(1300*time.Millisecond),
 	)
 
 	client := otlptracehttp.NewClient(opts...)

pkg/telemetry/no_telemetrywerfio.go

@@ -10,3 +10,4 @@ func (t *NoTelemetryWerfIO) SetProjectID(context.Context, string)
 func (t *NoTelemetryWerfIO) SetCommand(context.Context, string)                 {}
 func (t *NoTelemetryWerfIO) CommandExited(context.Context, int)                 {}
 func (t *NoTelemetryWerfIO) SetCommandOptions(context.Context, []CommandOption) {}
+func (t *NoTelemetryWerfIO) UnshallowFailed(context.Context, error)             {}

pkg/telemetry/telemetrywerfio.go

@@ -33,6 +33,7 @@ type TelemetryWerfIOInterface interface {
 
 	CommandStarted(ctx context.Context)
 	CommandExited(ctx context.Context, exitCode int)
+	UnshallowFailed(ctx context.Context, err error)
 }
 
 type TelemetryWerfIO struct {
@@ -63,7 +64,7 @@ func NewTelemetryWerfIO(url string, opts TelemetryWerfIOOptions) (*TelemetryWerf
 		tracerProvider: sdktrace.NewTracerProvider(
 			sdktrace.WithBatcher(e,
 				sdktrace.WithBatchTimeout(1*time.Millisecond), // send all available events immediately
-				sdktrace.WithExportTimeout(800*time.Millisecond),
+				sdktrace.WithExportTimeout(1300*time.Millisecond),
 			),
 		),
 		traceExporter: e,
@@ -132,6 +133,10 @@ func (t *TelemetryWerfIO) CommandExited(ctx context.Context, exitCode int) {
 	t.sendEvent(ctx, NewCommandExited(exitCode, int64(duration/time.Millisecond)))
 }
 
+func (t *TelemetryWerfIO) UnshallowFailed(ctx context.Context, err error) {
+	t.sendEvent(ctx, NewUnshallowFailed(err.Error()))
+}
+
 func (t *TelemetryWerfIO) getAttributes() map[string]interface{} {
 	attributes := map[string]interface{}{
 		"version": werf.Version,

pkg/true_git/repository.go

@@ -25,6 +25,10 @@ type FetchOptions struct {
 	RefSpecs  map[string]string
 }
 
+func IsShallowFileChangedSinceWeReadIt(err error) bool {
+	return err != nil && strings.Contains(err.Error(), "shallow file has changed since we read it")
+}
+
 func Fetch(ctx context.Context, path string, options FetchOptions) error {
 	commandArgs := []string{"fetch"}