Skip to content
Permalink
Browse files
fix(git): try to prevent unshallow error 'shallow file has changed si…
…nce we read it'

1. Added file lock for a fetch-with-unshallow operation.
2. Catch possible error with telemetry event.

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
distorhead committed Sep 6, 2022
1 parent 918fed4 commit e51546c78b3460700c30823c8f3de0975f4e027b
Show file tree
Hide file tree
Showing 9 changed files with 101 additions and 16 deletions.
@@ -907,8 +907,8 @@ func generateGitMappings(ctx context.Context, imageBaseConfig *config.StapelImag

if isShallowClone {
if c.werfConfig.Meta.GitWorktree.GetAllowUnshallow() {
if err := localGitRepo.FetchOrigin(ctx); err != nil {
return nil, err
if err := localGitRepo.Unshallow(ctx); err != nil {
return nil, fmt.Errorf("unable to fetch local git repo: %w", err)
}
} else {
logboek.Context(ctx).Warn().LogLn("The usage of shallow git clone may break reproducibility and slow down incremental rebuilds.")
@@ -43,7 +43,8 @@ type GitRepo interface {
GetWorkTreeDir() string
RemoteOriginUrl(_ context.Context) (string, error)
IsShallowClone(ctx context.Context) (bool, error)
FetchOrigin(ctx context.Context) error
FetchOrigin(ctx context.Context, opts FetchOptions) error
Unshallow(ctx context.Context) error
SyncWithOrigin(ctx context.Context) error

CreateDetachedMergeCommit(ctx context.Context, fromCommit, toCommit string) (string, error)
@@ -75,6 +76,10 @@ type GitRepo interface {
ValidateStatusResult(ctx context.Context, pathMatcher path_matcher.PathMatcher) error
}

type FetchOptions struct {
Unshallow bool
}

type gitRepo interface {
GitRepo

@@ -10,10 +10,12 @@ import (

"github.com/go-git/go-git/v5"

"github.com/werf/lockgate"
"github.com/werf/logboek"
"github.com/werf/logboek/pkg/types"
"github.com/werf/werf/pkg/git_repo/repo_handle"
"github.com/werf/werf/pkg/path_matcher"
"github.com/werf/werf/pkg/telemetry"
"github.com/werf/werf/pkg/true_git"
"github.com/werf/werf/pkg/true_git/status"
"github.com/werf/werf/pkg/util"
@@ -156,29 +158,74 @@ func (repo *Local) SyncWithOrigin(ctx context.Context) error {
})
}

func (repo *Local) FetchOrigin(ctx context.Context) error {
func (repo *Local) acquireFetchLock(ctx context.Context) (lockgate.LockHandle, error) {
_, lock, err := werf.AcquireHostLock(ctx, fmt.Sprintf("local_git_repo.fetch.%s", repo.GitDir), lockgate.AcquireOptions{})
return lock, err
}

func (repo *Local) Unshallow(ctx context.Context) error {
if lock, err := repo.acquireFetchLock(ctx); err != nil {
return fmt.Errorf("unable to acquire fetch lock: %w", err)
} else {
defer werf.ReleaseHostLock(lock)
}

isShallow, err := repo.IsShallowClone(ctx)
if err != nil {
return fmt.Errorf("check shallow clone failed: %w", err)
}
if !isShallow {
return nil
}

remoteOriginUrl, err := repo.RemoteOriginUrl(ctx)
err = repo.doFetchOrigin(ctx, true)
if err != nil {
return fmt.Errorf("get remote origin failed: %w", err)
return fmt.Errorf("unable to fetch origin: %w", err)
}

if remoteOriginUrl == "" {
return fmt.Errorf("git remote origin was not detected")
return nil
}

func (repo *Local) FetchOrigin(ctx context.Context, opts FetchOptions) error {
if lock, err := repo.acquireFetchLock(ctx); err != nil {
return fmt.Errorf("unable to acquire fetch lock: %w", err)
} else {
defer werf.ReleaseHostLock(lock)
}

var unshallow bool
if opts.Unshallow {
isShallow, err := repo.IsShallowClone(ctx)
if err != nil {
return fmt.Errorf("check shallow clone failed: %w", err)
}
unshallow = isShallow
}

return repo.doFetchOrigin(ctx, unshallow)
}

func (repo *Local) doFetchOrigin(ctx context.Context, unshallow bool) error {
return logboek.Context(ctx).Default().LogProcess("Fetching origin").DoError(func() error {
remoteOriginUrl, err := repo.RemoteOriginUrl(ctx)
if err != nil {
return fmt.Errorf("get remote origin failed: %w", err)
}

if remoteOriginUrl == "" {
return fmt.Errorf("git remote origin was not detected")
}

fetchOptions := true_git.FetchOptions{
Unshallow: isShallow,
Unshallow: unshallow,
RefSpecs: map[string]string{"origin": "+refs/heads/*:refs/remotes/origin/*"},
}

if err := true_git.Fetch(ctx, repo.WorkTreeDir, fetchOptions); err != nil {
return fmt.Errorf("fetch failed: %w", err)
if true_git.IsShallowFileChangedSinceWeReadIt(err) {
telemetry.GetTelemetryWerfIO().UnshallowFailed(ctx, err)
}
return err
}

return nil
@@ -107,7 +107,7 @@ func (repo *Remote) CloneAndFetch(ctx context.Context) error {
return nil
}

return repo.FetchOrigin(ctx)
return repo.FetchOrigin(ctx, FetchOptions{})
}

func (repo *Remote) isCloneExists() (bool, error) {
@@ -210,7 +210,11 @@ func (repo *Remote) SyncWithOrigin(ctx context.Context) error {
panic("not implemented")
}

func (repo *Remote) FetchOrigin(ctx context.Context) error {
func (repo *Remote) Unshallow(ctx context.Context) error {
panic("not implemented")
}

func (repo *Remote) FetchOrigin(ctx context.Context, opts FetchOptions) error {
if repo.IsDryRun {
return nil
}
@@ -1,10 +1,13 @@
package telemetry

import "os"

type EventType string

const (
CommandStartedEvent EventType = "CommandStarted"
CommandExitedEvent EventType = "CommandExited"
CommandStartedEvent EventType = "CommandStarted"
CommandExitedEvent EventType = "CommandExited"
UnshallowFailedEvent EventType = "UnshallowFailed"
)

type Event interface {
@@ -38,3 +41,19 @@ type CommandExited struct {
}

func (e *CommandExited) GetType() EventType { return CommandExitedEvent }

func NewUnshallowFailed(errorMessage string) *UnshallowFailed {
return &UnshallowFailed{
ErrorMessage: errorMessage,
GitlabRunnerVersion: os.Getenv("CI_RUNNER_VERSION"),
GitlabServerVersion: os.Getenv("CI_SERVER_VERSION"),
}
}

type UnshallowFailed struct {
ErrorMessage string `json:"errorMessage"`
GitlabRunnerVersion string `json:"gitlabRunnerVersion"`
GitlabServerVersion string `json:"gitlabServerVersion"`
}

func (*UnshallowFailed) GetType() EventType { return UnshallowFailedEvent }
@@ -25,7 +25,7 @@ func NewTraceExporter(url string) (*otlptrace.Exporter, error) {
otlptracehttp.WithEndpoint(urlObj.Host),
otlptracehttp.WithURLPath(urlObj.Path),
otlptracehttp.WithRetry(otlptracehttp.RetryConfig{Enabled: false}),
otlptracehttp.WithTimeout(800*time.Millisecond),
otlptracehttp.WithTimeout(1300*time.Millisecond),
)

client := otlptracehttp.NewClient(opts...)
@@ -10,3 +10,4 @@ func (t *NoTelemetryWerfIO) SetProjectID(context.Context, string)
func (t *NoTelemetryWerfIO) SetCommand(context.Context, string) {}
func (t *NoTelemetryWerfIO) CommandExited(context.Context, int) {}
func (t *NoTelemetryWerfIO) SetCommandOptions(context.Context, []CommandOption) {}
func (t *NoTelemetryWerfIO) UnshallowFailed(context.Context, error) {}
@@ -33,6 +33,7 @@ type TelemetryWerfIOInterface interface {

CommandStarted(ctx context.Context)
CommandExited(ctx context.Context, exitCode int)
UnshallowFailed(ctx context.Context, err error)
}

type TelemetryWerfIO struct {
@@ -63,7 +64,7 @@ func NewTelemetryWerfIO(url string, opts TelemetryWerfIOOptions) (*TelemetryWerf
tracerProvider: sdktrace.NewTracerProvider(
sdktrace.WithBatcher(e,
sdktrace.WithBatchTimeout(1*time.Millisecond), // send all available events immediately
sdktrace.WithExportTimeout(800*time.Millisecond),
sdktrace.WithExportTimeout(1300*time.Millisecond),
),
),
traceExporter: e,
@@ -132,6 +133,10 @@ func (t *TelemetryWerfIO) CommandExited(ctx context.Context, exitCode int) {
t.sendEvent(ctx, NewCommandExited(exitCode, int64(duration/time.Millisecond)))
}

func (t *TelemetryWerfIO) UnshallowFailed(ctx context.Context, err error) {
t.sendEvent(ctx, NewUnshallowFailed(err.Error()))
}

func (t *TelemetryWerfIO) getAttributes() map[string]interface{} {
attributes := map[string]interface{}{
"version": werf.Version,
@@ -25,6 +25,10 @@ type FetchOptions struct {
RefSpecs map[string]string
}

func IsShallowFileChangedSinceWeReadIt(err error) bool {
return err != nil && strings.Contains(err.Error(), "shallow file has changed since we read it")
}

func Fetch(ctx context.Context, path string, options FetchOptions) error {
commandArgs := []string{"fetch"}

0 comments on commit e51546c

Please sign in to comment.