fix(cleanup): fix "should reset storage cache" error during werf-cleanup and werf-purge

Automatically reset stages storage cache in the werf-cleanup and werf-purge commands as in werf-build.

Author: distorhead

cmd/werf/cleanup/cleanup.go

@@ -258,10 +258,8 @@ It is worth noting that auto-cleaning is enabled by default, and manual use is u
 		DryRun:                                  *commonCmdData.DryRun,
 	}
 
-	logboek.LogOptionalLn()
-	if err := cleaning.Cleanup(ctx, projectName, storageManager, storageLockManager, cleanupOptions); err != nil {
-		return err
-	}
-
-	return nil
+	return manager.RetryOnStagesStorageCacheResetError(ctx, storageManager, func() error {
+		logboek.LogOptionalLn()
+		return cleaning.Cleanup(ctx, projectName, storageManager, storageLockManager, cleanupOptions)
+	})
 }

cmd/werf/purge/purge.go

@@ -28,7 +28,7 @@ func NewCmd() *cobra.Command {
 		Use:                   "purge",
 		DisableFlagsInUseLine: true,
 		Short:                 "Purge all project images in the container registry",
-		Long: common.GetLongCommandDescription(`Purge all project images in the container registry. 
+		Long: common.GetLongCommandDescription(`Purge all project images in the container registry.
 
 WARNING: Images that are being used in the Kubernetes cluster will also be deleted.`),
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -150,7 +150,7 @@ func runPurge() error {
 
 	stagesStorageAddress, err := common.GetStagesStorageAddress(&commonCmdData)
 	if err != nil {
-		logboek.Context(ctx).Default().LogLnDetails(`The "werf purge" command is only used to cleaning the container registry. In case you need to clean the runner or the localhost, use the commands of the "werf host" group. 
+		logboek.Context(ctx).Default().LogLnDetails(`The "werf purge" command is only used to cleaning the container registry. In case you need to clean the runner or the localhost, use the commands of the "werf host" group.
 It is worth noting that auto-cleaning is enabled by default, and manual use is usually not required (if not, we would appreciate feedback and creating an issue https://github.com/werf/werf/issues/new).`)
 
 		return err
@@ -194,10 +194,8 @@ It is worth noting that auto-cleaning is enabled by default, and manual use is u
 		DryRun: *commonCmdData.DryRun,
 	}
 
-	logboek.LogOptionalLn()
-	if err := cleaning.Purge(ctx, projectName, storageManager, storageLockManager, purgeOptions); err != nil {
-		return err
-	}
-
-	return nil
+	return manager.RetryOnStagesStorageCacheResetError(ctx, storageManager, func() error {
+		logboek.LogOptionalLn()
+		return cleaning.Purge(ctx, projectName, storageManager, storageLockManager, purgeOptions)
+	})
 }

docs/documentation/_includes/reference/cli/werf_purge.md

@@ -3,7 +3,7 @@
 {% else %}
 {% assign header = "###" %}
 {% endif %}
-Purge all project images in the container registry. 
+Purge all project images in the container registry.
 
 WARNING: Images that are being used in the Kubernetes cluster will also be deleted.
 

pkg/build/conveyor_with_retry.go

@@ -3,7 +3,6 @@ package build
 import (
 	"context"
 
-	"github.com/werf/logboek"
 	"github.com/werf/werf/pkg/config"
 	"github.com/werf/werf/pkg/container_runtime"
 	"github.com/werf/werf/pkg/giterminism_manager"
@@ -45,37 +44,22 @@ func (wrapper *ConveyorWithRetryWrapper) Terminate() error {
 }
 
 func (wrapper *ConveyorWithRetryWrapper) WithRetryBlock(ctx context.Context, f func(c *Conveyor) error) error {
-Retry:
-	newConveyor := NewConveyor(
-		wrapper.WerfConfig,
-		wrapper.GiterminismManager,
-		wrapper.ImageNamesToProcess,
-		wrapper.ProjectDir,
-		wrapper.BaseTmpDir,
-		wrapper.SshAuthSock,
-		wrapper.ContainerRuntime,
-		wrapper.StorageManager,
-		wrapper.StorageLockManager,
-		wrapper.ConveyorOptions,
-	)
+	return manager.RetryOnStagesStorageCacheResetError(ctx, wrapper.StorageManager, func() error {
+		newConveyor := NewConveyor(
+			wrapper.WerfConfig,
+			wrapper.GiterminismManager,
+			wrapper.ImageNamesToProcess,
+			wrapper.ProjectDir,
+			wrapper.BaseTmpDir,
+			wrapper.SshAuthSock,
+			wrapper.ContainerRuntime,
+			wrapper.StorageManager,
+			wrapper.StorageLockManager,
+			wrapper.ConveyorOptions,
+		)
 
-	if shouldRetry, err := func() (bool, error) {
 		defer newConveyor.Terminate(ctx)
 
-		if err := f(newConveyor); manager.ShouldResetStagesStorageCache(err) {
-			logboek.Context(ctx).Error().LogF("Will reset stages storage cache due to error: %s\n", err)
-
-			if err := newConveyor.StorageManager.ResetStagesStorageCache(ctx); err != nil {
-				return false, err
-			}
-			return true, nil
-		} else {
-			return false, err
-		}
-	}(); err != nil {
-		return err
-	} else if shouldRetry {
-		goto Retry
-	}
-	return nil
+		return f(newConveyor)
+	})
 }

pkg/storage/manager/storage_manager.go

@@ -80,6 +80,23 @@ func ShouldResetStagesStorageCache(err error) bool {
 	return false
 }
 
+func RetryOnStagesStorageCacheResetError(ctx context.Context, manager StorageManagerInterface, f func() error) error {
+Retry:
+	err := f()
+
+	if ShouldResetStagesStorageCache(err) {
+		logboek.Context(ctx).Error().LogF("Will reset stages storage cache due to error: %s\n", err)
+
+		if err := manager.ResetStagesStorageCache(ctx); err != nil {
+			return fmt.Errorf("unable to reset stages storage cache: %s", err)
+		}
+
+		goto Retry
+	}
+
+	return err
+}
+
 func NewStorageManager(projectName string, stagesStorage storage.StagesStorage, finalStagesStorage storage.StagesStorage, secondaryStagesStorageList []storage.StagesStorage, cacheStagesStorageList []storage.StagesStorage, storageLockManager storage.LockManager, stagesStorageCache storage.StagesStorageCache) *StorageManager {
 	return &StorageManager{
 		ProjectName:        projectName,