Skip to content
Permalink
Browse files
fix(cleanup): fix "should reset storage cache" error during werf-clea…
…nup and werf-purge

Automatically reset stages storage cache in the werf-cleanup and werf-purge commands as in werf-build.
  • Loading branch information
distorhead committed Oct 5, 2021
1 parent 3dc7098 commit dd43b6839b2a28ce2b4273f77c47f2fa0994969e
@@ -258,10 +258,8 @@ It is worth noting that auto-cleaning is enabled by default, and manual use is u
DryRun: *commonCmdData.DryRun,
}

logboek.LogOptionalLn()
if err := cleaning.Cleanup(ctx, projectName, storageManager, storageLockManager, cleanupOptions); err != nil {
return err
}

return nil
return manager.RetryOnStagesStorageCacheResetError(ctx, storageManager, func() error {
logboek.LogOptionalLn()
return cleaning.Cleanup(ctx, projectName, storageManager, storageLockManager, cleanupOptions)
})
}
@@ -28,7 +28,7 @@ func NewCmd() *cobra.Command {
Use: "purge",
DisableFlagsInUseLine: true,
Short: "Purge all project images in the container registry",
Long: common.GetLongCommandDescription(`Purge all project images in the container registry.
Long: common.GetLongCommandDescription(`Purge all project images in the container registry.
WARNING: Images that are being used in the Kubernetes cluster will also be deleted.`),
RunE: func(cmd *cobra.Command, args []string) error {
@@ -150,7 +150,7 @@ func runPurge() error {

stagesStorageAddress, err := common.GetStagesStorageAddress(&commonCmdData)
if err != nil {
logboek.Context(ctx).Default().LogLnDetails(`The "werf purge" command is only used to cleaning the container registry. In case you need to clean the runner or the localhost, use the commands of the "werf host" group.
logboek.Context(ctx).Default().LogLnDetails(`The "werf purge" command is only used to cleaning the container registry. In case you need to clean the runner or the localhost, use the commands of the "werf host" group.
It is worth noting that auto-cleaning is enabled by default, and manual use is usually not required (if not, we would appreciate feedback and creating an issue https://github.com/werf/werf/issues/new).`)

return err
@@ -194,10 +194,8 @@ It is worth noting that auto-cleaning is enabled by default, and manual use is u
DryRun: *commonCmdData.DryRun,
}

logboek.LogOptionalLn()
if err := cleaning.Purge(ctx, projectName, storageManager, storageLockManager, purgeOptions); err != nil {
return err
}

return nil
return manager.RetryOnStagesStorageCacheResetError(ctx, storageManager, func() error {
logboek.LogOptionalLn()
return cleaning.Purge(ctx, projectName, storageManager, storageLockManager, purgeOptions)
})
}
@@ -3,7 +3,7 @@
{% else %}
{% assign header = "###" %}
{% endif %}
Purge all project images in the container registry.
Purge all project images in the container registry.

WARNING: Images that are being used in the Kubernetes cluster will also be deleted.

@@ -3,7 +3,6 @@ package build
import (
"context"

"github.com/werf/logboek"
"github.com/werf/werf/pkg/config"
"github.com/werf/werf/pkg/container_runtime"
"github.com/werf/werf/pkg/giterminism_manager"
@@ -45,37 +44,22 @@ func (wrapper *ConveyorWithRetryWrapper) Terminate() error {
}

func (wrapper *ConveyorWithRetryWrapper) WithRetryBlock(ctx context.Context, f func(c *Conveyor) error) error {
Retry:
newConveyor := NewConveyor(
wrapper.WerfConfig,
wrapper.GiterminismManager,
wrapper.ImageNamesToProcess,
wrapper.ProjectDir,
wrapper.BaseTmpDir,
wrapper.SshAuthSock,
wrapper.ContainerRuntime,
wrapper.StorageManager,
wrapper.StorageLockManager,
wrapper.ConveyorOptions,
)
return manager.RetryOnStagesStorageCacheResetError(ctx, wrapper.StorageManager, func() error {
newConveyor := NewConveyor(
wrapper.WerfConfig,
wrapper.GiterminismManager,
wrapper.ImageNamesToProcess,
wrapper.ProjectDir,
wrapper.BaseTmpDir,
wrapper.SshAuthSock,
wrapper.ContainerRuntime,
wrapper.StorageManager,
wrapper.StorageLockManager,
wrapper.ConveyorOptions,
)

if shouldRetry, err := func() (bool, error) {
defer newConveyor.Terminate(ctx)

if err := f(newConveyor); manager.ShouldResetStagesStorageCache(err) {
logboek.Context(ctx).Error().LogF("Will reset stages storage cache due to error: %s\n", err)

if err := newConveyor.StorageManager.ResetStagesStorageCache(ctx); err != nil {
return false, err
}
return true, nil
} else {
return false, err
}
}(); err != nil {
return err
} else if shouldRetry {
goto Retry
}
return nil
return f(newConveyor)
})
}
@@ -80,6 +80,23 @@ func ShouldResetStagesStorageCache(err error) bool {
return false
}

func RetryOnStagesStorageCacheResetError(ctx context.Context, manager StorageManagerInterface, f func() error) error {
Retry:
err := f()

if ShouldResetStagesStorageCache(err) {
logboek.Context(ctx).Error().LogF("Will reset stages storage cache due to error: %s\n", err)

if err := manager.ResetStagesStorageCache(ctx); err != nil {
return fmt.Errorf("unable to reset stages storage cache: %s", err)
}

goto Retry
}

return err
}

func NewStorageManager(projectName string, stagesStorage storage.StagesStorage, finalStagesStorage storage.StagesStorage, secondaryStagesStorageList []storage.StagesStorage, cacheStagesStorageList []storage.StagesStorage, storageLockManager storage.LockManager, stagesStorageCache storage.StagesStorageCache) *StorageManager {
return &StorageManager{
ProjectName: projectName,

0 comments on commit dd43b68

Please sign in to comment.