Skip to content

Commit 78f4e20

Browse files
iapershinalexey-igrychev
iapershin
and
alexey-igrychev
authored
feat(cleanup): add disable cleanup feature (#6737)
Signed-off-by: Yaroslav Pershin <62902094+iapershin@users.noreply.github.com> Signed-off-by: Aleksei Igrychev <aleksei.igrychev@palark.com> Co-authored-by: Aleksei Igrychev <aleksei.igrychev@palark.com>
1 parent e4fdf00 commit 78f4e20

File tree

7 files changed

+95
-45
lines changed

7 files changed

+95
-45
lines changed

Diff for: ‎cmd/werf/common/conveyor_options.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,8 @@ func GetBuildOptions(ctx context.Context, commonCmdData *CmdData, werfConfig *co
9393
}
9494

9595
buildOptions = build.BuildOptions{
96-
SkipImageMetadataPublication: *commonCmdData.Dev || werfConfig.Meta.Cleanup.DisableGitHistoryBasedPolicy,
96+
SkipAddManagedImagesRecords: werfConfig.Meta.Cleanup.DisableCleanup,
97+
SkipImageMetadataPublication: *commonCmdData.Dev || werfConfig.Meta.Cleanup.DisableGitHistoryBasedPolicy || werfConfig.Meta.Cleanup.DisableCleanup,
9798
CustomTagFuncList: customTagFuncList,
9899
ImageBuildOptions: container_backend.BuildOptions{
99100
IntrospectAfterError: *commonCmdData.IntrospectAfterError,

Diff for: ‎docs/_data/werf_yaml.yml

+5
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,11 @@ sections:
137137
collapsible: true
138138
isCollapsedByDefault: false
139139
directives:
140+
- name: disable
141+
value: "bool"
142+
description:
143+
en: Disable cleanup entirely. This prevents meta images from being published to the container registry. After disabling, you can run werf cleanup to remove existing meta images
144+
ru: Отключить очистку полностью, что предотвращает публикацию мета-образов в container registry. После отключения можно выполнить werf cleanup для удаления существующих мета-образов
140145
- name: disableKubernetesBasedPolicy
141146
value: "bool"
142147
description:

Diff for: ‎pkg/build/build_phase.go

+4-1
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ type BuildOptions struct {
4848
ReportFormat ReportFormat
4949

5050
SkipImageMetadataPublication bool
51+
SkipAddManagedImagesRecords bool
5152
CustomTagFuncList []imagePkg.CustomTagFunc
5253
}
5354

@@ -800,7 +801,9 @@ func (phase *BuildPhase) onImageStage(ctx context.Context, img *image.Image, stg
800801
}
801802

802803
// Add managed image record only if there was at least one newly built stage
803-
phase.Conveyor.SetShouldAddManagedImagesRecords()
804+
if !phase.BuildOptions.SkipAddManagedImagesRecords {
805+
phase.Conveyor.SetShouldAddManagedImagesRecords()
806+
}
804807

805808
return nil
806809
}

Diff for: ‎pkg/cleaning/cleanup.go

+76-4
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,18 @@ func (m *cleanupManager) init(ctx context.Context) error {
120120
}
121121

122122
func (m *cleanupManager) run(ctx context.Context) error {
123+
if m.ConfigMetaCleanup.DisableCleanup {
124+
if err := m.purgeManagedImages(ctx); err != nil {
125+
return err
126+
}
127+
128+
if err := m.purgeImageMetadata(ctx); err != nil {
129+
return err
130+
}
131+
132+
return nil
133+
}
134+
123135
if err := logboek.Context(ctx).LogProcess("Fetching manifests and metadata").DoError(func() error {
124136
return m.init(ctx)
125137
}); err != nil {
@@ -195,6 +207,10 @@ func (m *cleanupManager) run(ctx context.Context) error {
195207
return nil
196208
}
197209

210+
func (m *cleanupManager) purgeManagedImages(ctx context.Context) error {
211+
return purgeManagedImages(ctx, m.ProjectName, m.StorageManager, m.DryRun)
212+
}
213+
198214
func (m *cleanupManager) purgeImageMetadata(ctx context.Context) error {
199215
return purgeImageMetadata(ctx, m.ProjectName, m.StorageManager, m.DryRun)
200216
}
@@ -682,12 +698,23 @@ func (m *cleanupManager) deleteImageMetadata(ctx context.Context, imageName stri
682698
}
683699

684700
func purgeImageMetadata(ctx context.Context, projectName string, storageManager manager.StorageManagerInterface, dryRun bool) error {
685-
return logboek.Context(ctx).Default().LogProcess("Deleting images metadata").DoError(func() error {
686-
_, imageMetadataByImageName, err := storageManager.GetStagesStorage().GetAllAndGroupImageMetadataByImageName(ctx, projectName, []string{}, storage.WithCache())
687-
if err != nil {
688-
return err
701+
var imageMetadataByImageName map[string]map[string][]string
702+
if err := logboek.Context(ctx).Default().LogProcess("Fetching images metadata").DoError(func() error {
703+
var err error
704+
_, imageMetadataByImageName, err = storageManager.GetStagesStorage().GetAllAndGroupImageMetadataByImageName(ctx, projectName, []string{}, storage.WithCache())
705+
return err
706+
}); err != nil {
707+
return err
708+
}
709+
710+
var number int
711+
for _, stageIDCommitList := range imageMetadataByImageName {
712+
for _, commitList := range stageIDCommitList {
713+
number += len(commitList)
689714
}
715+
}
690716

717+
return logboek.Context(ctx).Default().LogProcess("Deleting images metadata (%d)", number).DoError(func() error {
691718
for imageNameID, stageIDCommitList := range imageMetadataByImageName {
692719
if err := deleteImageMetadata(ctx, projectName, storageManager, imageNameID, stageIDCommitList, dryRun); err != nil {
693720
return err
@@ -698,6 +725,51 @@ func purgeImageMetadata(ctx context.Context, projectName string, storageManager
698725
})
699726
}
700727

728+
func deleteManagedImages(ctx context.Context, projectName string, storageManager manager.StorageManagerInterface, managedImages []string, dryRun bool) error {
729+
if dryRun {
730+
for _, managedImage := range managedImages {
731+
logboek.Context(ctx).Default().LogFDetails(" name: %s\n", logging.ImageLogName(managedImage))
732+
logboek.Context(ctx).LogOptionalLn()
733+
}
734+
return nil
735+
}
736+
737+
return storageManager.ForEachRmManagedImage(ctx, projectName, managedImages, func(ctx context.Context, managedImage string, err error) error {
738+
if err != nil {
739+
if err := handleDeletionError(err); err != nil {
740+
return err
741+
}
742+
743+
logboek.Context(ctx).Warn().LogF("WARNING: Managed image %s deletion failed: %s\n", managedImage, err)
744+
745+
return nil
746+
}
747+
748+
logboek.Context(ctx).Default().LogFDetails(" name: %s\n", logging.ImageLogName(managedImage))
749+
750+
return nil
751+
})
752+
}
753+
754+
func purgeManagedImages(ctx context.Context, projectName string, storageManager manager.StorageManagerInterface, dryRun bool) error {
755+
var managedImages []string
756+
if err := logboek.Context(ctx).Default().LogProcess("Fetching managed images").DoError(func() error {
757+
var err error
758+
managedImages, err = storageManager.GetStagesStorage().GetManagedImages(ctx, projectName, storage.WithCache())
759+
return err
760+
}); err != nil {
761+
return err
762+
}
763+
764+
return logboek.Context(ctx).Default().LogProcess("Deleting managed images (%d)", len(managedImages)).DoError(func() error {
765+
if err := deleteManagedImages(ctx, projectName, storageManager, managedImages, dryRun); err != nil {
766+
return err
767+
}
768+
769+
return nil
770+
})
771+
}
772+
701773
func deleteImageMetadata(ctx context.Context, projectName string, storageManager manager.StorageManagerInterface, imageNameOrID string, stageIDCommitList map[string][]string, dryRun bool) error {
702774
if dryRun {
703775
for stageID, commitList := range stageIDCommitList {

Diff for: ‎pkg/cleaning/purge.go

+5-39
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import (
66
"github.com/werf/logboek"
77
"github.com/werf/werf/v2/pkg/cleaning/stage_manager"
88
"github.com/werf/werf/v2/pkg/image"
9-
"github.com/werf/werf/v2/pkg/logging"
109
"github.com/werf/werf/v2/pkg/storage"
1110
"github.com/werf/werf/v2/pkg/storage/manager"
1211
)
@@ -59,18 +58,7 @@ func (m *purgeManager) run(ctx context.Context) error {
5958
return err
6059
}
6160

62-
if err := logboek.Context(ctx).Default().LogProcess("Deleting managed images").DoError(func() error {
63-
managedImages, err := m.StorageManager.GetStagesStorage().GetManagedImages(ctx, m.ProjectName, storage.WithCache())
64-
if err != nil {
65-
return err
66-
}
67-
68-
if err := m.deleteManagedImages(ctx, managedImages); err != nil {
69-
return err
70-
}
71-
72-
return nil
73-
}); err != nil {
61+
if err := m.purgeManagedImages(ctx); err != nil {
7462
return err
7563
}
7664

@@ -117,36 +105,14 @@ func (m *purgeManager) deleteImportsMetadata(ctx context.Context, importsMetadat
117105
return deleteImportsMetadata(ctx, m.ProjectName, m.StorageManager, importsMetadataIDs, m.DryRun)
118106
}
119107

120-
func (m *purgeManager) deleteManagedImages(ctx context.Context, managedImages []string) error {
121-
if m.DryRun {
122-
for _, managedImage := range managedImages {
123-
logboek.Context(ctx).Default().LogFDetails(" name: %s\n", logging.ImageLogName(managedImage))
124-
logboek.Context(ctx).LogOptionalLn()
125-
}
126-
return nil
127-
}
128-
129-
return m.StorageManager.ForEachRmManagedImage(ctx, m.ProjectName, managedImages, func(ctx context.Context, managedImage string, err error) error {
130-
if err != nil {
131-
if err := handleDeletionError(err); err != nil {
132-
return err
133-
}
134-
135-
logboek.Context(ctx).Warn().LogF("WARNING: Managed image %s deletion failed: %s\n", managedImage, err)
136-
137-
return nil
138-
}
139-
140-
logboek.Context(ctx).Default().LogFDetails(" name: %s\n", logging.ImageLogName(managedImage))
141-
142-
return nil
143-
})
144-
}
145-
146108
func (m *purgeManager) purgeImageMetadata(ctx context.Context) error {
147109
return purgeImageMetadata(ctx, m.ProjectName, m.StorageManager, m.DryRun)
148110
}
149111

112+
func (m *purgeManager) purgeManagedImages(ctx context.Context) error {
113+
return purgeManagedImages(ctx, m.ProjectName, m.StorageManager, m.DryRun)
114+
}
115+
150116
func (m *purgeManager) deleteCustomTags(ctx context.Context) error {
151117
stageIDCustomTagList, err := stage_manager.GetCustomTagsMetadata(ctx, m.StorageManager)
152118
if err != nil {

Diff for: ‎pkg/config/meta_cleanup.go

+1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ var (
1818
)
1919

2020
type MetaCleanup struct {
21+
DisableCleanup bool
2122
DisableKubernetesBasedPolicy bool
2223
DisableGitHistoryBasedPolicy bool
2324
DisableBuiltWithinLastNHoursPolicy bool

Diff for: ‎pkg/config/raw_meta_cleanup.go

+2
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
const DefaultKeepImagesBuiltWithinLastNHours uint64 = 2
1111

1212
type rawMetaCleanup struct {
13+
DisableCleanup bool `yaml:"disable,omitempty"`
1314
DisableKubernetesBasedPolicy bool `yaml:"disableKubernetesBasedPolicy,omitempty"`
1415
DisableGitHistoryBasedPolicy bool `yaml:"disableGitHistoryBasedPolicy,omitempty"`
1516
DisableBuiltWithinLastNHoursPolicy bool `yaml:"disableBuiltWithinLastNHoursPolicy,omitempty"`
@@ -187,6 +188,7 @@ func (c *rawMetaCleanup) toMetaCleanup() MetaCleanup {
187188
metaCleanup.DisableKubernetesBasedPolicy = c.DisableKubernetesBasedPolicy
188189
metaCleanup.DisableBuiltWithinLastNHoursPolicy = c.DisableBuiltWithinLastNHoursPolicy
189190
metaCleanup.DisableGitHistoryBasedPolicy = c.DisableGitHistoryBasedPolicy
191+
metaCleanup.DisableCleanup = c.DisableCleanup
190192

191193
for _, policy := range c.KeepPolicies {
192194
metaCleanup.KeepPolicies = append(metaCleanup.KeepPolicies, policy.toMetaCleanupKeepPolicy())

0 commit comments

Comments
 (0)