Skip to content

Commit 04404a3

Browse files
alexey-igrychevalexey-igrychev
committed
feat(cleanup): cleaning up artifacts by git history-based policy as well as images
Update managed image and image metadata format - support spaces in image name (for artifacts) - support image name of any length - update naming Treat artifacts as images - add artifacts to managed images - put image metadata (commits for git history based cleanup) for artifacts Signed-off-by: Alexey Igrychev <alexey.igrychev@flant.com>
1 parent b59f07c commit 04404a3

File tree

5 files changed

+93
-71
lines changed

5 files changed

+93
-71
lines changed

‎pkg/build/build_phase.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -225,10 +225,6 @@ func (phase *BuildPhase) AfterImageStages(ctx context.Context, img *Image) error
225225
img.SetLastNonEmptyStage(phase.StagesIterator.PrevNonEmptyStage)
226226
img.SetContentDigest(phase.StagesIterator.PrevNonEmptyStage.GetContentDigest())
227227

228-
if img.isArtifact {
229-
return nil
230-
}
231-
232228
if err := phase.addManagedImage(ctx, img); err != nil {
233229
return err
234230
}
@@ -237,6 +233,10 @@ func (phase *BuildPhase) AfterImageStages(ctx context.Context, img *Image) error
237233
return err
238234
}
239235

236+
if img.isArtifact {
237+
return nil
238+
}
239+
240240
if !phase.ShouldBeBuiltMode {
241241
if err := phase.addCustomImageTagsToStagesStorage(ctx, img); err != nil {
242242
return fmt.Errorf("unable to add custom image tags to stages storage: %s", err)

‎pkg/slug/slug.go‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ var (
1616
DefaultSlugMaxSize = 42 // legacy
1717

1818
dockerTagRegexp = regexp.MustCompile(`^[\w][\w.-]*$`)
19-
dockerTagMaxSize = 128
19+
DockerTagMaxSize = 128
2020

2121
projectNameRegex = regexp.MustCompile(`^(?:[a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9])$`)
2222
projectNameMaxSize = 50
@@ -61,13 +61,21 @@ func validateProject(name string) error {
6161

6262
func DockerTag(name string) string {
6363
if err := ValidateDockerTag(name); err != nil {
64-
return slug(name, dockerTagMaxSize)
64+
return slug(name, DockerTagMaxSize)
6565
}
6666
return name
6767
}
6868

69+
func IsValidDockerTag(name string) bool {
70+
if shouldNotBeSlugged(name, dockerTagRegexp, DockerTagMaxSize) {
71+
return true
72+
}
73+
74+
return false
75+
}
76+
6977
func ValidateDockerTag(name string) error {
70-
if shouldNotBeSlugged(name, dockerTagRegexp, dockerTagMaxSize) {
78+
if IsValidDockerTag(name) {
7179
return nil
7280
}
7381

‎pkg/slug/slug_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -78,8 +78,8 @@ func TestDockerTag(t *testing.T) {
7878
},
7979
{
8080
name: "maxSizeExceeded",
81-
data: strings.Repeat("x", dockerTagMaxSize+1),
82-
result: strings.Repeat("x", dockerTagMaxSize-servicePartSize) + "-8cca70eb",
81+
data: strings.Repeat("x", DockerTagMaxSize+1),
82+
result: strings.Repeat("x", DockerTagMaxSize-servicePartSize) + "-8cca70eb",
8383
},
8484
}
8585

@@ -90,8 +90,8 @@ func TestDockerTag(t *testing.T) {
9090
t.Errorf("\n[EXPECTED]: %s (%d)\n[GOT]: %s (%d)", test.result, len(test.result), result, len(result))
9191
}
9292

93-
if len(result) > dockerTagMaxSize {
94-
t.Errorf("Max size exceeded: [EXPECTED]: %d [GOT]: %d", dockerTagMaxSize, len(result))
93+
if len(result) > DockerTagMaxSize {
94+
t.Errorf("Max size exceeded: [EXPECTED]: %d [GOT]: %d", DockerTagMaxSize, len(result))
9595
}
9696
})
9797

‎pkg/storage/repo_stages_storage.go‎

Lines changed: 62 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -411,14 +411,10 @@ func (storage *RepoStagesStorage) GetStageCustomTagMetadataIDs(ctx context.Conte
411411
return res, nil
412412
}
413413

414-
func (storage *RepoStagesStorage) AddManagedImage(ctx context.Context, projectName, imageName string) error {
415-
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.AddManagedImage %s %s\n", projectName, imageName)
414+
func (storage *RepoStagesStorage) AddManagedImage(ctx context.Context, projectName, imageNameOrManagedImageName string) error {
415+
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.AddManagedImage %s %s\n", projectName, imageNameOrManagedImageName)
416416

417-
if validateImageName(imageName) != nil {
418-
return nil
419-
}
420-
421-
fullImageName := makeRepoManagedImageRecord(storage.RepoAddress, imageName)
417+
fullImageName := makeRepoManagedImageRecord(storage.RepoAddress, imageNameOrManagedImageName)
422418
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.AddManagedImage full image name: %s\n", fullImageName)
423419

424420
if isExists, err := storage.DockerRegistry.IsRepoImageExists(ctx, fullImageName); err != nil {
@@ -439,10 +435,10 @@ func (storage *RepoStagesStorage) AddManagedImage(ctx context.Context, projectNa
439435
return nil
440436
}
441437

442-
func (storage *RepoStagesStorage) RmManagedImage(ctx context.Context, projectName, imageName string) error {
443-
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.RmManagedImage %s %s\n", projectName, imageName)
438+
func (storage *RepoStagesStorage) RmManagedImage(ctx context.Context, projectName, imageNameOrManagedImageName string) error {
439+
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.RmManagedImage %s %s\n", projectName, imageNameOrManagedImageName)
444440

445-
fullImageName := makeRepoManagedImageRecord(storage.RepoAddress, imageName)
441+
fullImageName := makeRepoManagedImageRecord(storage.RepoAddress, imageNameOrManagedImageName)
446442

447443
imgInfo, err := storage.DockerRegistry.TryGetRepoImage(ctx, fullImageName)
448444
if err != nil {
@@ -474,12 +470,7 @@ func (storage *RepoStagesStorage) GetManagedImages(ctx context.Context, projectN
474470
continue
475471
}
476472

477-
managedImageName := unslugDockerImageTagAsImageName(strings.TrimPrefix(tag, RepoManagedImageRecord_ImageTagPrefix))
478-
479-
if validateImageName(managedImageName) != nil {
480-
continue
481-
}
482-
473+
managedImageName := getManagedImageNameFromManagedImageID(strings.TrimPrefix(tag, RepoManagedImageRecord_ImageTagPrefix))
483474
res = append(res, managedImageName)
484475
}
485476
}
@@ -522,26 +513,26 @@ func (storage *RepoStagesStorage) ShouldFetchImage(ctx context.Context, img cont
522513
return true, nil
523514
}
524515

525-
func (storage *RepoStagesStorage) PutImageMetadata(ctx context.Context, projectName, imageName, commit, stageID string) error {
526-
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.PutImageMetadata %s %s %s %s\n", projectName, imageName, commit, stageID)
516+
func (storage *RepoStagesStorage) PutImageMetadata(ctx context.Context, projectName, imageNameOrManagedImageName, commit, stageID string) error {
517+
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.PutImageMetadata %s %s %s %s\n", projectName, imageNameOrManagedImageName, commit, stageID)
527518

528-
fullImageName := makeRepoImageMetadataName(storage.RepoAddress, imageName, commit, stageID)
519+
fullImageName := makeRepoImageMetadataName(storage.RepoAddress, imageNameOrManagedImageName, commit, stageID)
529520
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.PutImageMetadata full image name: %s\n", fullImageName)
530521

531522
opts := &docker_registry.PushImageOptions{Labels: map[string]string{image.WerfLabel: projectName}}
532523

533524
if err := storage.DockerRegistry.PushImage(ctx, fullImageName, opts); err != nil {
534525
return fmt.Errorf("unable to push image %s: %s", fullImageName, err)
535526
}
536-
logboek.Context(ctx).Info().LogF("Put image %s commit %s stage ID %s\n", imageName, commit, stageID)
527+
logboek.Context(ctx).Info().LogF("Put image %s commit %s stage ID %s\n", imageNameOrManagedImageName, commit, stageID)
537528

538529
return nil
539530
}
540531

541-
func (storage *RepoStagesStorage) RmImageMetadata(ctx context.Context, projectName, imageNameOrID, commit, stageID string) error {
542-
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.RmImageMetadata %s %s %s %s\n", projectName, imageNameOrID, commit, stageID)
532+
func (storage *RepoStagesStorage) RmImageMetadata(ctx context.Context, projectName, imageNameOrManagedImageNameOrImageMetadataID, commit, stageID string) error {
533+
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.RmImageMetadata %s %s %s %s\n", projectName, imageNameOrManagedImageNameOrImageMetadataID, commit, stageID)
543534

544-
img, err := storage.selectMetadataNameImage(ctx, imageNameOrID, commit, stageID)
535+
img, err := storage.selectMetadataNameImage(ctx, imageNameOrManagedImageNameOrImageMetadataID, commit, stageID)
545536
if err != nil {
546537
return err
547538
}
@@ -555,7 +546,7 @@ func (storage *RepoStagesStorage) RmImageMetadata(ctx context.Context, projectNa
555546
return fmt.Errorf("unable to remove repo image %s: %s", img.Tag, err)
556547
}
557548

558-
logboek.Context(ctx).Info().LogF("Removed image %s commit %s stage ID %s\n", imageNameOrID, commit, stageID)
549+
logboek.Context(ctx).Info().LogF("Removed image %s commit %s stage ID %s\n", imageNameOrManagedImageNameOrImageMetadataID, commit, stageID)
559550

560551
return nil
561552
}
@@ -575,25 +566,25 @@ func (storage *RepoStagesStorage) selectMetadataNameImage(ctx context.Context, i
575566
return nil, nil
576567
}
577568

578-
func (storage *RepoStagesStorage) IsImageMetadataExist(ctx context.Context, projectName, imageName, commit, stageID string) (bool, error) {
579-
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.IsImageMetadataExist %s %s %s %s\n", projectName, imageName, commit, stageID)
569+
func (storage *RepoStagesStorage) IsImageMetadataExist(ctx context.Context, projectName, imageNameOrManagedImageName, commit, stageID string) (bool, error) {
570+
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.IsImageMetadataExist %s %s %s %s\n", projectName, imageNameOrManagedImageName, commit, stageID)
580571

581-
fullImageName := makeRepoImageMetadataName(storage.RepoAddress, imageName, commit, stageID)
572+
fullImageName := makeRepoImageMetadataName(storage.RepoAddress, imageNameOrManagedImageName, commit, stageID)
582573
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.IsImageMetadataExist full image name: %s\n", fullImageName)
583574

584575
img, err := storage.DockerRegistry.TryGetRepoImage(ctx, fullImageName)
585576
return img != nil, err
586577
}
587578

588-
func (storage *RepoStagesStorage) GetAllAndGroupImageMetadataByImageName(ctx context.Context, projectName string, imageNameList []string) (map[string]map[string][]string, map[string]map[string][]string, error) {
579+
func (storage *RepoStagesStorage) GetAllAndGroupImageMetadataByImageName(ctx context.Context, projectName string, imageNameOrManagedImageList []string) (map[string]map[string][]string, map[string]map[string][]string, error) {
589580
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.GetImageNameStageIDCommitList %s %s\n", projectName)
590581

591582
tags, err := storage.DockerRegistry.Tags(ctx, storage.RepoAddress)
592583
if err != nil {
593584
return nil, nil, fmt.Errorf("unable to get repo %s tags: %s", storage.RepoAddress, err)
594585
}
595586

596-
return groupImageMetadataTagsByImageName(ctx, imageNameList, tags, RepoImageMetadataByCommitRecord_ImageTagPrefix)
587+
return groupImageMetadataTagsByImageName(ctx, imageNameOrManagedImageList, tags, RepoImageMetadataByCommitRecord_ImageTagPrefix)
597588
}
598589

599590
func (storage *RepoStagesStorage) GetImportMetadata(ctx context.Context, _, id string) (*ImportMetadata, error) {
@@ -680,10 +671,10 @@ func makeRepoImportMetadataName(repoAddress, importSourceID string) string {
680671
return fmt.Sprintf(RepoImportMetadata_ImageNameFormat, repoAddress, importSourceID)
681672
}
682673

683-
func groupImageMetadataTagsByImageName(ctx context.Context, imageNameList []string, tags []string, imageTagPrefix string) (map[string]map[string][]string, map[string]map[string][]string, error) {
684-
imageNameNameByID := map[string]string{}
685-
for _, imageName := range imageNameList {
686-
imageNameNameByID[imageNameID(imageName)] = imageName
674+
func groupImageMetadataTagsByImageName(ctx context.Context, imageNameOrManagedImageList []string, tags []string, imageTagPrefix string) (map[string]map[string][]string, map[string]map[string][]string, error) {
675+
imageMetadataIDImageNameOrManagedImageName := map[string]string{}
676+
for _, imageNameOrManagedImageName := range imageNameOrManagedImageList {
677+
imageMetadataIDImageNameOrManagedImageName[getImageMetadataID(imageNameOrManagedImageName)] = imageNameOrManagedImageName
687678
}
688679

689680
result := map[string]map[string][]string{}
@@ -708,7 +699,7 @@ func groupImageMetadataTagsByImageName(ctx context.Context, imageNameList []stri
708699

709700
logboek.Context(ctx).Debug().LogF("Found image ID %s commit %s stage ID %s\n", tagImageNameID, tagCommit, tagStageID)
710701

711-
imageName, ok := imageNameNameByID[tagImageNameID]
702+
imageName, ok := imageMetadataIDImageNameOrManagedImageName[tagImageNameID]
712703
if !ok {
713704
res = resultNotManagedImageName
714705
imageName = tagImageNameID
@@ -734,15 +725,20 @@ func groupImageMetadataTagsByImageName(ctx context.Context, imageNameList []stri
734725
return result, resultNotManagedImageName, nil
735726
}
736727

737-
func makeRepoImageMetadataName(repoAddress, imageNameOrID, commit, stageID string) string {
738-
return makeRepoImageMetadataNameByImageID(repoAddress, imageNameID(imageNameOrID), commit, stageID)
728+
func makeRepoImageMetadataName(repoAddress, imageNameOrManagedImageName, commit, stageID string) string {
729+
return strings.Join([]string{repoAddress, makeRepoImageMetadataTagName(imageNameOrManagedImageName, commit, stageID)}, ":")
730+
}
731+
732+
func makeRepoImageMetadataNameByImageMetadataID(repoAddress, imageMetadataID, commit, stageID string) string {
733+
return strings.Join([]string{repoAddress, makeRepoImageMetadataTagNameByImageID(imageMetadataID, commit, stageID)}, ":")
739734
}
740735

741-
func makeRepoImageMetadataNameByImageID(repoAddress, imageID, commit, stageID string) string {
742-
return strings.Join([]string{
743-
repoAddress,
744-
slug.DockerTag(fmt.Sprintf(RepoImageMetadataByCommitRecord_TagFormat, imageID, commit, stageID)),
745-
}, ":")
736+
func makeRepoImageMetadataTagName(imageNameOrManagedImageName, commit, stageID string) string {
737+
return makeRepoImageMetadataTagNameByImageID(getImageMetadataID(imageNameOrManagedImageName), commit, stageID)
738+
}
739+
740+
func makeRepoImageMetadataTagNameByImageID(imageMetadataID, commit, stageID string) string {
741+
return fmt.Sprintf(RepoImageMetadataByCommitRecord_TagFormat, imageMetadataID, commit, stageID)
746742
}
747743

748744
func (storage *RepoStagesStorage) String() string {
@@ -757,16 +753,34 @@ func makeRepoCustomTagMetadataRecord(repoAddress, tag string) string {
757753
return fmt.Sprintf(RepoCustomTagMetadata_ImageNameFormat, repoAddress, slug.LimitedSlug(tag, 48))
758754
}
759755

760-
func makeRepoManagedImageRecord(repoAddress, imageName string) string {
761-
return fmt.Sprintf(RepoManagedImageRecord_ImageNameFormat, repoAddress, slugImageNameAsDockerImageTag(imageName))
756+
func makeRepoManagedImageRecord(repoAddress, imageNameOrManagedImageName string) string {
757+
return fmt.Sprintf(RepoManagedImageRecord_ImageNameFormat, repoAddress, getManagedImageID(imageNameOrManagedImageName))
762758
}
763759

764-
func imageNameID(imageName string) string {
765-
return util.MurmurHash(imageName)
760+
func getManagedImageID(imageNameOrManagedImageName string) string {
761+
imageNameOrManagedImageName = slugImageName(imageNameOrManagedImageName)
762+
if !slug.IsValidDockerTag(imageNameOrManagedImageName) {
763+
imageNameOrManagedImageName = slug.LimitedSlug(imageNameOrManagedImageName, slug.DockerTagMaxSize-len(RepoManagedImageRecord_ImageTagPrefix))
764+
}
765+
766+
return imageNameOrManagedImageName
767+
}
768+
769+
func getManagedImageNameFromManagedImageID(managedImageID string) string {
770+
return unslugImageName(managedImageID)
766771
}
767772

768-
func slugImageNameAsDockerImageTag(imageName string) string {
773+
func getImageMetadataID(imageNameOrManagedImageName string) string {
774+
return util.MurmurHash(getManagedImageNameByImageNameOrManagedImage(imageNameOrManagedImageName))
775+
}
776+
777+
func getManagedImageNameByImageNameOrManagedImage(imageNameOrManagedImageName string) string {
778+
return getManagedImageNameFromManagedImageID(getManagedImageID(imageNameOrManagedImageName))
779+
}
780+
781+
func slugImageName(imageName string) string {
769782
res := imageName
783+
res = strings.ReplaceAll(res, " ", "__space__")
770784
res = strings.ReplaceAll(res, "/", "__slash__")
771785
res = strings.ReplaceAll(res, "+", "__plus__")
772786

@@ -777,8 +791,9 @@ func slugImageNameAsDockerImageTag(imageName string) string {
777791
return res
778792
}
779793

780-
func unslugDockerImageTagAsImageName(tag string) string {
794+
func unslugImageName(tag string) string {
781795
res := tag
796+
res = strings.ReplaceAll(res, "__space__", " ")
782797
res = strings.ReplaceAll(res, "__slash__", "/")
783798
res = strings.ReplaceAll(res, "__plus__", "+")
784799

@@ -789,13 +804,6 @@ func unslugDockerImageTagAsImageName(tag string) string {
789804
return res
790805
}
791806

792-
func validateImageName(name string) error {
793-
if strings.ToLower(name) != name {
794-
return fmt.Errorf("no upcase symbols allowed")
795-
}
796-
return nil
797-
}
798-
799807
func (storage *RepoStagesStorage) GetClientIDRecords(ctx context.Context, projectName string) ([]*ClientIDRecord, error) {
800808
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.GetClientIDRecords for project %s\n", projectName)
801809

‎pkg/storage/stages_storage.go‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,14 +44,20 @@ type StagesStorage interface {
4444
CreateRepo(ctx context.Context) error
4545
DeleteRepo(ctx context.Context) error
4646

47-
AddManagedImage(ctx context.Context, projectName, imageName string) error
48-
RmManagedImage(ctx context.Context, projectName, imageName string) error
47+
// AddManagedImage adds managed image.
48+
AddManagedImage(ctx context.Context, projectName, imageNameOrManagedImageName string) error
49+
// RmManagedImage removes managed image by imageName or managedImageName.
50+
// Typically, the managedImageName is the same as the imageName, but may be different
51+
// if the name contains unsupported special characters, or
52+
// if the name exceeds the docker tag limit.
53+
RmManagedImage(ctx context.Context, projectName, imageNameOrManagedImageName string) error
54+
// GetManagedImages returns the list of managedImageName.
4955
GetManagedImages(ctx context.Context, projectName string) ([]string, error)
5056

51-
PutImageMetadata(ctx context.Context, projectName, imageName, commit, stageID string) error
52-
RmImageMetadata(ctx context.Context, projectName, imageNameOrID, commit, stageID string) error
53-
IsImageMetadataExist(ctx context.Context, projectName, imageName, commit, stageID string) (bool, error)
54-
GetAllAndGroupImageMetadataByImageName(ctx context.Context, projectName string, imageNameList []string) (map[string]map[string][]string, map[string]map[string][]string, error)
57+
PutImageMetadata(ctx context.Context, projectName, imageNameOrManagedImageName, commit, stageID string) error
58+
RmImageMetadata(ctx context.Context, projectName, imageNameOrManagedImageNameOrImageMetadataID, commit, stageID string) error
59+
IsImageMetadataExist(ctx context.Context, projectName, imageNameOrManagedImageName, commit, stageID string) (bool, error)
60+
GetAllAndGroupImageMetadataByImageName(ctx context.Context, projectName string, imageNameOrManagedImageList []string) (map[string]map[string][]string, map[string]map[string][]string, error)
5561

5662
GetImportMetadata(ctx context.Context, projectName, id string) (*ImportMetadata, error)
5763
PutImportMetadata(ctx context.Context, projectName string, metadata *ImportMetadata) error

0 commit comments

Comments
 (0)