Skip to content
Permalink
Browse files
fix(custom-tags): do not store metadata in the --final-repo
Split storage of metadata for custom tags: store metadata in the primary `--repo` and store custom tags itself in the `--final-repo` (if final-repo has been specified).

Refs #4636

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
distorhead committed Aug 24, 2022
1 parent 9fb05a4 commit 1a780c519ef4553693e2a4e5c13945d723432eea
Show file tree
Hide file tree
Showing 11 changed files with 66 additions and 32 deletions.
@@ -743,11 +743,11 @@ func GetParallelTasksLimit(cmdData *CmdData) (int64, error) {
}
}

func GetLocalStagesStorage(containerBackend container_backend.ContainerBackend) storage.StagesStorage {
func GetLocalStagesStorage(containerBackend container_backend.ContainerBackend) *storage.DockerServerStagesStorage {
return storage.NewDockerServerStagesStorage(containerBackend.(*container_backend.DockerServerBackend))
}

func GetStagesStorage(ctx context.Context, containerBackend container_backend.ContainerBackend, cmdData *CmdData) (storage.StagesStorage, error) {
func GetStagesStorage(ctx context.Context, containerBackend container_backend.ContainerBackend, cmdData *CmdData) (storage.PrimaryStagesStorage, error) {
if _, match := containerBackend.(*container_backend.BuildahBackend); match {
addr, err := cmdData.Repo.GetAddress()
if err != nil {
@@ -34,7 +34,7 @@ func (repoData *RepoData) CreateDockerRegistry(ctx context.Context, insecureRegi
return dockerRegistry, nil
}

func (repoData *RepoData) CreateStagesStorage(ctx context.Context, containerBackend container_backend.ContainerBackend, insecureRegistry, skipTlsVerifyRegistry bool) (storage.StagesStorage, error) {
func (repoData *RepoData) CreateStagesStorage(ctx context.Context, containerBackend container_backend.ContainerBackend, insecureRegistry, skipTlsVerifyRegistry bool) (storage.PrimaryStagesStorage, error) {
addr, err := repoData.GetAddress()
if err != nil {
return nil, err
@@ -32,7 +32,7 @@ func TestSuite(t *testing.T) {
var SuiteData struct {
suite_init.SuiteData
TestImplementation string
StagesStorage storage.StagesStorage
StagesStorage storage.PrimaryStagesStorage
ContainerRegistry docker_registry.Interface
}

@@ -31,7 +31,7 @@ func TestSuite(t *testing.T) {

var SuiteData = struct {
suite_init.SuiteData
StagesStorage storage.StagesStorage
StagesStorage storage.PrimaryStagesStorage
ContainerRegistry docker_registry.Interface
}{}

@@ -264,7 +264,7 @@ func (phase *BuildPhase) AfterImageStages(ctx context.Context, img *Image) error
return err
}
} else {
if err := phase.addCustomImageTagsToStagesStorage(ctx, img.GetName(), customTagStage, customTagStorage); err != nil {
if err := phase.addCustomImageTags(ctx, img.GetName(), customTagStage, customTagStorage, phase.Conveyor.StorageManager.GetStagesStorage(), phase.CustomTagFuncList); err != nil {
return fmt.Errorf("unable to add custom image tags to stages storage: %w", err)
}
}
@@ -327,11 +327,7 @@ func (phase *BuildPhase) publishImageMetadata(ctx context.Context, img *Image) e
})
}

func (phase *BuildPhase) addCustomImageTagsToStagesStorage(ctx context.Context, imageName string, stageDesc *imagePkg.StageDescription, stagesStorage storage.StagesStorage) error {
return addCustomImageTags(ctx, stagesStorage, imageName, stageDesc, phase.CustomTagFuncList)
}

func addCustomImageTags(ctx context.Context, stagesStorage storage.StagesStorage, imageName string, stageDesc *imagePkg.StageDescription, customTagFuncList []imagePkg.CustomTagFunc) error {
func (phase *BuildPhase) addCustomImageTags(ctx context.Context, imageName string, stageDesc *imagePkg.StageDescription, stagesStorage storage.StagesStorage, primaryStagesStorage storage.PrimaryStagesStorage, customTagFuncList []imagePkg.CustomTagFunc) error {
if len(customTagFuncList) == 0 {
return nil
}
@@ -343,7 +339,7 @@ func addCustomImageTags(ctx context.Context, stagesStorage storage.StagesStorage
DoError(func() error {
for _, tagFunc := range customTagFuncList {
tag := tagFunc(imageName, stageDesc.Info.Tag)
if err := addCustomImageTag(ctx, stagesStorage, stageDesc, tag); err != nil {
if err := addCustomImageTag(ctx, stagesStorage, primaryStagesStorage, stageDesc, tag); err != nil {
return err
}
}
@@ -352,11 +348,14 @@ func addCustomImageTags(ctx context.Context, stagesStorage storage.StagesStorage
})
}

func addCustomImageTag(ctx context.Context, stagesStorage storage.StagesStorage, stageDesc *imagePkg.StageDescription, tag string) error {
func addCustomImageTag(ctx context.Context, stagesStorage storage.StagesStorage, primaryStagesStorage storage.PrimaryStagesStorage, stageDesc *imagePkg.StageDescription, tag string) error {
return logboek.Context(ctx).Default().LogProcess("tag %s", tag).
DoError(func() error {
if err := stagesStorage.AddStageCustomTag(ctx, stageDesc, tag); err != nil {
return err
return fmt.Errorf("unable to add stage %s custom tag %s in the storage %s: %w", stageDesc.StageID.String(), tag, stagesStorage.String(), err)
}
if err := primaryStagesStorage.RegisterStageCustomTag(ctx, stageDesc, tag); err != nil {
return fmt.Errorf("unable to register stage %s custom tag %s in the primary storage %s: %w", stageDesc.StageID.String(), tag, primaryStagesStorage.String(), err)
}

logboek.Context(ctx).LogFDetails(" name: %s:%s\n", stageDesc.Info.Repository, tag)
@@ -147,6 +147,14 @@ func (storage *DockerServerStagesStorage) GetStageCustomTagMetadataIDs(_ context
return nil, nil
}

func (storage *DockerServerStagesStorage) RegisterStageCustomTag(_ context.Context, _ *image.StageDescription, _ string) error {
return nil
}

func (storage *DockerServerStagesStorage) UnregisterStageCustomTag(_ context.Context, _ string) error {
return nil
}

func (storage *DockerServerStagesStorage) AddManagedImage(_ context.Context, _, _ string) error {
return nil
}
@@ -50,7 +50,7 @@ type ForEachDeleteStageOptions struct {
type StorageManagerInterface interface {
InitCache(ctx context.Context) error

GetStagesStorage() storage.StagesStorage
GetStagesStorage() storage.PrimaryStagesStorage
GetFinalStagesStorage() storage.StagesStorage
GetSecondaryStagesStorageList() []storage.StagesStorage
GetImageInfoGetter(imageName string, stg stage.Interface, opts image.InfoGetterOptions) *image.InfoGetter
@@ -95,7 +95,7 @@ Retry:
return err
}

func NewStorageManager(projectName string, stagesStorage, finalStagesStorage storage.StagesStorage, secondaryStagesStorageList, cacheStagesStorageList []storage.StagesStorage, storageLockManager storage.LockManager) *StorageManager {
func NewStorageManager(projectName string, stagesStorage storage.PrimaryStagesStorage, finalStagesStorage storage.StagesStorage, secondaryStagesStorageList, cacheStagesStorageList []storage.StagesStorage, storageLockManager storage.LockManager) *StorageManager {
return &StorageManager{
ProjectName: projectName,
StorageLockManager: storageLockManager,
@@ -152,7 +152,7 @@ type StorageManager struct {

StorageLockManager storage.LockManager

StagesStorage storage.StagesStorage
StagesStorage storage.PrimaryStagesStorage
FinalStagesStorage storage.StagesStorage
CacheStagesStorageList []storage.StagesStorage
SecondaryStagesStorageList []storage.StagesStorage
@@ -164,7 +164,7 @@ type StorageManager struct {
FinalStagesListCache *StagesList
}

func (m *StorageManager) GetStagesStorage() storage.StagesStorage {
func (m *StorageManager) GetStagesStorage() storage.PrimaryStagesStorage {
return m.StagesStorage
}

@@ -1028,8 +1028,15 @@ func (m *StorageManager) ForEachDeleteStageCustomTag(ctx context.Context, ids []
MaxNumberOfWorkers: m.MaxNumberOfWorkers(),
}, func(ctx context.Context, taskId int) error {
id := ids[taskId]
err := m.StagesStorage.DeleteStageCustomTag(ctx, id)
return f(ctx, id, err)

if err := m.StagesStorage.DeleteStageCustomTag(ctx, id); err != nil {
return f(ctx, id, fmt.Errorf("unable to delete stage custom tag: %w", err))
}
if err := m.StagesStorage.UnregisterStageCustomTag(ctx, id); err != nil {
return f(ctx, id, fmt.Errorf("unable to unregister stage custom tag: %w", err))
}

return f(ctx, id, nil)
})
}

@@ -0,0 +1,16 @@
package storage

import (
"context"

"github.com/werf/werf/pkg/image"
)

type PrimaryStagesStorage interface {
StagesStorage

GetStageCustomTagMetadataIDs(ctx context.Context, opts ...Option) ([]string, error)
GetStageCustomTagMetadata(ctx context.Context, tagOrID string) (*CustomTagMetadata, error)
RegisterStageCustomTag(ctx context.Context, stageDescription *image.StageDescription, tag string) error
UnregisterStageCustomTag(ctx context.Context, tag string) error
}
@@ -296,18 +296,10 @@ func (storage *RepoStagesStorage) CheckStageCustomTag(ctx context.Context, stage
}

func (storage *RepoStagesStorage) AddStageCustomTag(ctx context.Context, stageDescription *image.StageDescription, tag string) error {
if err := storage.addStageCustomTagMetadata(ctx, stageDescription, tag); err != nil {
return fmt.Errorf("unable to add stage custom tag metadata: %w", err)
}

return storage.DockerRegistry.TagRepoImage(ctx, stageDescription.Info, tag)
}

func (storage *RepoStagesStorage) DeleteStageCustomTag(ctx context.Context, tag string) error {
if err := storage.deleteStageCustomTagMetadata(ctx, tag); err != nil {
return fmt.Errorf("unable to delete stage custom tag metadata: %w", err)
}

fullImageName := strings.Join([]string{storage.RepoAddress, tag}, ":")
imgInfo, err := storage.DockerRegistry.TryGetRepoImage(ctx, fullImageName)
if err != nil {
@@ -392,6 +384,20 @@ func (storage *RepoStagesStorage) GetStageCustomTagMetadataIDs(ctx context.Conte
return res, nil
}

func (storage *RepoStagesStorage) RegisterStageCustomTag(ctx context.Context, stageDescription *image.StageDescription, tag string) error {
if err := storage.addStageCustomTagMetadata(ctx, stageDescription, tag); err != nil {
return fmt.Errorf("unable to add stage custom tag metadata: %w", err)
}
return nil
}

func (storage *RepoStagesStorage) UnregisterStageCustomTag(ctx context.Context, tag string) error {
if err := storage.deleteStageCustomTagMetadata(ctx, tag); err != nil {
return fmt.Errorf("unable to delete stage custom tag metadata: %w", err)
}
return nil
}

func (storage *RepoStagesStorage) AddManagedImage(ctx context.Context, projectName, imageNameOrManagedImageName string) error {
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.AddManagedImage %s %s\n", projectName, imageNameOrManagedImageName)

@@ -33,8 +33,6 @@ type StagesStorage interface {
AddStageCustomTag(ctx context.Context, stageDescription *image.StageDescription, tag string) error
CheckStageCustomTag(ctx context.Context, stageDescription *image.StageDescription, tag string) error
DeleteStageCustomTag(ctx context.Context, tag string) error
GetStageCustomTagMetadataIDs(ctx context.Context, opts ...Option) ([]string, error)
GetStageCustomTagMetadata(ctx context.Context, tagOrID string) (*CustomTagMetadata, error)

RejectStage(ctx context.Context, projectName, digest string, uniqueID int64) error

@@ -10,7 +10,7 @@ import (
"github.com/werf/werf/pkg/storage"
)

func NewStagesStorage(stagesStorageAddress, implementationName string, dockerRegistryOptions docker_registry.DockerRegistryOptions) storage.StagesStorage {
func NewStagesStorage(stagesStorageAddress, implementationName string, dockerRegistryOptions docker_registry.DockerRegistryOptions) storage.PrimaryStagesStorage {
if stagesStorageAddress == storage.LocalStorageAddress {
return storage.NewDockerServerStagesStorage(&container_backend.DockerServerBackend{})
} else {
@@ -32,7 +32,7 @@ func ManagedImagesCount(ctx context.Context, stagesStorage storage.StagesStorage
return len(managedImages)
}

func CustomTagsMetadataList(ctx context.Context, stagesStorage storage.StagesStorage) []*storage.CustomTagMetadata {
func CustomTagsMetadataList(ctx context.Context, stagesStorage storage.PrimaryStagesStorage) []*storage.CustomTagMetadata {
customTagMetadataIDs, err := stagesStorage.GetStageCustomTagMetadataIDs(ctx)
Ω(err).ShouldNot(HaveOccurred())

0 comments on commit 1a780c5

Please sign in to comment.