Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
fix(multiarch): publish git metadata for multiplatform mode images 
Introduced MultiplatformImage descriptor to manage multiplatform images and maintain list of multiplatform images in the ImagesTree primitive.

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
@distorhead
distorhead committed Apr 6, 2023
1 parent e685e5e commit 1913c95
Show file tree
Hide file tree
Showing 3 changed files with 131 additions and 58 deletions.
110 changes: 52 additions & 58 deletions pkg/build/build_phase.go
View file
Expand Up @@ -245,7 +245,13 @@ func (phase *BuildPhase) publishImageMetadata(ctx context.Context, name string,
if !phase.BuildPhaseOptions.SkipImageMetadataPublication {
if err := logboek.Context(ctx).Info().
LogProcess(fmt.Sprintf("Publish image %s git metadata", img.GetName())).
DoError(func() error { return phase.publishImageGitMetadata(ctx, img) }); err != nil {
DoError(func() error {
return phase.publishImageGitMetadata(
ctx, img.GetName(),
img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription().Info.Repository,
*img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription().StageID,
)
}); err != nil {
return err
}
}
Expand Down Expand Up @@ -286,72 +292,35 @@ func (phase *BuildPhase) publishImageMetadata(ctx context.Context, name string,
return nil
}

func calculateMuiltiplatformStageDigest(images []*image.Image) string {
metaStageDeps := util.MapFuncToSlice(images, func(img *image.Image) string {
stageDesc := img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription()
return stageDesc.StageID.String()
})
return util.Sha3_224Hash(metaStageDeps...)
}

func getImagesInfoList(images []*image.Image) []*imagePkg.Info {
return util.MapFuncToSlice(images, func(img *image.Image) *imagePkg.Info {
stageDesc := img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription()
return stageDesc.Info
})
}

func (phase *BuildPhase) publishMultiplatformImageMetadata(ctx context.Context, name string, images []*image.Image) error {
if err := phase.addManagedImage(ctx, name); err != nil {
return err
}

digest := calculateMuiltiplatformStageDigest(images)
allPlatformsImages := getImagesInfoList(images)
_, uniqueID := phase.Conveyor.StorageManager.GenerateStageUniqueID(digest, nil)
metaStageID := imagePkg.StageID{Digest: digest, UniqueID: uniqueID}
opts := image.MultiplatformImageOptions{
IsArtifact: images[0].IsArtifact,
IsDockerfileImage: images[0].IsDockerfileImage,
IsDockerfileTargetStage: images[0].IsDockerfileTargetStage,
}

img := image.NewMultiplatformImage(name, images, phase.Conveyor.StorageManager, opts)
stagesStorage := phase.Conveyor.StorageManager.GetStagesStorage()

// FIXME(multiarch): Copy manifest list into final stages storage.
// FIXME(multiarch): Git metadata for artifacts and stages per platform or

// if !phase.BuildPhaseOptions.SkipImageMetadataPublication {
// if err := logboek.Context(ctx).Info().
// LogProcess(fmt.Sprintf("Publish image %s git metadata", img.GetName())).
// DoError(func() error { return phase.publishImageGitMetadata(ctx, img) }); err != nil {
// return err
// }
// }

// if img.IsArtifact {
// return nil
// }
// if img.IsDockerfileImage && !img.IsDockerfileTargetStage {
// return nil
// }

// if phase.Conveyor.StorageManager.GetFinalStagesStorage() != nil {
// if err := phase.Conveyor.StorageManager.CopyStageIntoFinalStorage(ctx, img.GetLastNonEmptyStage(), phase.Conveyor.ContainerBackend, manager.CopyStageIntoFinalStorageOptions{ShouldBeBuiltMode: phase.ShouldBeBuiltMode}); err != nil {
// return err
// }
// }

if len(phase.CustomTagFuncList) == 0 {
platforms := util.MapFuncToSlice(images, func(img *image.Image) string { return img.TargetPlatform })

container_backend.LogImageName(ctx, fmt.Sprintf("%s:%s", stagesStorage.Address(), metaStageID))
container_backend.LogImageName(ctx, fmt.Sprintf("%s:%s", stagesStorage.Address(), img.GetStageID()))
container_backend.LogMultiplatformImageInfo(ctx, platforms)

if err := stagesStorage.PostMultiplatformImage(ctx, phase.Conveyor.ProjectName(), metaStageID.String(), allPlatformsImages); err != nil {
return fmt.Errorf("unable to post multiplatform image %s %s: %w", name, metaStageID, err)
if err := stagesStorage.PostMultiplatformImage(ctx, phase.Conveyor.ProjectName(), img.GetStageID().String(), img.GetImagesInfoList()); err != nil {
return fmt.Errorf("unable to post multiplatform image %s %s: %w", name, img.GetStageID(), err)
}
} else {
for _, tagFunc := range phase.CustomTagFuncList {
tag := tagFunc(name, metaStageID.String())
tag := tagFunc(name, img.GetStageID().String())

if err := stagesStorage.PostMultiplatformImage(ctx, phase.Conveyor.ProjectName(), tag, allPlatformsImages); err != nil {
return fmt.Errorf("unable to post multiplatform image %s %s: %w", name, metaStageID, err)
if err := stagesStorage.PostMultiplatformImage(ctx, phase.Conveyor.ProjectName(), tag, img.GetImagesInfoList()); err != nil {
return fmt.Errorf("unable to post multiplatform image %s %s: %w", name, img.GetStageID(), err)
}

// FIXME(multiarch): custom tag registration for cleanup, shall we do it?
Expand All @@ -361,6 +330,33 @@ func (phase *BuildPhase) publishMultiplatformImageMetadata(ctx context.Context,
}
}

if !phase.BuildPhaseOptions.SkipImageMetadataPublication {
if err := logboek.Context(ctx).Info().
LogProcess(fmt.Sprintf("Publish multiarch image %s git metadata", name)).
DoError(func() error {
return phase.publishImageGitMetadata(ctx, name, stagesStorage.Address(), img.GetStageID())
}); err != nil {
return err
}
}

if img.IsArtifact {
return nil
}
if img.IsDockerfileImage && !img.IsDockerfileTargetStage {
return nil
}

if phase.Conveyor.StorageManager.GetFinalStagesStorage() != nil {
// FIXME(multiarch): copy manifest list into final stages storage with all dependant images
// if err := phase.Conveyor.StorageManager.CopyStageIntoFinalStorage(
// ctx, img.GetLastNonEmptyStage(), phase.Conveyor.ContainerBackend,
// manager.CopyStageIntoFinalStorageOptions{ShouldBeBuiltMode: phase.ShouldBeBuiltMode},
// ); err != nil {
// return err
// }
}

return nil
}

Expand Down Expand Up @@ -490,7 +486,7 @@ func (phase *BuildPhase) addManagedImage(ctx context.Context, name string) error
return nil
}

func (phase *BuildPhase) publishImageGitMetadata(ctx context.Context, img *image.Image) error {
func (phase *BuildPhase) publishImageGitMetadata(ctx context.Context, imageName, repository string, stageID imagePkg.StageID) error {
var commits []string

headCommit := phase.Conveyor.giterminismManager.HeadCommit()
Expand All @@ -507,22 +503,20 @@ func (phase *BuildPhase) publishImageGitMetadata(ctx context.Context, img *image

stagesStorage := phase.Conveyor.StorageManager.GetStagesStorage()

info := img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription().Info

logboek.Context(ctx).Info().LogF("name: %s:%s\n", info.Repository, info.Tag)
logboek.Context(ctx).Info().LogF("name: %s:%s\n", repository, stageID)
logboek.Context(ctx).Info().LogF("commits:\n")

for _, commit := range commits {
logboek.Context(ctx).Info().LogF(" %s\n", commit)

exist, err := stagesStorage.IsImageMetadataExist(ctx, phase.Conveyor.ProjectName(), img.GetName(), commit, img.GetStageID(), storage.WithCache())
exist, err := stagesStorage.IsImageMetadataExist(ctx, phase.Conveyor.ProjectName(), imageName, commit, stageID.String(), storage.WithCache())
if err != nil {
return fmt.Errorf("unable to get image %s metadata by commit %s and stage ID %s: %w", img.GetName(), commit, img.GetStageID(), err)
return fmt.Errorf("unable to get image %s metadata by commit %s and stage ID %s: %w", imageName, commit, stageID.String(), err)
}

if !exist {
if err := stagesStorage.PutImageMetadata(ctx, phase.Conveyor.ProjectName(), img.GetName(), commit, img.GetStageID()); err != nil {
return fmt.Errorf("unable to put image %s metadata by commit %s and stage ID %s: %w", img.GetName(), commit, img.GetStageID(), err)
if err := stagesStorage.PutImageMetadata(ctx, phase.Conveyor.ProjectName(), imageName, commit, stageID.String()); err != nil {
return fmt.Errorf("unable to put image %s metadata by commit %s and stage ID %s: %w", imageName, commit, stageID.String(), err)
}
}
}
Expand Down
20 changes: 20 additions & 0 deletions pkg/build/image/image_tree.go
View file
Expand Up @@ -27,6 +27,8 @@ type ImagesTree struct {

allImages []*Image
imagesSets ImagesSets

multiplatformImages []*MultiplatformImage
}

type ImagesTreeOptions struct {
Expand Down Expand Up @@ -187,6 +189,24 @@ func (tree *ImagesTree) GetImagesSets() ImagesSets {
return tree.imagesSets
}

func (tree *ImagesTree) GetMultiplatformImage(name string) *MultiplatformImage {
for _, img := range tree.multiplatformImages {
if img.Name == name {
return img
}
}
return nil
}

func (tree *ImagesTree) SetMultiplatformImage(newImg *MultiplatformImage) {
for _, img := range tree.multiplatformImages {
if img.Name == newImg.Name {
return
}
}
tree.multiplatformImages = append(tree.multiplatformImages, newImg)
}

func getFromFields(imageBaseConfig *config.StapelImageBase) (string, string, bool) {
var from string
var fromImageName string
Expand Down
59 changes: 59 additions & 0 deletions pkg/build/image/multiplatform_image.go
View file
@@ -0,0 +1,59 @@
package image

import (
common_image "github.com/werf/werf/pkg/image"
"github.com/werf/werf/pkg/storage/manager"
"github.com/werf/werf/pkg/util"
)

type MultiplatformImage struct {
Name string
Images []*Image

MultiplatformImageOptions

calculatedDigest string
stageID common_image.StageID
}

type MultiplatformImageOptions struct {
IsArtifact, IsDockerfileImage, IsDockerfileTargetStage bool
}

func NewMultiplatformImage(name string, images []*Image, storageManager manager.StorageManagerInterface, opts MultiplatformImageOptions) *MultiplatformImage {
img := &MultiplatformImage{
Name: name,
Images: images,
MultiplatformImageOptions: opts,
}

metaStageDeps := util.MapFuncToSlice(images, func(img *Image) string {
stageDesc := img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription()
return stageDesc.StageID.String()
})
img.calculatedDigest = util.Sha3_224Hash(metaStageDeps...)

_, uniqueID := storageManager.GenerateStageUniqueID(img.GetDigest(), nil)
img.stageID = common_image.StageID{Digest: img.GetDigest(), UniqueID: uniqueID}

return img
}

func (img *MultiplatformImage) GetPlatforms() []string {
return util.MapFuncToSlice(img.Images, func(img *Image) string { return img.TargetPlatform })
}

func (img *MultiplatformImage) GetDigest() string {
return img.calculatedDigest
}

func (img *MultiplatformImage) GetStageID() common_image.StageID {
return img.stageID
}

func (img *MultiplatformImage) GetImagesInfoList() []*common_image.Info {
return util.MapFuncToSlice(img.Images, func(img *Image) *common_image.Info {
stageDesc := img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription()
return stageDesc.Info
})
}

0 comments on commit 1913c95

Please sign in to comment.