Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
fix(multiarch): fix werf converge, kube-run, run and other commands w… 
…hen platform param specified

Conveyor images storage now can query image by name and target platform.

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
@distorhead
distorhead committed Mar 13, 2023
1 parent c329a16 commit 3e2add1
Show file tree
Hide file tree
Showing 23 changed files with 158 additions and 117 deletions.
5 changes: 4 additions & 1 deletion cmd/werf/bundle/export/export.go
View file
Expand Up @@ -311,7 +311,10 @@ func runExport(ctx context.Context, imagesToProcess build.ImagesToProcess) error
}
}

imagesInfoGetters = c.GetImageInfoGetters(image.InfoGetterOptions{CustomTagFunc: useCustomTagFunc})
imagesInfoGetters, err = c.GetImageInfoGetters(image.InfoGetterOptions{CustomTagFunc: useCustomTagFunc})
if err != nil {
return err
}

return nil
}); err != nil {
Expand Down
5 changes: 4 additions & 1 deletion cmd/werf/bundle/publish/publish.go
View file
Expand Up @@ -317,7 +317,10 @@ func runPublish(ctx context.Context, imagesToProcess build.ImagesToProcess) erro
}
}

imagesInfoGetters = c.GetImageInfoGetters(image.InfoGetterOptions{CustomTagFunc: useCustomTagFunc})
imagesInfoGetters, err = c.GetImageInfoGetters(image.InfoGetterOptions{CustomTagFunc: useCustomTagFunc})
if err != nil {
return err
}

return nil
}); err != nil {
Expand Down
4 changes: 2 additions & 2 deletions cmd/werf/compose/main.go
View file
Expand Up @@ -418,8 +418,8 @@ func run(ctx context.Context, containerBackend container_backend.ContainerBacken
}
}

for _, imageName := range c.GetExportedImagesNames() {
if err := c.FetchLastImageStage(ctx, imageName); err != nil {
for _, img := range c.GetExportedImages() {
if err := c.FetchLastImageStage(ctx, img.TargetPlatform, img.Name); err != nil {
return err
}
}
Expand Down
5 changes: 4 additions & 1 deletion cmd/werf/converge/converge.go
View file
Expand Up @@ -359,7 +359,10 @@ func run(ctx context.Context, containerBackend container_backend.ContainerBacken
}
}

imagesInfoGetters = c.GetImageInfoGetters(image.InfoGetterOptions{CustomTagFunc: useCustomTagFunc})
imagesInfoGetters, err = c.GetImageInfoGetters(image.InfoGetterOptions{CustomTagFunc: useCustomTagFunc})
if err != nil {
return err
}

return nil
}); err != nil {
Expand Down
5 changes: 4 additions & 1 deletion cmd/werf/helm/get_autogenerated_values.go
View file
Expand Up @@ -227,7 +227,10 @@ func runGetServiceValues(ctx context.Context, imagesToProcess build.ImagesToProc
}

imagesRepository = storageManager.StagesStorage.String()
imagesInfoGetters = c.GetImageInfoGetters(image.InfoGetterOptions{CustomTagFunc: useCustomTagFunc})
imagesInfoGetters, err = c.GetImageInfoGetters(image.InfoGetterOptions{CustomTagFunc: useCustomTagFunc})
if err != nil {
return err
}

return nil
}); err != nil {
Expand Down
6 changes: 4 additions & 2 deletions cmd/werf/kube_run/kube_run.go
View file
Expand Up @@ -416,11 +416,13 @@ func run(ctx context.Context, pod, secret, namespace string, werfConfig *config.
}
}

if err := c.FetchLastImageStage(ctx, imageName); err != nil {
// FIXME(multiarch): specify multiarch manifest here
if err := c.FetchLastImageStage(ctx, "", imageName); err != nil {
return err
}

image = c.GetImageNameForLastImageStage(imageName)
// FIXME(multiarch): specify multiarch manifest here
image = c.GetImageNameForLastImageStage("", imageName)
return nil
}); err != nil {
return err
Expand Down
5 changes: 4 additions & 1 deletion cmd/werf/render/render.go
View file
Expand Up @@ -348,7 +348,10 @@ func runRender(ctx context.Context, imagesToProcess build.ImagesToProcess) error
}
}

imagesInfoGetters = c.GetImageInfoGetters(image.InfoGetterOptions{CustomTagFunc: useCustomTagFunc})
imagesInfoGetters, err = c.GetImageInfoGetters(image.InfoGetterOptions{CustomTagFunc: useCustomTagFunc})
if err != nil {
return err
}

return nil
}); err != nil {
Expand Down
4 changes: 2 additions & 2 deletions cmd/werf/run/run.go
View file
Expand Up @@ -402,11 +402,11 @@ func run(ctx context.Context, containerBackend container_backend.ContainerBacken
}
}

if err := c.FetchLastImageStage(ctx, imageName); err != nil {
if err := c.FetchLastImageStage(ctx, "", imageName); err != nil {
return err
}

dockerImageName = c.GetImageNameForLastImageStage(imageName)
dockerImageName = c.GetImageNameForLastImageStage("", imageName)
return nil
}); err != nil {
return err
Expand Down
3 changes: 2 additions & 1 deletion cmd/werf/stage/image/main.go
View file
Expand Up @@ -207,7 +207,8 @@ func run(ctx context.Context, imageName string) error {
return err
}

fmt.Println(c.GetImageNameForLastImageStage(imageName))
// FIXME(multiarch): specify multiarch manifest here
fmt.Println(c.GetImageNameForLastImageStage("", imageName))

return nil
}); err != nil {
Expand Down
105 changes: 59 additions & 46 deletions pkg/build/conveyor.go
View file
Expand Up @@ -215,14 +215,18 @@ func (c *Conveyor) GetLocalGitRepoVirtualMergeOptions() stage.VirtualMergeOption
return c.ConveyorOptions.LocalGitRepoVirtualMergeOptions
}

func (c *Conveyor) GetImportServer(ctx context.Context, imageName, stageName string) (import_server.ImportServer, error) {
func (c *Conveyor) GetImportServer(ctx context.Context, targetPlatform, imageName, stageName string) (import_server.ImportServer, error) {
c.GetServiceRWMutex("ImportServer").Lock()
defer c.GetServiceRWMutex("ImportServer").Unlock()

importServerName := imageName
importServerName := fmt.Sprintf("%s")
if stageName != "" {
importServerName += "/" + stageName
}
// FIXME(multiarch): in this place we should get our current platform from the container backend in the case when targetPlatform is empty
if targetPlatform != "" && targetPlatform != "linux/amd64" {
importServerName += "[" + targetPlatform + "]"
}
if srv, hasKey := c.importServers[importServerName]; hasKey {
return srv, nil
}
Expand All @@ -232,9 +236,9 @@ func (c *Conveyor) GetImportServer(ctx context.Context, imageName, stageName str
var stg stage.Interface

if stageName != "" {
stg = c.getImageStage(imageName, stageName)
stg = c.getImageStage(targetPlatform, imageName, stageName)
} else {
stg = c.GetImage(imageName).GetLastNonEmptyStage()
stg = c.GetImage(targetPlatform, imageName).GetLastNonEmptyStage()
}

if err := c.StorageManager.FetchStage(ctx, c.ContainerBackend, stg); err != nil {
Expand All @@ -256,9 +260,9 @@ func (c *Conveyor) GetImportServer(ctx context.Context, imageName, stageName str

var dockerImageName string
if stageName == "" {
dockerImageName = c.GetImageNameForLastImageStage(imageName)
dockerImageName = c.GetImageNameForLastImageStage(targetPlatform, imageName)
} else {
dockerImageName = c.GetImageNameForImageStage(imageName, stageName)
dockerImageName = c.GetImageNameForImageStage(targetPlatform, imageName, stageName)
}

var err error
Expand Down Expand Up @@ -356,36 +360,45 @@ func (c *Conveyor) ShouldBeBuilt(ctx context.Context, opts ShouldBeBuiltOptions)
return nil
}

func (c *Conveyor) FetchLastImageStage(ctx context.Context, imageName string) error {
lastImageStage := c.GetImage(imageName).GetLastNonEmptyStage()
func (c *Conveyor) FetchLastImageStage(ctx context.Context, targetPlatform, imageName string) error {
lastImageStage := c.GetImage(targetPlatform, imageName).GetLastNonEmptyStage()
return c.StorageManager.FetchStage(ctx, c.ContainerBackend, lastImageStage)
}

func (c *Conveyor) GetImageInfoGetters(opts imagePkg.InfoGetterOptions) (images []*imagePkg.InfoGetter) {
func (c *Conveyor) GetImageInfoGetters(opts imagePkg.InfoGetterOptions) ([]*imagePkg.InfoGetter, error) {
targetPlatforms, err := c.GetTargetPlatforms()
if err != nil {
return nil, fmt.Errorf("unable to get target platforms: %w", err)
}
var targetPlatform string
if len(targetPlatforms) > 0 {
// FIXME(multiarch): instead of using first specified platform use multiarch-manifest
targetPlatform = targetPlatforms[0]
}

var images []*imagePkg.InfoGetter
for _, img := range c.imagesTree.GetImages() {
if img.IsArtifact {
continue
}
if img.TargetPlatform != targetPlatform {
continue
}

getter := c.StorageManager.GetImageInfoGetter(img.Name, img.GetLastNonEmptyStage(), opts)
images = append(images, getter)
}

return
return images, nil
}

func (c *Conveyor) GetExportedImagesNames() []string {
var res []string

func (c *Conveyor) GetExportedImages() (res []*image.Image) {
for _, img := range c.imagesTree.GetImages() {
if img.IsArtifact {
continue
}

res = append(res, img.Name)
res = append(res, img)
}

return res
return
}

func (c *Conveyor) GetImagesEnvArray() []string {
Expand All @@ -395,7 +408,7 @@ func (c *Conveyor) GetImagesEnvArray() []string {
continue
}

envArray = append(envArray, generateImageEnv(img.Name, c.GetImageNameForLastImageStage(img.Name)))
envArray = append(envArray, generateImageEnv(img.Name, c.GetImageNameForLastImageStage(img.TargetPlatform, img.Name)))
}

return envArray
Expand Down Expand Up @@ -687,67 +700,67 @@ func (c *Conveyor) GetOrCreateStageImage(name string, prevStageImage *stage.Stag
return stageImage
}

func (c *Conveyor) GetImage(name string) *image.Image {
func (c *Conveyor) GetImage(targetPlatform, name string) *image.Image {
for _, img := range c.imagesTree.GetImages() {
if img.GetName() == name {
if img.GetName() == name && img.TargetPlatform == targetPlatform {
return img
}
}

panic(fmt.Sprintf("Image %q not found!", name))
}

func (c *Conveyor) GetImageStageContentDigest(imageName, stageName string) string {
return c.getImageStage(imageName, stageName).GetContentDigest()
func (c *Conveyor) GetImageStageContentDigest(targetPlatform, imageName, stageName string) string {
return c.getImageStage(targetPlatform, imageName, stageName).GetContentDigest()
}

func (c *Conveyor) GetImageContentDigest(imageName string) string {
return c.GetImage(imageName).GetContentDigest()
func (c *Conveyor) GetImageContentDigest(targetPlatform, imageName string) string {
return c.GetImage(targetPlatform, imageName).GetContentDigest()
}

func (c *Conveyor) getImageStage(imageName, stageName string) stage.Interface {
if stg := c.GetImage(imageName).GetStage(stage.StageName(stageName)); stg != nil {
func (c *Conveyor) getImageStage(targetPlatform, imageName, stageName string) stage.Interface {
if stg := c.GetImage(targetPlatform, imageName).GetStage(stage.StageName(stageName)); stg != nil {
return stg
} else {
return c.getLastNonEmptyImageStage(imageName)
return c.getLastNonEmptyImageStage(targetPlatform, imageName)
}
}

func (c *Conveyor) getLastNonEmptyImageStage(imageName string) stage.Interface {
func (c *Conveyor) getLastNonEmptyImageStage(targetPlatform, imageName string) stage.Interface {
// FIXME: find first existing stage after specified unexisting
return c.GetImage(imageName).GetLastNonEmptyStage()
return c.GetImage(targetPlatform, imageName).GetLastNonEmptyStage()
}

func (c *Conveyor) FetchImageStage(ctx context.Context, imageName, stageName string) error {
return c.StorageManager.FetchStage(ctx, c.ContainerBackend, c.getImageStage(imageName, stageName))
func (c *Conveyor) FetchImageStage(ctx context.Context, targetPlatform, imageName, stageName string) error {
return c.StorageManager.FetchStage(ctx, c.ContainerBackend, c.getImageStage(targetPlatform, imageName, stageName))
}

func (c *Conveyor) FetchLastNonEmptyImageStage(ctx context.Context, imageName string) error {
return c.StorageManager.FetchStage(ctx, c.ContainerBackend, c.getLastNonEmptyImageStage(imageName))
func (c *Conveyor) FetchLastNonEmptyImageStage(ctx context.Context, targetPlatform, imageName string) error {
return c.StorageManager.FetchStage(ctx, c.ContainerBackend, c.getLastNonEmptyImageStage(targetPlatform, imageName))
}

func (c *Conveyor) GetImageNameForLastImageStage(imageName string) string {
return c.GetImage(imageName).GetLastNonEmptyStage().GetStageImage().Image.Name()
func (c *Conveyor) GetImageNameForLastImageStage(targetPlatform, imageName string) string {
return c.GetImage(targetPlatform, imageName).GetLastNonEmptyStage().GetStageImage().Image.Name()
}

func (c *Conveyor) GetImageNameForImageStage(imageName, stageName string) string {
return c.getImageStage(imageName, stageName).GetStageImage().Image.Name()
func (c *Conveyor) GetImageNameForImageStage(targetPlatform, imageName, stageName string) string {
return c.getImageStage(targetPlatform, imageName, stageName).GetStageImage().Image.Name()
}

func (c *Conveyor) GetStageID(imageName string) string {
return c.GetImage(imageName).GetStageID()
func (c *Conveyor) GetStageID(targetPlatform, imageName string) string {
return c.GetImage(targetPlatform, imageName).GetStageID()
}

func (c *Conveyor) GetImageIDForLastImageStage(imageName string) string {
return c.GetImage(imageName).GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription().Info.ID
func (c *Conveyor) GetImageIDForLastImageStage(targetPlatform, imageName string) string {
return c.GetImage(targetPlatform, imageName).GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription().Info.ID
}

func (c *Conveyor) GetImageDigestForLastImageStage(imageName string) string {
return c.GetImage(imageName).GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription().Info.RepoDigest
func (c *Conveyor) GetImageDigestForLastImageStage(targetPlatform, imageName string) string {
return c.GetImage(targetPlatform, imageName).GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription().Info.RepoDigest
}

func (c *Conveyor) GetImageIDForImageStage(imageName, stageName string) string {
return c.getImageStage(imageName, stageName).GetStageImage().Image.GetStageDescription().Info.ID
func (c *Conveyor) GetImageIDForImageStage(targetPlatform, imageName, stageName string) string {
return c.getImageStage(targetPlatform, imageName, stageName).GetStageImage().Image.GetStageDescription().Info.ID
}

func (c *Conveyor) GetImportMetadata(ctx context.Context, projectName, id string) (*storage.ImportMetadata, error) {
Expand Down
2 changes: 1 addition & 1 deletion pkg/build/image/conveyor.go
View file
Expand Up @@ -10,7 +10,7 @@ import (
type Conveyor interface {
stage.Conveyor

GetImage(name string) *Image
GetImage(targetPlatform, name string) *Image
GetOrCreateStageImage(name string, prevStageImage *stage.StageImage, stg stage.Interface, img *Image) *stage.StageImage
GetTargetPlatforms() ([]string, error)

Expand Down
6 changes: 4 additions & 2 deletions pkg/build/image/dockerfile.go
View file
Expand Up @@ -137,6 +137,7 @@ func mapDockerfileToImagesSets(ctx context.Context, cfg *dockerfile.Dockerfile,
stageLogName := fmt.Sprintf("%s%d", strings.ToUpper(instr.GetInstructionData().Name()), ind+1)
isFirstStage := (len(img.stages) == 0)
baseStageOptions := &stage.BaseStageOptions{
TargetPlatform: img.TargetPlatform,
ImageName: img.Name,
LogName: stageLogName,
ImageTmpDir: img.TmpDir,
Expand Down Expand Up @@ -257,8 +258,9 @@ func mapLegacyDockerfileToImage(ctx context.Context, dockerfileImageConfig *conf
)

baseStageOptions := &stage.BaseStageOptions{
ImageName: dockerfileImageConfig.Name,
ProjectName: opts.ProjectName,
TargetPlatform: opts.TargetPlatform,
ImageName: dockerfileImageConfig.Name,
ProjectName: opts.ProjectName,
}

dockerfileStage := stage.GenerateFullDockerfileStage(
Expand Down
4 changes: 2 additions & 2 deletions pkg/build/image/image.go
View file
Expand Up @@ -223,13 +223,13 @@ func isUnsupportedMediaTypeError(err error) bool {
func (i *Image) SetupBaseImage(ctx context.Context, storageManager manager.StorageManagerInterface, storageOpts manager.StorageOptions) error {
switch i.baseImageType {
case StageAsBaseImage:
i.stageAsBaseImage = i.Conveyor.GetImage(i.baseImageName).GetLastNonEmptyStage()
i.stageAsBaseImage = i.Conveyor.GetImage(i.TargetPlatform, i.baseImageName).GetLastNonEmptyStage()
i.baseImageReference = i.stageAsBaseImage.GetStageImage().Image.Name()
i.baseStageImage = i.stageAsBaseImage.GetStageImage()

case ImageFromRegistryAsBaseImage:
if i.IsDockerfileImage && i.dockerfileExpanderFactory != nil {
dependenciesArgs := stage.ResolveDependenciesArgs(i.DockerfileImageConfig.Dependencies, i.Conveyor)
dependenciesArgs := stage.ResolveDependenciesArgs(i.TargetPlatform, i.DockerfileImageConfig.Dependencies, i.Conveyor)
ref, err := i.dockerfileExpanderFactory.GetExpander(dockerfile.ExpandOptions{SkipUnsetEnv: false}).ProcessWordWithMap(i.baseImageReference, dependenciesArgs)
if err != nil {
return fmt.Errorf("unable to expand dockerfile base image reference %q: %w", i.baseImageReference, err)
Expand Down
1 change: 1 addition & 0 deletions pkg/build/image/stapel.go
View file
Expand Up @@ -66,6 +66,7 @@ func initStages(ctx context.Context, image *Image, metaConfig *config.Meta, stap
imageArtifact := stapelImageConfig.IsArtifact()

baseStageOptions := &stage.BaseStageOptions{
TargetPlatform: image.TargetPlatform,
ImageName: imageName,
ConfigMounts: imageBaseConfig.Mount,
ImageTmpDir: filepath.Join(opts.TmpDir, "image", imageBaseConfig.Name),
Expand Down

0 comments on commit 3e2add1

Please sign in to comment.