Skip to content
Permalink
Browse files
feat(staged-dockerfile): refactored conveyor, debug container backend…
… staged dockerfile builder

* Implemented test instruction building using new staged-dockerfile building method of container backend.
* Refactored conveyor so that new builder now correctly coupled to the existing building mechanics.

refs #2215

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
distorhead committed Oct 10, 2022
1 parent 121ac0c commit 62b2181cf86e730de5996651c3e7d402dfb6ddc9
Show file tree
Hide file tree
Showing 23 changed files with 257 additions and 205 deletions.
@@ -451,7 +451,7 @@ func (phase *BuildPhase) onImageStage(ctx context.Context, img *image.Image, stg
}

// Will build a new stage
i := phase.Conveyor.GetOrCreateStageImage(extractLegacyStageImage(phase.StagesIterator.GetPrevImage(img, stg)), uuid.New().String())
i := phase.Conveyor.GetOrCreateStageImage(uuid.New().String(), phase.StagesIterator.GetPrevImage(img, stg), stg, img)
stg.SetStageImage(i)

if err := phase.fetchBaseImageForStage(ctx, img, stg); err != nil {
@@ -502,7 +502,7 @@ func (phase *BuildPhase) findAndFetchStageFromSecondaryStagesStorage(ctx context
if copiedStageDesc, err := storageManager.CopySuitableByDigestStage(ctx, secondaryStageDesc, secondaryStagesStorage, storageManager.GetStagesStorage(), phase.Conveyor.ContainerBackend); err != nil {
return fmt.Errorf("unable to copy suitable stage %s from %s to %s: %w", secondaryStageDesc.StageID.String(), secondaryStagesStorage.String(), storageManager.GetStagesStorage().String(), err)
} else {
i := phase.Conveyor.GetOrCreateStageImage(extractLegacyStageImage(phase.StagesIterator.GetPrevImage(img, stg)), copiedStageDesc.Info.Name)
i := phase.Conveyor.GetOrCreateStageImage(copiedStageDesc.Info.Name, phase.StagesIterator.GetPrevImage(img, stg), stg, img)
i.Image.SetStageDescription(copiedStageDesc)
stg.SetStageImage(i)

@@ -547,27 +547,16 @@ ScanSecondaryStagesStorageList:
}

func (phase *BuildPhase) fetchBaseImageForStage(ctx context.Context, img *image.Image, stg stage.Interface) error {
switch {
case stg.Name() == "from":
if stg.HasPrevStage() {
return phase.Conveyor.StorageManager.FetchStage(ctx, phase.Conveyor.ContainerBackend, phase.StagesIterator.PrevBuiltStage)
} else {
if err := img.FetchBaseImage(ctx); err != nil {
return fmt.Errorf("unable to fetch base image %s for stage %s: %w", img.GetBaseImage().Image.Name(), stg.LogDetailedName(), err)
return fmt.Errorf("unable to fetch base image %s for stage %s: %w", img.GetBaseStageImage().Image.Name(), stg.LogDetailedName(), err)
}
case stg.Name() == "dockerfile":
return nil
case stg.HasPrevStage():
return phase.Conveyor.StorageManager.FetchStage(ctx, phase.Conveyor.ContainerBackend, phase.StagesIterator.PrevBuiltStage)
}

return nil
}

func extractLegacyStageImage(stageImage *stage.StageImage) *container_backend.LegacyStageImage {
if stageImage == nil || stageImage.Image == nil {
return nil
}
return stageImage.Image.(*container_backend.LegacyStageImage)
}

func (phase *BuildPhase) calculateStage(ctx context.Context, img *image.Image, stg stage.Interface) (bool, func(), error) {
// FIXME(stapel-to-buildah): store StageImage-s everywhere in stage and build pkgs
stageDependencies, err := stg.GetDependencies(ctx, phase.Conveyor, phase.Conveyor.ContainerBackend, phase.StagesIterator.GetPrevImage(img, stg), phase.StagesIterator.GetPrevBuiltImage(img, stg))
@@ -597,7 +586,7 @@ func (phase *BuildPhase) calculateStage(ctx context.Context, img *image.Image, s
if stageDesc, err := storageManager.SelectSuitableStage(ctx, phase.Conveyor, stg, stages); err != nil {
return false, phase.Conveyor.GetStageDigestMutex(stg.GetDigest()).Unlock, err
} else if stageDesc != nil {
i := phase.Conveyor.GetOrCreateStageImage(extractLegacyStageImage(phase.StagesIterator.GetPrevImage(img, stg)), stageDesc.Info.Name)
i := phase.Conveyor.GetOrCreateStageImage(stageDesc.Info.Name, phase.StagesIterator.GetPrevImage(img, stg), stg, img)
i.Image.SetStageDescription(stageDesc)
stg.SetStageImage(i)
foundSuitableStage = true
@@ -630,19 +619,7 @@ func (phase *BuildPhase) prepareStageInstructions(ctx context.Context, img *imag
imagePkg.WerfStageContentDigestLabel: stg.GetContentDigest(),
}

switch stg.(type) {
case *stage.FullDockerfileStage:
var labels []string
for key, value := range serviceLabels {
labels = append(labels, fmt.Sprintf("%s=%v", key, value))
}
stageImage.Builder.DockerfileBuilder().AppendLabels(labels...)

phase.Conveyor.AppendOnTerminateFunc(func() error {
return stageImage.Builder.DockerfileBuilder().Cleanup(ctx)
})

default:
if stg.IsStapelStage() {
if phase.Conveyor.UseLegacyStapelBuilder(phase.Conveyor.ContainerBackend) {
stageImage.Builder.LegacyStapelStageBuilder().Container().ServiceCommitChangeOptions().AddLabel(serviceLabels)
} else {
@@ -688,6 +665,19 @@ func (phase *BuildPhase) prepareStageInstructions(ctx context.Context, img *imag
} else {
stageImage.Builder.StapelStageBuilder().AddEnvs(commitEnvs)
}
} else if _, ok := stg.(*stage.FullDockerfileStage); ok {
var labels []string
for key, value := range serviceLabels {
labels = append(labels, fmt.Sprintf("%s=%v", key, value))
}
stageImage.Builder.DockerfileBuilder().AppendLabels(labels...)

phase.Conveyor.AppendOnTerminateFunc(func() error {
return stageImage.Builder.DockerfileBuilder().Cleanup(ctx)
})
} else {
stageImage.Builder.DockerfileStageBuilder().AppendPostCommands(&container_backend.InstructionLabel{Labels: serviceLabels})
// staged dockerfile
}

err := stg.PrepareImage(ctx, phase.Conveyor, phase.Conveyor.ContainerBackend, phase.StagesIterator.GetPrevBuiltImage(img, stg), stageImage)
@@ -789,7 +779,7 @@ func (phase *BuildPhase) atomicBuildStageImage(ctx context.Context, img *image.I
stg.LogDetailedName(), stg.GetDigest(), stageDesc.Info.Name,
)

i := phase.Conveyor.GetOrCreateStageImage(extractLegacyStageImage(phase.StagesIterator.GetPrevImage(img, stg)), stageDesc.Info.Name)
i := phase.Conveyor.GetOrCreateStageImage(stageDesc.Info.Name, phase.StagesIterator.GetPrevImage(img, stg), stg, img)
i.Image.SetStageDescription(stageDesc)
stg.SetStageImage(i)

@@ -632,16 +632,34 @@ func (c *Conveyor) SetStageImage(stageImage *stage.StageImage) {
c.stageImages[stageImage.Image.Name()] = stageImage
}

func (c *Conveyor) GetOrCreateStageImage(fromImage *container_backend.LegacyStageImage, name string) *stage.StageImage {
if img := c.GetStageImage(name); img != nil {
return img
func extractLegacyStageImage(stageImage *stage.StageImage) *container_backend.LegacyStageImage {
if stageImage == nil || stageImage.Image == nil {
return nil
}
return stageImage.Image.(*container_backend.LegacyStageImage)
}

func (c *Conveyor) GetOrCreateStageImage(name string, prevStageImage *stage.StageImage, stg stage.Interface, img *image.Image) *stage.StageImage {
if stageImage := c.GetStageImage(name); stageImage != nil {
return stageImage
}

i := container_backend.NewLegacyStageImage(fromImage, name, c.ContainerBackend)
img := stage.NewStageImage(c.ContainerBackend, fromImage, i)
i := container_backend.NewLegacyStageImage(extractLegacyStageImage(prevStageImage), name, c.ContainerBackend)

var baseImage string
if stg != nil {
if stg.HasPrevStage() {
baseImage = prevStageImage.Image.Name()
} else if stg.IsStapelStage() && stg.Name() == "from" {
baseImage = prevStageImage.Image.Name()
} else {
baseImage = img.GetBaseImageReference()
}
}

c.SetStageImage(img)
return img
stageImage := stage.NewStageImage(c.ContainerBackend, baseImage, i)
c.SetStageImage(stageImage)
return stageImage
}

func (c *Conveyor) GetImage(name string) *image.Image {

This file was deleted.

@@ -4,15 +4,14 @@ import (
"sync"

"github.com/werf/werf/pkg/build/stage"
"github.com/werf/werf/pkg/container_backend"
"github.com/werf/werf/pkg/git_repo"
)

type Conveyor interface {
stage.Conveyor

GetImage(name string) *Image
GetOrCreateStageImage(fromImage *container_backend.LegacyStageImage, name string) *stage.StageImage
GetOrCreateStageImage(name string, prevStageImage *stage.StageImage, stg stage.Interface, img *Image) *stage.StageImage

IsBaseImagesRepoIdsCacheExist(key string) bool
GetBaseImagesRepoIdsCache(key string) string
@@ -14,6 +14,7 @@ import (
"github.com/werf/werf/pkg/build/stage"
"github.com/werf/werf/pkg/build/stage/dockerfile_instruction"
"github.com/werf/werf/pkg/config"
"github.com/werf/werf/pkg/container_backend"
"github.com/werf/werf/pkg/dockerfile"
"github.com/werf/werf/pkg/path_matcher"
"github.com/werf/werf/pkg/util"
@@ -70,12 +71,18 @@ func mapDockerfileToImagesSets(ctx context.Context, cfg *dockerfile.Dockerfile,
}

{
img := NewImage("test", ImageOptions{
// TODO parse FROM instruction properly, set correct BaseImageReference here

img, err := NewImage(ctx, "test", ImageFromRegistryAsBaseImage, ImageOptions{
IsDockerfileImage: true,
CommonImageOptions: opts,
BaseImageReference: "ubuntu:22.04",
})
if err != nil {
return nil, fmt.Errorf("unable to create image %q: %w", "test", err)
}

img.stages = append(img.stages, dockerfile_instruction.NewRun(&dockerfile.InstructionRun{Command: []string{"ls", "/"}}, nil, false, &stage.BaseStageOptions{
img.stages = append(img.stages, dockerfile_instruction.NewRun(&container_backend.InstructionRun{Command: []string{"ls", "/"}}, nil, false, &stage.BaseStageOptions{
ImageName: img.Name,
ImageTmpDir: img.TmpDir,
ContainerWerfDir: img.ContainerWerfDir,
@@ -89,10 +96,13 @@ func mapDockerfileToImagesSets(ctx context.Context, cfg *dockerfile.Dockerfile,
}

func mapLegacyDockerfileToImage(ctx context.Context, dockerfileImageConfig *config.ImageFromDockerfile, opts CommonImageOptions) (*Image, error) {
img := NewImage(dockerfileImageConfig.Name, ImageOptions{
img, err := NewImage(ctx, dockerfileImageConfig.Name, NoBaseImage, ImageOptions{
CommonImageOptions: opts,
IsDockerfileImage: true,
})
if err != nil {
return nil, fmt.Errorf("unable to create image %q: %w", dockerfileImageConfig.Name, err)
}

for _, contextAddFile := range dockerfileImageConfig.ContextAddFiles {
relContextAddFile := filepath.Join(dockerfileImageConfig.Context, contextAddFile)

0 comments on commit 62b2181

Please sign in to comment.