Skip to content
Permalink
Browse files
feat(staged-dockerfile): implement full chain of staged dockerfile bu…
…ilding only for single instruction (RUN)

* Small refactor of container_backend staged-dockerfile-builder: set context at configure-time, only pass options to the Build method.
* Only RUN instruction stage simple example implementation.
* No real Dockerfile parsing yet.

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
distorhead committed Oct 5, 2022
1 parent db8d337 commit 121ac0c2c3ad1f4074ed9111880b2789e5b3a61f
Show file tree
Hide file tree
Showing 34 changed files with 210 additions and 113 deletions.
@@ -404,7 +404,7 @@ func (phase *BuildPhase) onImageStage(ctx context.Context, img *image.Image, stg
return fmt.Errorf("unable to fetch dependencies for stage %s: %w", stg.LogDetailedName(), err)
}

if stg.Name() != "from" && stg.Name() != "dockerfile" {
if stg.HasPrevStage() {
if phase.StagesIterator.PrevNonEmptyStage == nil {
panic(fmt.Sprintf("expected PrevNonEmptyStage to be set for image %q stage %s", img.GetName(), stg.Name()))
}
@@ -554,7 +554,7 @@ func (phase *BuildPhase) fetchBaseImageForStage(ctx context.Context, img *image.
}
case stg.Name() == "dockerfile":
return nil
default:
case stg.HasPrevStage():
return phase.Conveyor.StorageManager.FetchStage(ctx, phase.Conveyor.ContainerBackend, phase.StagesIterator.PrevBuiltStage)
}

@@ -748,14 +748,7 @@ func (phase *BuildPhase) atomicBuildStageImage(ctx context.Context, img *image.I
}

if err := logboek.Context(ctx).Streams().DoErrorWithTag(fmt.Sprintf("%s/%s", img.LogName(), stg.Name()), img.LogTagStyle(), func() error {
switch {
case stg.Name() == "dockerfile":
return stageImage.Builder.DockerfileBuilder().Build(ctx)
case phase.Conveyor.UseLegacyStapelBuilder(phase.Conveyor.ContainerBackend):
return stageImage.Builder.LegacyStapelStageBuilder().Build(ctx, phase.ImageBuildOptions)
default:
return stageImage.Builder.StapelStageBuilder().Build(ctx, phase.ImageBuildOptions)
}
return stageImage.Builder.Build(ctx, phase.ImageBuildOptions)
}); err != nil {
return fmt.Errorf("failed to build image for stage %s with digest %s: %w", stg.Name(), stg.GetDigest(), err)
}
@@ -12,6 +12,7 @@ import (

"github.com/werf/logboek"
"github.com/werf/werf/pkg/build/stage"
"github.com/werf/werf/pkg/build/stage/dockerfile_instruction"
"github.com/werf/werf/pkg/config"
"github.com/werf/werf/pkg/dockerfile"
"github.com/werf/werf/pkg/path_matcher"
@@ -37,7 +38,7 @@ func MapDockerfileConfigToImagesSets(ctx context.Context, dockerfileImageConfig
return nil, fmt.Errorf("unable to parse dockerfile %s: %w", relDockerfilePath, err)
}

return mapDockerfileToImagesSets(ctx, d)
return mapDockerfileToImagesSets(ctx, d, opts)
}

img, err := mapLegacyDockerfileToImage(ctx, dockerfileImageConfig, opts)
@@ -52,7 +53,7 @@ func MapDockerfileConfigToImagesSets(ctx context.Context, dockerfileImageConfig
return ret, nil
}

func mapDockerfileToImagesSets(ctx context.Context, cfg *dockerfile.Dockerfile) (ImagesSets, error) {
func mapDockerfileToImagesSets(ctx context.Context, cfg *dockerfile.Dockerfile, opts CommonImageOptions) (ImagesSets, error) {
var ret ImagesSets

stagesSets, err := cfg.GroupStagesByIndependentSets(ctx)
@@ -68,6 +69,22 @@ func mapDockerfileToImagesSets(ctx context.Context, cfg *dockerfile.Dockerfile)
}
}

{
img := NewImage("test", ImageOptions{
IsDockerfileImage: true,
CommonImageOptions: opts,
})

img.stages = append(img.stages, dockerfile_instruction.NewRun(&dockerfile.InstructionRun{Command: []string{"ls", "/"}}, nil, false, &stage.BaseStageOptions{
ImageName: img.Name,
ImageTmpDir: img.TmpDir,
ContainerWerfDir: img.ContainerWerfDir,
ProjectName: opts.ProjectName,
}))

ret = append(ret, []*Image{img})
}

return ret, nil
}

@@ -162,7 +179,7 @@ func mapLegacyDockerfileToImage(ctx context.Context, dockerfileImageConfig *conf
dockerTargetIndex,
)

baseStageOptions := &stage.NewBaseStageOptions{
baseStageOptions := &stage.BaseStageOptions{
ImageName: dockerfileImageConfig.Name,
ProjectName: opts.ProjectName,
}
@@ -59,7 +59,7 @@ func initStages(ctx context.Context, image *Image, metaConfig *config.Meta, stap
imageName := imageBaseConfig.Name
imageArtifact := stapelImageConfig.IsArtifact()

baseStageOptions := &stage.NewBaseStageOptions{
baseStageOptions := &stage.BaseStageOptions{
ImageName: imageName,
ConfigMounts: imageBaseConfig.Mount,
ImageTmpDir: filepath.Join(opts.TmpDir, "image", imageBaseConfig.Name),
@@ -73,15 +73,15 @@ var AllStages = []StageName{
Dockerfile,
}

type NewBaseStageOptions struct {
type BaseStageOptions struct {
ImageName string
ConfigMounts []*config.Mount
ImageTmpDir string
ContainerWerfDir string
ProjectName string
}

func newBaseStage(name StageName, options *NewBaseStageOptions) *BaseStage {
func NewBaseStage(name StageName, options *BaseStageOptions) *BaseStage {
s := &BaseStage{}
s.name = name
s.imageName = options.ImageName
@@ -105,6 +105,14 @@ type BaseStage struct {
projectName string
}

func (s *BaseStage) HasPrevStage() bool {
return true
}

func (s *BaseStage) IsStapelStage() bool {
return true
}

func (s *BaseStage) LogDetailedName() string {
imageName := s.imageName
if imageName == "" {
@@ -8,7 +8,7 @@ import (
"github.com/werf/werf/pkg/container_backend"
)

func GenerateBeforeInstallStage(ctx context.Context, imageBaseConfig *config.StapelImageBase, baseStageOptions *NewBaseStageOptions) *BeforeInstallStage {
func GenerateBeforeInstallStage(ctx context.Context, imageBaseConfig *config.StapelImageBase, baseStageOptions *BaseStageOptions) *BeforeInstallStage {
b := getBuilder(imageBaseConfig, baseStageOptions)
if b != nil && !b.IsBeforeInstallEmpty(ctx) {
return newBeforeInstallStage(b, baseStageOptions)
@@ -17,7 +17,7 @@ func GenerateBeforeInstallStage(ctx context.Context, imageBaseConfig *config.Sta
return nil
}

func newBeforeInstallStage(builder builder.Builder, baseStageOptions *NewBaseStageOptions) *BeforeInstallStage {
func newBeforeInstallStage(builder builder.Builder, baseStageOptions *BaseStageOptions) *BeforeInstallStage {
s := &BeforeInstallStage{}
s.UserStage = newUserStage(builder, BeforeInstall, baseStageOptions)
return s
@@ -9,7 +9,7 @@ import (
"github.com/werf/werf/pkg/util"
)

func GenerateBeforeSetupStage(ctx context.Context, imageBaseConfig *config.StapelImageBase, gitPatchStageOptions *NewGitPatchStageOptions, baseStageOptions *NewBaseStageOptions) *BeforeSetupStage {
func GenerateBeforeSetupStage(ctx context.Context, imageBaseConfig *config.StapelImageBase, gitPatchStageOptions *NewGitPatchStageOptions, baseStageOptions *BaseStageOptions) *BeforeSetupStage {
b := getBuilder(imageBaseConfig, baseStageOptions)
if b != nil && !b.IsBeforeSetupEmpty(ctx) {
return newBeforeSetupStage(b, gitPatchStageOptions, baseStageOptions)
@@ -18,7 +18,7 @@ func GenerateBeforeSetupStage(ctx context.Context, imageBaseConfig *config.Stape
return nil
}

func newBeforeSetupStage(builder builder.Builder, gitPatchStageOptions *NewGitPatchStageOptions, baseStageOptions *NewBaseStageOptions) *BeforeSetupStage {
func newBeforeSetupStage(builder builder.Builder, gitPatchStageOptions *NewGitPatchStageOptions, baseStageOptions *BaseStageOptions) *BeforeSetupStage {
s := &BeforeSetupStage{}
s.UserWithGitPatchStage = newUserWithGitPatchStage(builder, BeforeSetup, gitPatchStageOptions, baseStageOptions)
return s
@@ -52,11 +52,11 @@ func getDependencies(imageBaseConfig *config.StapelImageBase, options *getImport
return dependencies
}

func newDependenciesStage(imports []*config.Import, dependencies []*config.Dependency, name StageName, baseStageOptions *NewBaseStageOptions) *DependenciesStage {
func newDependenciesStage(imports []*config.Import, dependencies []*config.Dependency, name StageName, baseStageOptions *BaseStageOptions) *DependenciesStage {
s := &DependenciesStage{}
s.imports = imports
s.dependencies = dependencies
s.BaseStage = newBaseStage(name, baseStageOptions)
s.BaseStage = NewBaseStage(name, baseStageOptions)
return s
}

@@ -2,7 +2,7 @@ package stage

import "github.com/werf/werf/pkg/config"

func GenerateDependenciesAfterInstallStage(imageBaseConfig *config.StapelImageBase, baseStageOptions *NewBaseStageOptions) *DependenciesAfterInstallStage {
func GenerateDependenciesAfterInstallStage(imageBaseConfig *config.StapelImageBase, baseStageOptions *BaseStageOptions) *DependenciesAfterInstallStage {
imports := getImports(imageBaseConfig, &getImportsOptions{After: Install})
dependencies := getDependencies(imageBaseConfig, &getImportsOptions{After: Install})
if len(imports)+len(dependencies) > 0 {
@@ -12,7 +12,7 @@ func GenerateDependenciesAfterInstallStage(imageBaseConfig *config.StapelImageBa
return nil
}

func newDependenciesAfterInstallStage(imports []*config.Import, dependencies []*config.Dependency, baseStageOptions *NewBaseStageOptions) *DependenciesAfterInstallStage {
func newDependenciesAfterInstallStage(imports []*config.Import, dependencies []*config.Dependency, baseStageOptions *BaseStageOptions) *DependenciesAfterInstallStage {
s := &DependenciesAfterInstallStage{}
s.DependenciesStage = newDependenciesStage(imports, dependencies, DependenciesAfterInstall, baseStageOptions)
return s
@@ -2,7 +2,7 @@ package stage

import "github.com/werf/werf/pkg/config"

func GenerateDependenciesAfterSetupStage(imageBaseConfig *config.StapelImageBase, baseStageOptions *NewBaseStageOptions) *DependenciesAfterSetupStage {
func GenerateDependenciesAfterSetupStage(imageBaseConfig *config.StapelImageBase, baseStageOptions *BaseStageOptions) *DependenciesAfterSetupStage {
imports := getImports(imageBaseConfig, &getImportsOptions{After: Setup})
dependencies := getDependencies(imageBaseConfig, &getImportsOptions{After: Setup})
if len(imports)+len(dependencies) > 0 {
@@ -12,7 +12,7 @@ func GenerateDependenciesAfterSetupStage(imageBaseConfig *config.StapelImageBase
return nil
}

func newDependenciesAfterSetupStage(imports []*config.Import, dependencies []*config.Dependency, baseStageOptions *NewBaseStageOptions) *DependenciesAfterSetupStage {
func newDependenciesAfterSetupStage(imports []*config.Import, dependencies []*config.Dependency, baseStageOptions *BaseStageOptions) *DependenciesAfterSetupStage {
s := &DependenciesAfterSetupStage{}
s.DependenciesStage = newDependenciesStage(imports, dependencies, DependenciesAfterSetup, baseStageOptions)
return s
@@ -2,7 +2,7 @@ package stage

import "github.com/werf/werf/pkg/config"

func GenerateDependenciesBeforeInstallStage(imageBaseConfig *config.StapelImageBase, baseStageOptions *NewBaseStageOptions) *DependenciesBeforeInstallStage {
func GenerateDependenciesBeforeInstallStage(imageBaseConfig *config.StapelImageBase, baseStageOptions *BaseStageOptions) *DependenciesBeforeInstallStage {
imports := getImports(imageBaseConfig, &getImportsOptions{Before: Install})
dependencies := getDependencies(imageBaseConfig, &getImportsOptions{Before: Install})
if len(imports)+len(dependencies) > 0 {
@@ -12,7 +12,7 @@ func GenerateDependenciesBeforeInstallStage(imageBaseConfig *config.StapelImageB
return nil
}

func newDependenciesBeforeInstallStage(imports []*config.Import, dependencies []*config.Dependency, baseStageOptions *NewBaseStageOptions) *DependenciesBeforeInstallStage {
func newDependenciesBeforeInstallStage(imports []*config.Import, dependencies []*config.Dependency, baseStageOptions *BaseStageOptions) *DependenciesBeforeInstallStage {
s := &DependenciesBeforeInstallStage{}
s.DependenciesStage = newDependenciesStage(imports, dependencies, DependenciesBeforeInstall, baseStageOptions)
return s
@@ -2,7 +2,7 @@ package stage

import "github.com/werf/werf/pkg/config"

func GenerateDependenciesBeforeSetupStage(imageBaseConfig *config.StapelImageBase, baseStageOptions *NewBaseStageOptions) *DependenciesBeforeSetupStage {
func GenerateDependenciesBeforeSetupStage(imageBaseConfig *config.StapelImageBase, baseStageOptions *BaseStageOptions) *DependenciesBeforeSetupStage {
imports := getImports(imageBaseConfig, &getImportsOptions{Before: Setup})
dependencies := getDependencies(imageBaseConfig, &getImportsOptions{Before: Setup})
if len(imports)+len(dependencies) > 0 {
@@ -12,7 +12,7 @@ func GenerateDependenciesBeforeSetupStage(imageBaseConfig *config.StapelImageBas
return nil
}

func newDependenciesBeforeSetupStage(imports []*config.Import, dependencies []*config.Dependency, baseStageOptions *NewBaseStageOptions) *DependenciesBeforeSetupStage {
func newDependenciesBeforeSetupStage(imports []*config.Import, dependencies []*config.Dependency, baseStageOptions *BaseStageOptions) *DependenciesBeforeSetupStage {
s := &DependenciesBeforeSetupStage{}
s.DependenciesStage = newDependenciesStage(imports, dependencies, DependenciesBeforeSetup, baseStageOptions)
return s
@@ -18,7 +18,7 @@ var _ = Describe("DependenciesStage", func() {
conveyor := NewConveyorStubForDependencies(NewGiterminismManagerStub(NewLocalGitRepoStub("9d8059842b6fde712c58315ca0ab4713d90761c0"), NewGiterminismInspectorStub()), data.Dependencies)
containerBackend := NewContainerBackendMock()

stage := newDependenciesStage(nil, GetConfigDependencies(data.Dependencies), "example-stage", &NewBaseStageOptions{
stage := newDependenciesStage(nil, GetConfigDependencies(data.Dependencies), "example-stage", &BaseStageOptions{
ImageName: "example-image",
ProjectName: "example-project",
})

This file was deleted.

@@ -0,0 +1,29 @@
package dockerfile_instruction

import (
"github.com/werf/werf/pkg/build/stage"
"github.com/werf/werf/pkg/config"
)

type Base struct {
*stage.BaseStage

dependencies []*config.Dependency
hasPrevStage bool
}

func NewBase(name stage.StageName, dependencies []*config.Dependency, hasPrevStage bool, opts *stage.BaseStageOptions) *Base {
return &Base{
BaseStage: stage.NewBaseStage(name, opts),
dependencies: dependencies,
hasPrevStage: hasPrevStage,
}
}

func (stage *Base) HasPrevStage() bool {
return stage.hasPrevStage
}

func (s *Base) IsStapelStage() bool {
return false
}
@@ -0,0 +1,5 @@
package dockerfile_instruction

type DockerfileInstruction interface {
Name() string
}
@@ -0,0 +1,32 @@
package dockerfile_instruction

import (
"context"

"github.com/werf/werf/pkg/build/stage"
"github.com/werf/werf/pkg/config"
"github.com/werf/werf/pkg/container_backend"
"github.com/werf/werf/pkg/dockerfile"
"github.com/werf/werf/pkg/util"
)

type Run struct {
*Base
instruction *dockerfile.InstructionRun
}

func NewRun(instruction *dockerfile.InstructionRun, dependencies []*config.Dependency, hasPrevStage bool, opts *stage.BaseStageOptions) *Run {
return &Run{
Base: NewBase(stage.StageName(instruction.Name()), dependencies, hasPrevStage, opts),
instruction: instruction,
}
}

func (stage *Run) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage) (string, error) {
return util.Sha256Hash(stage.instruction.Command...), nil
}

func (stage *Run) PrepareImage(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *stage.StageImage) error {
stageImage.Builder.DockerfileStageBuilder().AppendMainCommands(stage.instruction)
return nil
}

0 comments on commit 121ac0c

Please sign in to comment.