Skip to content
Permalink
Browse files
feat(stapel-to-buildah): implement 'from' stage 
* New stapel builder method introduced into ContainerRuntime, not fully implemented yet.
* pkg/build/stage/from implemented using new stapel builder.
* refactored all StageBuilder related implementations and moved to pkg/container_runtime/stage_builder.

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
@distorhead
distorhead committed Mar 29, 2022
1 parent 558a55e commit 7cc7d71ccc2ec03aa1f039610d7d7011d6b1e1f2
Showing with 573 additions and 377 deletions.
  1. +1 −1 cmd/werf/common/conveyor_options.go
  2. +10 −8 pkg/build/build_phase.go
  3. +14 −13 pkg/build/builder/ansible.go
  4. +5 −4 pkg/build/builder/builder.go
  5. +14 −13 pkg/build/builder/shell.go
  6. +0 −123 pkg/build/builder/stage_builder.go
  7. +5 −1 pkg/build/conveyor.go
  8. +33 −22 pkg/build/stage/base.go
  9. +1 −1 pkg/build/stage/before_install.go
  10. +1 −1 pkg/build/stage/before_setup.go
  11. +3 −0 pkg/build/stage/conveyor.go
  12. +6 −6 pkg/build/stage/data_test.go
  13. +7 −7 pkg/build/stage/dependencies.go
  14. +5 −5 pkg/build/stage/dependencies_test.go
  15. +5 −5 pkg/build/stage/docker_instructions.go
  16. +6 −6 pkg/build/stage/dockerfile.go
  17. +5 −5 pkg/build/stage/dockerfile_test.go
  18. +32 −17 pkg/build/stage/from.go
  19. +5 −5 pkg/build/stage/git.go
  20. +6 −7 pkg/build/stage/git_archive.go
  21. +5 −5 pkg/build/stage/git_mapping.go
  22. +1 −1 pkg/build/stage/git_mapping_test.go
  23. +7 −7 pkg/build/stage/git_patch.go
  24. +1 −1 pkg/build/stage/install.go
  25. +1 −1 pkg/build/stage/setup.go
  26. +6 −6 pkg/build/stage/stage_image.go
  27. +4 −0 pkg/build/stage/stubs_test.go
  28. +55 −1 pkg/container_runtime/buildah_runtime.go
  29. +6 −2 pkg/container_runtime/docker_server_runtime.go
  30. +0 −93 pkg/container_runtime/dockerfile_image_builder.go
  31. +4 −1 pkg/container_runtime/image_interface.go
  32. +19 −2 pkg/container_runtime/interface.go
  33. +3 −2 pkg/container_runtime/legacy_interface.go
  34. +6 −2 pkg/container_runtime/legacy_stage_image.go
  35. +10 −2 pkg/container_runtime/perf_check_container_runtime.go
  36. +110 −0 pkg/container_runtime/stage_builder/dockerfile_stage_builder.go
  37. +37 −0 pkg/container_runtime/stage_builder/legacy_stapel_stage_builder.go
  38. +52 −0 pkg/container_runtime/stage_builder/stage_builder.go
  39. +80 −0 pkg/container_runtime/stage_builder/stapel_stage_builder.go
  40. +1 −1 pkg/container_runtime/utils.go
  41. +1 −0 pkg/storage/repo_stages_storage.go
2 cmd/werf/common/conveyor_options.go
View file
@@ -66,7 +66,7 @@ func GetBuildOptions(commonCmdData *CmdData, giterminismManager giterminism_mana
buildOptions = build.BuildOptions{
SkipImageMetadataPublication: *commonCmdData.Dev,
CustomTagFuncList: customTagFuncList,
ImageBuildOptions: container_runtime.LegacyBuildOptions{
ImageBuildOptions: container_runtime.BuildOptions{
IntrospectAfterError: *commonCmdData.IntrospectAfterError,
IntrospectBeforeError: *commonCmdData.IntrospectBeforeError,
},
18 pkg/build/build_phase.go
View file
@@ -36,7 +36,7 @@ type BuildPhaseOptions struct {
}

type BuildOptions struct {
ImageBuildOptions container_runtime.LegacyBuildOptions
ImageBuildOptions container_runtime.BuildOptions
IntrospectOptions

ReportPath string
@@ -628,10 +628,10 @@ func (phase *BuildPhase) prepareStageInstructions(ctx context.Context, img *Imag
for key, value := range serviceLabels {
labels = append(labels, fmt.Sprintf("%s=%v", key, value))
}
stageImage.StageBuilderAccessor.NativeDockerfileStageBuilder().AppendLabels(labels...)
stageImage.Builder.DockerfileStageBuilder().AppendLabels(labels...)

phase.Conveyor.AppendOnTerminateFunc(func() error {
return stageImage.StageBuilderAccessor.NativeDockerfileStageBuilder().Cleanup(ctx)
return stageImage.Builder.DockerfileStageBuilder().Cleanup(ctx)
})

default:
@@ -675,7 +675,7 @@ func (phase *BuildPhase) prepareStageInstructions(ctx context.Context, img *Imag
}

func (phase *BuildPhase) buildStage(ctx context.Context, img *Image, stg stage.Interface) error {
if !img.isDockerfileImage {
if !img.isDockerfileImage && phase.Conveyor.UseLegacyStapelBuilder(phase.Conveyor.ContainerRuntime) {
_, err := stapel.GetOrCreateContainer(ctx)
if err != nil {
return fmt.Errorf("get or create stapel container failed: %s", err)
@@ -724,11 +724,13 @@ func (phase *BuildPhase) atomicBuildStageImage(ctx context.Context, img *Image,
}

if err := logboek.Context(ctx).Streams().DoErrorWithTag(fmt.Sprintf("%s/%s", img.LogName(), stg.Name()), img.LogTagStyle(), func() error {
switch stg.Name() {
case "dockerfile":
return stageImage.StageBuilderAccessor.NativeDockerfileStageBuilder().Build(ctx)
switch {
case stg.Name() == "dockerfile":
return stageImage.Builder.DockerfileStageBuilder().Build(ctx)
case phase.Conveyor.UseLegacyStapelBuilder(phase.Conveyor.ContainerRuntime):
return stageImage.Builder.LegacyStapelStageBuilder().Build(ctx, phase.ImageBuildOptions)
default:
return stageImage.StageBuilderAccessor.LegacyStapelStageBuilder().Build(ctx, phase.ImageBuildOptions)
return stageImage.Builder.StapelStageBuilder().Build(ctx, phase.ImageBuildOptions)
}
}); err != nil {
return fmt.Errorf("failed to build image for stage %s with digest %s: %s", stg.Name(), stg.GetDigest(), err)
27 pkg/build/builder/ansible.go
View file
@@ -17,6 +17,7 @@ import (
"github.com/werf/logboek"
"github.com/werf/werf/pkg/config"
"github.com/werf/werf/pkg/container_runtime"
"github.com/werf/werf/pkg/container_runtime/stage_builder"
"github.com/werf/werf/pkg/stapel"
"github.com/werf/werf/pkg/util"
)
@@ -44,20 +45,20 @@ func (b *Ansible) IsBeforeSetupEmpty(ctx context.Context) bool {
}
func (b *Ansible) IsSetupEmpty(ctx context.Context) bool { return b.isEmptyStage(ctx, "Setup") }

func (b *Ansible) BeforeInstall(ctx context.Context, cr container_runtime.ContainerRuntime, stageBuilder StageBuilderAccessorInterface) error {
return b.stage(ctx, cr, stageBuilder, "BeforeInstall")
func (b *Ansible) BeforeInstall(ctx context.Context, cr container_runtime.ContainerRuntime, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool) error {
return b.stage(ctx, cr, stageBuilder, useLegacyStapelBuilder, "BeforeInstall")
}

func (b *Ansible) Install(ctx context.Context, cr container_runtime.ContainerRuntime, stageBuilder StageBuilderAccessorInterface) error {
return b.stage(ctx, cr, stageBuilder, "Install")
func (b *Ansible) Install(ctx context.Context, cr container_runtime.ContainerRuntime, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool) error {
return b.stage(ctx, cr, stageBuilder, useLegacyStapelBuilder, "Install")
}

func (b *Ansible) BeforeSetup(ctx context.Context, cr container_runtime.ContainerRuntime, stageBuilder StageBuilderAccessorInterface) error {
return b.stage(ctx, cr, stageBuilder, "BeforeSetup")
func (b *Ansible) BeforeSetup(ctx context.Context, cr container_runtime.ContainerRuntime, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool) error {
return b.stage(ctx, cr, stageBuilder, useLegacyStapelBuilder, "BeforeSetup")
}

func (b *Ansible) Setup(ctx context.Context, cr container_runtime.ContainerRuntime, stageBuilder StageBuilderAccessorInterface) error {
return b.stage(ctx, cr, stageBuilder, "Setup")
func (b *Ansible) Setup(ctx context.Context, cr container_runtime.ContainerRuntime, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool) error {
return b.stage(ctx, cr, stageBuilder, useLegacyStapelBuilder, "Setup")
}

func (b *Ansible) BeforeInstallChecksum(ctx context.Context) string {
@@ -73,11 +74,8 @@ func (b *Ansible) isEmptyStage(ctx context.Context, userStageName string) bool {
return b.stageChecksum(ctx, userStageName) == ""
}

func (b *Ansible) stage(ctx context.Context, cr container_runtime.ContainerRuntime, stageBuilder StageBuilderAccessorInterface, userStageName string) error {
if cr.HasContainerRootMountSupport() {
// TODO(stapel-to-buildah)
panic("not implemented")
} else {
func (b *Ansible) stage(ctx context.Context, cr container_runtime.ContainerRuntime, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool, userStageName string) error {
if useLegacyStapelBuilder {
container := stageBuilder.LegacyStapelStageBuilder().BuilderContainer()

if len(b.stageTasks(userStageName)) == 0 {
@@ -136,6 +134,9 @@ func (b *Ansible) stage(ctx context.Context, cr container_runtime.ContainerRunti
container.AddServiceRunCommands(command)

return nil
} else {
// TODO(stapel-to-buildah)
panic("not implemented")
}
}

9 pkg/build/builder/builder.go
View file
@@ -5,17 +5,18 @@ import (
"os"

"github.com/werf/werf/pkg/container_runtime"
"github.com/werf/werf/pkg/container_runtime/stage_builder"
)

type Builder interface {
IsBeforeInstallEmpty(ctx context.Context) bool
IsInstallEmpty(ctx context.Context) bool
IsBeforeSetupEmpty(ctx context.Context) bool
IsSetupEmpty(ctx context.Context) bool
BeforeInstall(ctx context.Context, cr container_runtime.ContainerRuntime, stageBuilder StageBuilderAccessorInterface) error
Install(ctx context.Context, cr container_runtime.ContainerRuntime, stageBuilder StageBuilderAccessorInterface) error
BeforeSetup(ctx context.Context, cr container_runtime.ContainerRuntime, stageBuilder StageBuilderAccessorInterface) error
Setup(ctx context.Context, cr container_runtime.ContainerRuntime, stageBuilder StageBuilderAccessorInterface) error
BeforeInstall(ctx context.Context, cr container_runtime.ContainerRuntime, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool) error
Install(ctx context.Context, cr container_runtime.ContainerRuntime, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool) error
BeforeSetup(ctx context.Context, cr container_runtime.ContainerRuntime, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool) error
Setup(ctx context.Context, cr container_runtime.ContainerRuntime, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool) error
BeforeInstallChecksum(ctx context.Context) string
InstallChecksum(ctx context.Context) string
BeforeSetupChecksum(ctx context.Context) string
27 pkg/build/builder/shell.go
View file
@@ -12,6 +12,7 @@ import (
"github.com/werf/logboek"
"github.com/werf/werf/pkg/config"
"github.com/werf/werf/pkg/container_runtime"
"github.com/werf/werf/pkg/container_runtime/stage_builder"
"github.com/werf/werf/pkg/stapel"
"github.com/werf/werf/pkg/util"
)
@@ -36,20 +37,20 @@ func (b *Shell) IsBeforeSetupEmpty(ctx context.Context) bool {
}
func (b *Shell) IsSetupEmpty(ctx context.Context) bool { return b.isEmptyStage(ctx, "Setup") }

func (b *Shell) BeforeInstall(_ context.Context, cr container_runtime.ContainerRuntime, stageBuilder StageBuilderAccessorInterface) error {
return b.stage(cr, stageBuilder, "BeforeInstall")
func (b *Shell) BeforeInstall(_ context.Context, cr container_runtime.ContainerRuntime, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool) error {
return b.stage(cr, stageBuilder, useLegacyStapelBuilder, "BeforeInstall")
}

func (b *Shell) Install(_ context.Context, cr container_runtime.ContainerRuntime, stageBuilder StageBuilderAccessorInterface) error {
return b.stage(cr, stageBuilder, "Install")
func (b *Shell) Install(_ context.Context, cr container_runtime.ContainerRuntime, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool) error {
return b.stage(cr, stageBuilder, useLegacyStapelBuilder, "Install")
}

func (b *Shell) BeforeSetup(_ context.Context, cr container_runtime.ContainerRuntime, stageBuilder StageBuilderAccessorInterface) error {
return b.stage(cr, stageBuilder, "BeforeSetup")
func (b *Shell) BeforeSetup(_ context.Context, cr container_runtime.ContainerRuntime, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool) error {
return b.stage(cr, stageBuilder, useLegacyStapelBuilder, "BeforeSetup")
}

func (b *Shell) Setup(_ context.Context, cr container_runtime.ContainerRuntime, stageBuilder StageBuilderAccessorInterface) error {
return b.stage(cr, stageBuilder, "Setup")
func (b *Shell) Setup(_ context.Context, cr container_runtime.ContainerRuntime, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool) error {
return b.stage(cr, stageBuilder, useLegacyStapelBuilder, "Setup")
}

func (b *Shell) BeforeInstallChecksum(ctx context.Context) string {
@@ -65,11 +66,8 @@ func (b *Shell) isEmptyStage(ctx context.Context, userStageName string) bool {
return b.stageChecksum(ctx, userStageName) == ""
}

func (b *Shell) stage(cr container_runtime.ContainerRuntime, stageBuilder StageBuilderAccessorInterface, userStageName string) error {
if cr.HasContainerRootMountSupport() {
// TODO(stapel-to-buildah)
panic("not implemented")
} else {
func (b *Shell) stage(cr container_runtime.ContainerRuntime, stageBuilder stage_builder.StageBuilderInterface, useLegacyStapelBuilder bool, userStageName string) error {
if useLegacyStapelBuilder {
container := stageBuilder.LegacyStapelStageBuilder().BuilderContainer()

stageHostTmpDir, err := b.stageHostTmpDir(userStageName)
@@ -91,6 +89,9 @@ func (b *Shell) stage(cr container_runtime.ContainerRuntime, stageBuilder StageB
container.AddServiceRunCommands(containerTmpScriptFilePath)

return nil
} else {
// TODO(stapel-to-buildah)
panic("not implemented")
}
}

123 pkg/build/builder/stage_builder.go
View file

This file was deleted.

6 pkg/build/conveyor.go
View file
@@ -125,6 +125,10 @@ func (c *Conveyor) getServiceRWMutex(service string) *sync.RWMutex {
return rwMutex
}

func (c *Conveyor) UseLegacyStapelBuilder(cr container_runtime.ContainerRuntime) bool {
return !cr.HasStapelBuildSupport()
}

func (c *Conveyor) IsBaseImagesRepoIdsCacheExist(key string) bool {
c.getServiceRWMutex("BaseImagesRepoIdsCache").RLock()
defer c.getServiceRWMutex("BaseImagesRepoIdsCache").RUnlock()
@@ -688,7 +692,7 @@ func (c *Conveyor) GetOrCreateStageImage(fromImage *container_runtime.LegacyStag
}

i := container_runtime.NewLegacyStageImage(fromImage, name, c.ContainerRuntime)
img := stage.NewStageImage(c.ContainerRuntime, i)
img := stage.NewStageImage(c.ContainerRuntime, fromImage, i)

c.SetStageImage(img)
return img

0 comments on commit 7cc7d71

Please sign in to comment.