Skip to content

Commit 9ecb737

Browse files
distorheaddistorhead
committed
feat(staged-dockerfile): refactor build package conveyor images tree creation
* Introduce subpackage pkg/build/image which contains Image, ImageTree and ImagesSets primitives. * This package contains Image+Stage creation procedures which previously were in Conveyor itself. * Implement ImagesSets merging procedure to construct resulting images-sets to build stapel-images and all Dockerfiles stages concurrently using werf's conveyor parallelization. Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
1 parent cda37cb commit 9ecb737

File tree

15 files changed

+996
-724
lines changed

15 files changed

+996
-724
lines changed

‎pkg/build/base_phase.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package build
33
import (
44
"context"
55

6+
"github.com/werf/werf/pkg/build/image"
67
"github.com/werf/werf/pkg/build/stage"
78
)
89

@@ -18,18 +19,18 @@ func (phase *BasePhase) AfterImages(_ context.Context) error {
1819
return nil
1920
}
2021

21-
func (phase *BasePhase) BeforeImageStages(_ context.Context, _ *Image) error {
22+
func (phase *BasePhase) BeforeImageStages(_ context.Context, _ *image.Image) error {
2223
return nil
2324
}
2425

25-
func (phase *BasePhase) OnImageStage(_ context.Context, _ *Image, _ stage.Interface) error {
26+
func (phase *BasePhase) OnImageStage(_ context.Context, _ *image.Image, _ stage.Interface) error {
2627
return nil
2728
}
2829

29-
func (phase *BasePhase) AfterImageStages(ctx context.Context, img *Image) error {
30+
func (phase *BasePhase) AfterImageStages(ctx context.Context, img *image.Image) error {
3031
return nil
3132
}
3233

33-
func (phase *BasePhase) ImageProcessingShouldBeStopped(_ context.Context, _ *Image) bool {
34+
func (phase *BasePhase) ImageProcessingShouldBeStopped(_ context.Context, _ *image.Image) bool {
3435
return false
3536
}

‎pkg/build/build_phase.go‎

Lines changed: 34 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"github.com/werf/logboek"
1919
"github.com/werf/logboek/pkg/style"
2020
"github.com/werf/logboek/pkg/types"
21+
"github.com/werf/werf/pkg/build/image"
2122
"github.com/werf/werf/pkg/build/stage"
2223
"github.com/werf/werf/pkg/container_backend"
2324
"github.com/werf/werf/pkg/docker_registry"
@@ -169,8 +170,8 @@ func (phase *BuildPhase) AfterImages(ctx context.Context) error {
169170
}
170171

171172
func (phase *BuildPhase) createReport(ctx context.Context) error {
172-
for _, img := range phase.Conveyor.images {
173-
if img.isArtifact {
173+
for _, img := range phase.Conveyor.imagesTree.GetImages() {
174+
if img.IsArtifact {
174175
continue
175176
}
176177

@@ -213,19 +214,19 @@ func (phase *BuildPhase) createReport(ctx context.Context) error {
213214
return nil
214215
}
215216

216-
func (phase *BuildPhase) ImageProcessingShouldBeStopped(_ context.Context, _ *Image) bool {
217+
func (phase *BuildPhase) ImageProcessingShouldBeStopped(_ context.Context, _ *image.Image) bool {
217218
return false
218219
}
219220

220-
func (phase *BuildPhase) BeforeImageStages(_ context.Context, img *Image) error {
221+
func (phase *BuildPhase) BeforeImageStages(_ context.Context, img *image.Image) error {
221222
phase.StagesIterator = NewStagesIterator(phase.Conveyor)
222223

223-
img.SetupBaseImage(phase.Conveyor)
224+
img.SetupBaseImage()
224225

225226
return nil
226227
}
227228

228-
func (phase *BuildPhase) AfterImageStages(ctx context.Context, img *Image) error {
229+
func (phase *BuildPhase) AfterImageStages(ctx context.Context, img *image.Image) error {
229230
img.SetLastNonEmptyStage(phase.StagesIterator.PrevNonEmptyStage)
230231
img.SetContentDigest(phase.StagesIterator.PrevNonEmptyStage.GetContentDigest())
231232

@@ -239,7 +240,7 @@ func (phase *BuildPhase) AfterImageStages(ctx context.Context, img *Image) error
239240
}
240241
}
241242

242-
if img.isArtifact {
243+
if img.IsArtifact {
243244
return nil
244245
}
245246

@@ -272,10 +273,10 @@ func (phase *BuildPhase) AfterImageStages(ctx context.Context, img *Image) error
272273
return nil
273274
}
274275

275-
func (phase *BuildPhase) addManagedImage(ctx context.Context, img *Image) error {
276+
func (phase *BuildPhase) addManagedImage(ctx context.Context, img *image.Image) error {
276277
if phase.ShouldAddManagedImageRecord {
277278
stagesStorage := phase.Conveyor.StorageManager.GetStagesStorage()
278-
exist, err := stagesStorage.IsManagedImageExist(ctx, phase.Conveyor.projectName(), img.GetName(), storage.WithCache())
279+
exist, err := stagesStorage.IsManagedImageExist(ctx, phase.Conveyor.ProjectName(), img.GetName(), storage.WithCache())
279280
if err != nil {
280281
return fmt.Errorf("unable to check existence of managed image: %w", err)
281282
}
@@ -284,15 +285,15 @@ func (phase *BuildPhase) addManagedImage(ctx context.Context, img *Image) error
284285
return nil
285286
}
286287

287-
if err := stagesStorage.AddManagedImage(ctx, phase.Conveyor.projectName(), img.GetName()); err != nil {
288-
return fmt.Errorf("unable to add image %q to the managed images of project %q: %w", img.GetName(), phase.Conveyor.projectName(), err)
288+
if err := stagesStorage.AddManagedImage(ctx, phase.Conveyor.ProjectName(), img.GetName()); err != nil {
289+
return fmt.Errorf("unable to add image %q to the managed images of project %q: %w", img.GetName(), phase.Conveyor.ProjectName(), err)
289290
}
290291
}
291292

292293
return nil
293294
}
294295

295-
func (phase *BuildPhase) publishImageMetadata(ctx context.Context, img *Image) error {
296+
func (phase *BuildPhase) publishImageMetadata(ctx context.Context, img *image.Image) error {
296297
return logboek.Context(ctx).Info().LogProcess(fmt.Sprintf("Processing image %s git metadata", img.GetName())).
297298
DoError(func() error {
298299
var commits []string
@@ -311,13 +312,13 @@ func (phase *BuildPhase) publishImageMetadata(ctx context.Context, img *Image) e
311312

312313
for _, commit := range commits {
313314
stagesStorage := phase.Conveyor.StorageManager.GetStagesStorage()
314-
exist, err := stagesStorage.IsImageMetadataExist(ctx, phase.Conveyor.projectName(), img.GetName(), commit, img.GetStageID(), storage.WithCache())
315+
exist, err := stagesStorage.IsImageMetadataExist(ctx, phase.Conveyor.ProjectName(), img.GetName(), commit, img.GetStageID(), storage.WithCache())
315316
if err != nil {
316317
return fmt.Errorf("unable to get image %s metadata by commit %s and stage ID %s: %w", img.GetName(), commit, img.GetStageID(), err)
317318
}
318319

319320
if !exist {
320-
if err := stagesStorage.PutImageMetadata(ctx, phase.Conveyor.projectName(), img.GetName(), commit, img.GetStageID()); err != nil {
321+
if err := stagesStorage.PutImageMetadata(ctx, phase.Conveyor.ProjectName(), img.GetName(), commit, img.GetStageID()); err != nil {
321322
return fmt.Errorf("unable to put image %s metadata by commit %s and stage ID %s: %w", img.GetName(), commit, img.GetStageID(), err)
322323
}
323324
}
@@ -388,13 +389,13 @@ func (phase *BuildPhase) getPrevNonEmptyStageImageSize() int64 {
388389
return 0
389390
}
390391

391-
func (phase *BuildPhase) OnImageStage(ctx context.Context, img *Image, stg stage.Interface) error {
392-
return phase.StagesIterator.OnImageStage(ctx, img, stg, func(img *Image, stg stage.Interface, isEmpty bool) error {
392+
func (phase *BuildPhase) OnImageStage(ctx context.Context, img *image.Image, stg stage.Interface) error {
393+
return phase.StagesIterator.OnImageStage(ctx, img, stg, func(img *image.Image, stg stage.Interface, isEmpty bool) error {
393394
return phase.onImageStage(ctx, img, stg, isEmpty)
394395
})
395396
}
396397

397-
func (phase *BuildPhase) onImageStage(ctx context.Context, img *Image, stg stage.Interface, isEmpty bool) error {
398+
func (phase *BuildPhase) onImageStage(ctx context.Context, img *image.Image, stg stage.Interface, isEmpty bool) error {
398399
if isEmpty {
399400
return nil
400401
}
@@ -474,16 +475,16 @@ func (phase *BuildPhase) onImageStage(ctx context.Context, img *Image, stg stage
474475
return nil
475476
}
476477

477-
func (phase *BuildPhase) findAndFetchStageFromSecondaryStagesStorage(ctx context.Context, img *Image, stg stage.Interface) (bool, error) {
478+
func (phase *BuildPhase) findAndFetchStageFromSecondaryStagesStorage(ctx context.Context, img *image.Image, stg stage.Interface) (bool, error) {
478479
foundSuitableStage := false
479480

480481
storageManager := phase.Conveyor.StorageManager
481482
atomicCopySuitableStageFromSecondaryStagesStorage := func(secondaryStageDesc *imagePkg.StageDescription, secondaryStagesStorage storage.StagesStorage) error {
482483
// Lock the primary stages storage
483484
var stageUnlocked bool
484485
var unlockStage func()
485-
if lock, err := phase.Conveyor.StorageLockManager.LockStage(ctx, phase.Conveyor.projectName(), stg.GetDigest()); err != nil {
486-
return fmt.Errorf("unable to lock project %s digest %s: %w", phase.Conveyor.projectName(), stg.GetDigest(), err)
486+
if lock, err := phase.Conveyor.StorageLockManager.LockStage(ctx, phase.Conveyor.ProjectName(), stg.GetDigest()); err != nil {
487+
return fmt.Errorf("unable to lock project %s digest %s: %w", phase.Conveyor.ProjectName(), stg.GetDigest(), err)
487488
} else {
488489
unlockStage = func() {
489490
if stageUnlocked {
@@ -545,10 +546,10 @@ ScanSecondaryStagesStorageList:
545546
return foundSuitableStage, nil
546547
}
547548

548-
func (phase *BuildPhase) fetchBaseImageForStage(ctx context.Context, img *Image, stg stage.Interface) error {
549+
func (phase *BuildPhase) fetchBaseImageForStage(ctx context.Context, img *image.Image, stg stage.Interface) error {
549550
switch {
550551
case stg.Name() == "from":
551-
if err := img.FetchBaseImage(ctx, phase.Conveyor); err != nil {
552+
if err := img.FetchBaseImage(ctx); err != nil {
552553
return fmt.Errorf("unable to fetch base image %s for stage %s: %w", img.GetBaseImage().Image.Name(), stg.LogDetailedName(), err)
553554
}
554555
case stg.Name() == "dockerfile":
@@ -567,7 +568,7 @@ func extractLegacyStageImage(stageImage *stage.StageImage) *container_backend.Le
567568
return stageImage.Image.(*container_backend.LegacyStageImage)
568569
}
569570

570-
func (phase *BuildPhase) calculateStage(ctx context.Context, img *Image, stg stage.Interface) (bool, func(), error) {
571+
func (phase *BuildPhase) calculateStage(ctx context.Context, img *image.Image, stg stage.Interface) (bool, func(), error) {
571572
// FIXME(stapel-to-buildah): store StageImage-s everywhere in stage and build pkgs
572573
stageDependencies, err := stg.GetDependencies(ctx, phase.Conveyor, phase.Conveyor.ContainerBackend, phase.StagesIterator.GetPrevImage(img, stg), phase.StagesIterator.GetPrevBuiltImage(img, stg))
573574
if err != nil {
@@ -614,14 +615,14 @@ func (phase *BuildPhase) calculateStage(ctx context.Context, img *Image, stg sta
614615
return foundSuitableStage, phase.Conveyor.GetStageDigestMutex(stg.GetDigest()).Unlock, nil
615616
}
616617

617-
func (phase *BuildPhase) prepareStageInstructions(ctx context.Context, img *Image, stg stage.Interface) error {
618+
func (phase *BuildPhase) prepareStageInstructions(ctx context.Context, img *image.Image, stg stage.Interface) error {
618619
logboek.Context(ctx).Debug().LogF("-- BuildPhase.prepareStage %s %s\n", img.LogDetailedName(), stg.LogDetailedName())
619620

620621
stageImage := stg.GetStageImage()
621622

622623
serviceLabels := map[string]string{
623624
imagePkg.WerfDockerImageName: stageImage.Image.Name(),
624-
imagePkg.WerfLabel: phase.Conveyor.projectName(),
625+
imagePkg.WerfLabel: phase.Conveyor.ProjectName(),
625626
imagePkg.WerfVersionLabel: werf.Version,
626627
imagePkg.WerfCacheVersionLabel: imagePkg.BuildCacheVersion,
627628
imagePkg.WerfImageLabel: "false",
@@ -697,8 +698,8 @@ func (phase *BuildPhase) prepareStageInstructions(ctx context.Context, img *Imag
697698
return nil
698699
}
699700

700-
func (phase *BuildPhase) buildStage(ctx context.Context, img *Image, stg stage.Interface) error {
701-
if !img.isDockerfileImage && phase.Conveyor.UseLegacyStapelBuilder(phase.Conveyor.ContainerBackend) {
701+
func (phase *BuildPhase) buildStage(ctx context.Context, img *image.Image, stg stage.Interface) error {
702+
if !img.IsDockerfileImage && phase.Conveyor.UseLegacyStapelBuilder(phase.Conveyor.ContainerBackend) {
702703
_, err := stapel.GetOrCreateContainer(ctx)
703704
if err != nil {
704705
return fmt.Errorf("get or create stapel container failed: %w", err)
@@ -736,7 +737,7 @@ func (phase *BuildPhase) buildStage(ctx context.Context, img *Image, stg stage.I
736737
return nil
737738
}
738739

739-
func (phase *BuildPhase) atomicBuildStageImage(ctx context.Context, img *Image, stg stage.Interface) error {
740+
func (phase *BuildPhase) atomicBuildStageImage(ctx context.Context, img *image.Image, stg stage.Interface) error {
740741
stageImage := stg.GetStageImage()
741742

742743
if v := os.Getenv("WERF_TEST_ATOMIC_STAGE_BUILD__SLEEP_SECONDS_BEFORE_STAGE_BUILD"); v != "" {
@@ -768,8 +769,8 @@ func (phase *BuildPhase) atomicBuildStageImage(ctx context.Context, img *Image,
768769

769770
var stageUnlocked bool
770771
var unlockStage func()
771-
if lock, err := phase.Conveyor.StorageLockManager.LockStage(ctx, phase.Conveyor.projectName(), stg.GetDigest()); err != nil {
772-
return fmt.Errorf("unable to lock project %s digest %s: %w", phase.Conveyor.projectName(), stg.GetDigest(), err)
772+
if lock, err := phase.Conveyor.StorageLockManager.LockStage(ctx, phase.Conveyor.ProjectName(), stg.GetDigest()); err != nil {
773+
return fmt.Errorf("unable to lock project %s digest %s: %w", phase.Conveyor.ProjectName(), stg.GetDigest(), err)
773774
} else {
774775
unlockStage = func() {
775776
if stageUnlocked {
@@ -813,7 +814,7 @@ func (phase *BuildPhase) atomicBuildStageImage(ctx context.Context, img *Image,
813814
return fmt.Errorf("unable to store stage %s digest %s image %s into repo %s: %w", stg.LogDetailedName(), stg.GetDigest(), stageImage.Image.Name(), phase.Conveyor.StorageManager.GetStagesStorage().String(), err)
814815
}
815816

816-
if desc, err := phase.Conveyor.StorageManager.GetStagesStorage().GetStageDescription(ctx, phase.Conveyor.projectName(), stg.GetDigest(), uniqueID); err != nil {
817+
if desc, err := phase.Conveyor.StorageManager.GetStagesStorage().GetStageDescription(ctx, phase.Conveyor.ProjectName(), stg.GetDigest(), uniqueID); err != nil {
817818
return fmt.Errorf("unable to get stage %s digest %s image %s description from repo %s after stages has been stored into repo: %w", stg.LogDetailedName(), stg.GetDigest(), stageImage.Image.Name(), phase.Conveyor.StorageManager.GetStagesStorage().String(), err)
818819
} else {
819820
stageImage.Image.SetStageDescription(desc)
@@ -882,7 +883,7 @@ func calculateDigest(ctx context.Context, stageName, stageDependencies string, p
882883
}
883884

884885
// TODO: move these prints to the after-images hook, print summary over all images
885-
func (phase *BuildPhase) printShouldBeBuiltError(ctx context.Context, img *Image, stg stage.Interface) {
886+
func (phase *BuildPhase) printShouldBeBuiltError(ctx context.Context, img *image.Image, stg stage.Interface) {
886887
logboek.Context(ctx).Default().LogProcess("Built stages cache check").
887888
Options(func(options types.LogProcessOptionsInterface) {
888889
options.Style(style.Highlight())
@@ -900,7 +901,7 @@ func (phase *BuildPhase) printShouldBeBuiltError(ctx context.Context, img *Image
900901
logboek.Context(ctx).Warn().LogLn("There are some possible reasons:")
901902
logboek.Context(ctx).Warn().LogLn()
902903

903-
if img.isDockerfileImage {
904+
if img.IsDockerfileImage {
904905
logboek.Context(ctx).Warn().LogLn(reasonNumberFunc() + `Dockerfile has COPY or ADD instruction which uses non-permanent data that affects stage digest:
905906
- .git directory which should be excluded with .dockerignore file (https://docs.docker.com/engine/reference/builder/#dockerignore-file)
906907
- auto-generated file`)

0 commit comments

Comments
 (0)