Skip to content

Commit a13a7b0

Browse files
alexey-igrychevalexey-igrychev
committed
feat(build): speeding up with runtime caching for stages
Signed-off-by: Alexey Igrychev <alexey.igrychev@flant.com>
1 parent cbb31b2 commit a13a7b0

File tree

5 files changed

+56
-75
lines changed

5 files changed

+56
-75
lines changed

‎pkg/build/build_phase.go‎

Lines changed: 25 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -412,7 +412,6 @@ func (phase *BuildPhase) onImageStage(ctx context.Context, img *Image, stg stage
412412
return err
413413
}
414414

415-
// Stage is cached in the stages storage
416415
if foundSuitableStage {
417416
logboek.Context(ctx).Default().LogFHighlight("Use cache image for %s\n", stg.LogDetailedName())
418417
container_runtime.LogImageInfo(ctx, stg.GetImage(), phase.getPrevNonEmptyStageImageSize())
@@ -467,6 +466,7 @@ func (phase *BuildPhase) onImageStage(ctx context.Context, img *Image, stg stage
467466
func (phase *BuildPhase) findAndFetchStageFromSecondaryStagesStorage(ctx context.Context, img *Image, stg stage.Interface) (bool, error) {
468467
foundSuitableStage := false
469468

469+
storageManager := phase.Conveyor.StorageManager
470470
atomicCopySuitableStageFromSecondaryStagesStorage := func(secondaryStageDesc *imagePkg.StageDescription, secondaryStagesStorage storage.StagesStorage) error {
471471
// Lock the primary stages storage
472472
var stageUnlocked bool
@@ -484,66 +484,42 @@ func (phase *BuildPhase) findAndFetchStageFromSecondaryStagesStorage(ctx context
484484
defer unlockStage()
485485
}
486486

487-
// Query the primary stages storage for suitable stage again.
488-
// Suitable stage can be found this time and should be used in this case
489-
if stages, err := phase.Conveyor.StorageManager.GetStagesByDigest(ctx, stg.LogDetailedName(), stg.GetDigest()); err != nil {
490-
return err
491-
} else {
492-
if stageDesc, err := phase.Conveyor.StorageManager.SelectSuitableStage(ctx, phase.Conveyor, stg, stages); err != nil {
493-
return err
494-
} else if stageDesc != nil {
495-
i := phase.Conveyor.GetOrCreateStageImage(castToStageImage(phase.StagesIterator.GetPrevImage(img, stg)), stageDesc.Info.Name)
496-
i.SetStageDescription(stageDesc)
487+
err := logboek.Context(ctx).Default().LogProcess("Copy suitable stage from secondary %s", secondaryStagesStorage.String()).DoError(func() error {
488+
// Copy suitable stage from a secondary stages storage to the primary stages storage
489+
// while primary stages storage lock for this digest is held
490+
if copiedStageDesc, err := storageManager.CopySuitableByDigestStage(ctx, secondaryStageDesc, secondaryStagesStorage, storageManager.GetStagesStorage(), phase.Conveyor.ContainerRuntime); err != nil {
491+
return fmt.Errorf("unable to copy suitable stage %s from %s to %s: %s", secondaryStageDesc.StageID.String(), secondaryStagesStorage.String(), storageManager.GetStagesStorage().String(), err)
492+
} else {
493+
i := phase.Conveyor.GetOrCreateStageImage(castToStageImage(phase.StagesIterator.GetPrevImage(img, stg)), copiedStageDesc.Info.Name)
494+
i.SetStageDescription(copiedStageDesc)
497495
stg.SetImage(i)
498496

499497
logboek.Context(ctx).Default().LogFHighlight("Use cache image for %s\n", stg.LogDetailedName())
500498
container_runtime.LogImageInfo(ctx, stg.GetImage(), phase.getPrevNonEmptyStageImageSize())
501499

502500
return nil
503501
}
502+
})
503+
if err != nil {
504+
return err
505+
}
504506

505-
err = logboek.Context(ctx).Default().LogProcess("Copy suitable stage from secondary %s", secondaryStagesStorage.String()).DoError(func() error {
506-
// Copy suitable stage from a secondary stages storage to the primary stages storage
507-
// while primary stages storage lock for this digest is held
508-
if copiedStageDesc, err := phase.Conveyor.StorageManager.CopySuitableByDigestStage(ctx, secondaryStageDesc, secondaryStagesStorage, phase.Conveyor.StorageManager.GetStagesStorage(), phase.Conveyor.ContainerRuntime); err != nil {
509-
return fmt.Errorf("unable to copy suitable stage %s from %s to %s: %s", secondaryStageDesc.StageID.String(), secondaryStagesStorage.String(), phase.Conveyor.StorageManager.GetStagesStorage().String(), err)
510-
} else {
511-
i := phase.Conveyor.GetOrCreateStageImage(castToStageImage(phase.StagesIterator.GetPrevImage(img, stg)), copiedStageDesc.Info.Name)
512-
i.SetStageDescription(copiedStageDesc)
513-
stg.SetImage(i)
514-
515-
var stageIDs []imagePkg.StageID
516-
for _, stageDesc := range stages {
517-
stageIDs = append(stageIDs, *stageDesc.StageID)
518-
}
519-
stageIDs = append(stageIDs, *copiedStageDesc.StageID)
520-
521-
logboek.Context(ctx).Default().LogFHighlight("Use cache image for %s\n", stg.LogDetailedName())
522-
container_runtime.LogImageInfo(ctx, stg.GetImage(), phase.getPrevNonEmptyStageImageSize())
523-
524-
return nil
525-
}
526-
})
527-
if err != nil {
528-
return err
529-
}
530-
531-
unlockStage()
532-
533-
if err := phase.Conveyor.StorageManager.CopyStageIntoCacheStorages(ctx, stg, phase.Conveyor.ContainerRuntime); err != nil {
534-
return fmt.Errorf("unable to copy stage %s into cache storages: %s", stg.GetImage().GetStageDescription().StageID.String(), err)
535-
}
507+
unlockStage()
536508

537-
return nil
509+
if err := storageManager.CopyStageIntoCacheStorages(ctx, stg, phase.Conveyor.ContainerRuntime); err != nil {
510+
return fmt.Errorf("unable to copy stage %s into cache storages: %s", stg.GetImage().GetStageDescription().StageID.String(), err)
538511
}
512+
513+
return nil
539514
}
540515

541516
ScanSecondaryStagesStorageList:
542-
for _, secondaryStagesStorage := range phase.Conveyor.StorageManager.GetSecondaryStagesStorageList() {
543-
if secondaryStages, err := phase.Conveyor.StorageManager.GetStagesByDigestFromStagesStorage(ctx, stg.LogDetailedName(), stg.GetDigest(), secondaryStagesStorage); err != nil {
517+
for _, secondaryStagesStorage := range storageManager.GetSecondaryStagesStorageList() {
518+
secondaryStages, err := storageManager.GetStagesByDigestFromStagesStorageWithCache(ctx, stg.LogDetailedName(), stg.GetDigest(), secondaryStagesStorage)
519+
if err != nil {
544520
return false, err
545521
} else {
546-
if secondaryStageDesc, err := phase.Conveyor.StorageManager.SelectSuitableStage(ctx, phase.Conveyor, stg, secondaryStages); err != nil {
522+
if secondaryStageDesc, err := storageManager.SelectSuitableStage(ctx, phase.Conveyor, stg, secondaryStages); err != nil {
547523
return false, err
548524
} else if secondaryStageDesc != nil {
549525
if err := atomicCopySuitableStageFromSecondaryStagesStorage(secondaryStageDesc, secondaryStagesStorage); err != nil {
@@ -601,10 +577,11 @@ func (phase *BuildPhase) calculateStage(ctx context.Context, img *Image, stg sta
601577
Do(phase.Conveyor.GetStageDigestMutex(stg.GetDigest()).Lock)
602578

603579
foundSuitableStage := false
604-
if stages, err := phase.Conveyor.StorageManager.GetStagesByDigest(ctx, stg.LogDetailedName(), stageDigest); err != nil {
580+
storageManager := phase.Conveyor.StorageManager
581+
if stages, err := storageManager.GetStagesByDigestWithCache(ctx, stg.LogDetailedName(), stageDigest); err != nil {
605582
return false, phase.Conveyor.GetStageDigestMutex(stg.GetDigest()).Unlock, err
606583
} else {
607-
if stageDesc, err := phase.Conveyor.StorageManager.SelectSuitableStage(ctx, phase.Conveyor, stg, stages); err != nil {
584+
if stageDesc, err := storageManager.SelectSuitableStage(ctx, phase.Conveyor, stg, stages); err != nil {
608585
return false, phase.Conveyor.GetStageDigestMutex(stg.GetDigest()).Unlock, err
609586
} else if stageDesc != nil {
610587
i := phase.Conveyor.GetOrCreateStageImage(castToStageImage(phase.StagesIterator.GetPrevImage(img, stg)), stageDesc.Info.Name)
@@ -811,12 +788,6 @@ func (phase *BuildPhase) atomicBuildStageImage(ctx context.Context, img *Image,
811788
return err
812789
}
813790

814-
var stageIDs []imagePkg.StageID
815-
for _, stageDesc := range stages {
816-
stageIDs = append(stageIDs, *stageDesc.StageID)
817-
}
818-
stageIDs = append(stageIDs, *stageImage.GetStageDescription().StageID)
819-
820791
unlockStage()
821792

822793
if err := phase.Conveyor.StorageManager.CopyStageIntoCacheStorages(ctx, stg, phase.Conveyor.ContainerRuntime); err != nil {

‎pkg/storage/docker_server_stages_storage.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ func (storage *DockerServerStagesStorage) ConstructStageImageName(projectName, d
5656
return fmt.Sprintf(LocalStage_ImageFormat, projectName, digest, uniqueID)
5757
}
5858

59-
func (storage *DockerServerStagesStorage) GetStagesIDs(ctx context.Context, projectName string) ([]image.StageID, error) {
59+
func (storage *DockerServerStagesStorage) GetStagesIDs(ctx context.Context, projectName string, _ ...Option) ([]image.StageID, error) {
6060
filterSet := localStagesStorageFilterSetBase(projectName)
6161
images, err := docker.Images(ctx, types.ImageListOptions{Filters: filterSet})
6262
if err != nil {
@@ -163,7 +163,7 @@ func (storage *DockerServerStagesStorage) GetManagedImages(_ context.Context, _
163163
return []string{}, nil
164164
}
165165

166-
func (storage *DockerServerStagesStorage) GetStagesIDsByDigest(ctx context.Context, projectName, digest string) ([]image.StageID, error) {
166+
func (storage *DockerServerStagesStorage) GetStagesIDsByDigest(ctx context.Context, projectName, digest string, _ ...Option) ([]image.StageID, error) {
167167
filterSet := filters.NewArgs()
168168
filterSet.Add("reference", fmt.Sprintf(LocalStage_ImageRepoFormat, projectName))
169169
// NOTE digest already depends on build-cache-version

‎pkg/storage/manager/storage_manager.go‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,9 @@ type StorageManagerInterface interface {
6161

6262
LockStageImage(ctx context.Context, imageName string) error
6363
GetStagesByDigest(ctx context.Context, stageName, stageDigest string) ([]*image.StageDescription, error)
64+
GetStagesByDigestWithCache(ctx context.Context, stageName, stageDigest string) ([]*image.StageDescription, error)
6465
GetStagesByDigestFromStagesStorage(ctx context.Context, stageName, stageDigest string, stagesStorage storage.StagesStorage) ([]*image.StageDescription, error)
66+
GetStagesByDigestFromStagesStorageWithCache(ctx context.Context, stageName, stageDigest string, stagesStorage storage.StagesStorage) ([]*image.StageDescription, error)
6567
GetStageDescriptionList(ctx context.Context) ([]*image.StageDescription, error)
6668
GetStageDescriptionListWithCache(ctx context.Context) ([]*image.StageDescription, error)
6769
GetFinalStageDescriptionList(ctx context.Context) ([]*image.StageDescription, error)
@@ -694,27 +696,33 @@ func (m *StorageManager) SelectSuitableStage(ctx context.Context, c stage.Convey
694696
return stageDesc, nil
695697
}
696698

699+
func (m *StorageManager) GetStagesByDigestWithCache(ctx context.Context, stageName, stageDigest string) ([]*image.StageDescription, error) {
700+
return m.GetStagesByDigestFromStagesStorageWithCache(ctx, stageName, stageDigest, m.StagesStorage)
701+
}
702+
697703
func (m *StorageManager) GetStagesByDigest(ctx context.Context, stageName, stageDigest string) ([]*image.StageDescription, error) {
698-
stageIDs, err := m.getStagesIDsByDigestFromStagesStorage(ctx, stageName, stageDigest, m.StagesStorage)
699-
if err != nil {
700-
return nil, fmt.Errorf("unable to get stages ids from %s by digest %s for stage %s: %s", m.StagesStorage.String(), stageDigest, stageName, err)
701-
}
704+
return m.GetStagesByDigestFromStagesStorage(ctx, stageName, stageDigest, m.StagesStorage)
705+
}
702706

703-
validStages, err := m.getStagesDescriptions(ctx, stageIDs, m.StagesStorage, m.CacheStagesStorageList)
707+
func (m *StorageManager) GetStagesByDigestFromStagesStorageWithCache(ctx context.Context, stageName, stageDigest string, stagesStorage storage.StagesStorage) ([]*image.StageDescription, error) {
708+
cachedStageDescriptionList, err := m.getStagesByDigestFromStagesStorage(ctx, stageName, stageDigest, stagesStorage, storage.WithCache())
704709
if err != nil {
705-
return nil, fmt.Errorf("unable to get stage descriptions by ids from %s: %s", m.StagesStorage.String(), err)
710+
return nil, err
706711
}
707712

708-
var validStageIDs []image.StageID
709-
for _, s := range validStages {
710-
validStageIDs = append(validStageIDs, *s.StageID)
713+
if len(cachedStageDescriptionList) != 0 {
714+
return cachedStageDescriptionList, nil
711715
}
712716

713-
return validStages, nil
717+
return m.getStagesByDigestFromStagesStorage(ctx, stageName, stageDigest, stagesStorage)
714718
}
715719

716720
func (m *StorageManager) GetStagesByDigestFromStagesStorage(ctx context.Context, stageName, stageDigest string, stagesStorage storage.StagesStorage) ([]*image.StageDescription, error) {
717-
stageIDs, err := m.getStagesIDsByDigestFromStagesStorage(ctx, stageName, stageDigest, stagesStorage)
721+
return m.getStagesByDigestFromStagesStorage(ctx, stageName, stageDigest, stagesStorage)
722+
}
723+
724+
func (m *StorageManager) getStagesByDigestFromStagesStorage(ctx context.Context, stageName, stageDigest string, stagesStorage storage.StagesStorage, opts ...storage.Option) ([]*image.StageDescription, error) {
725+
stageIDs, err := m.getStagesIDsByDigestFromStagesStorage(ctx, stageName, stageDigest, stagesStorage, opts...)
718726
if err != nil {
719727
return nil, fmt.Errorf("unable to get stages ids from %s by digest %s for stage %s: %s", stagesStorage.String(), stageDigest, stageName, err)
720728
}
@@ -757,12 +765,12 @@ func (m *StorageManager) getWithLocalManifestCacheOption() bool {
757765
return m.StagesStorage.Address() != storage.LocalStorageAddress
758766
}
759767

760-
func (m *StorageManager) getStagesIDsByDigestFromStagesStorage(ctx context.Context, stageName, stageDigest string, stagesStorage storage.StagesStorage) ([]image.StageID, error) {
768+
func (m *StorageManager) getStagesIDsByDigestFromStagesStorage(ctx context.Context, stageName, stageDigest string, stagesStorage storage.StagesStorage, opts ...storage.Option) ([]image.StageID, error) {
761769
var stageIDs []image.StageID
762770
if err := logboek.Context(ctx).Info().LogProcess("Get %s stages by digest %s from storage", stageName, stageDigest).
763771
DoError(func() error {
764772
var err error
765-
stageIDs, err = stagesStorage.GetStagesIDsByDigest(ctx, m.ProjectName, stageDigest)
773+
stageIDs, err = stagesStorage.GetStagesIDsByDigest(ctx, m.ProjectName, stageDigest, opts...)
766774
if err != nil {
767775
return fmt.Errorf("error getting project %s stage %s images from storage: %s", m.StagesStorage.String(), stageDigest, err)
768776
}

‎pkg/storage/repo_stages_storage.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -86,10 +86,11 @@ func (storage *RepoStagesStorage) ConstructStageImageName(_, digest string, uniq
8686
return fmt.Sprintf(RepoStage_ImageFormat, storage.RepoAddress, digest, uniqueID)
8787
}
8888

89-
func (storage *RepoStagesStorage) GetStagesIDs(ctx context.Context, _ string) ([]image.StageID, error) {
89+
func (storage *RepoStagesStorage) GetStagesIDs(ctx context.Context, _ string, opts ...Option) ([]image.StageID, error) {
9090
var res []image.StageID
9191

92-
if tags, err := storage.DockerRegistry.Tags(ctx, storage.RepoAddress); err != nil {
92+
o := makeOptions(opts...)
93+
if tags, err := storage.DockerRegistry.Tags(ctx, storage.RepoAddress, o.dockerRegistryOptions...); err != nil {
9394
return nil, fmt.Errorf("unable to fetch tags for repo %q: %s", storage.RepoAddress, err)
9495
} else {
9596
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.GetStagesIDs fetched tags for %q: %#v\n", storage.RepoAddress, tags)
@@ -184,10 +185,11 @@ func (storage *RepoStagesStorage) DeleteRepo(ctx context.Context) error {
184185
return storage.DockerRegistry.DeleteRepo(ctx, storage.RepoAddress)
185186
}
186187

187-
func (storage *RepoStagesStorage) GetStagesIDsByDigest(ctx context.Context, _, digest string) ([]image.StageID, error) {
188+
func (storage *RepoStagesStorage) GetStagesIDsByDigest(ctx context.Context, _, digest string, opts ...Option) ([]image.StageID, error) {
188189
var res []image.StageID
189190

190-
if tags, err := storage.DockerRegistry.Tags(ctx, storage.RepoAddress); err != nil {
191+
o := makeOptions(opts...)
192+
if tags, err := storage.DockerRegistry.Tags(ctx, storage.RepoAddress, o.dockerRegistryOptions...); err != nil {
191193
return nil, fmt.Errorf("unable to fetch tags for repo %q: %s", storage.RepoAddress, err)
192194
} else {
193195
var rejectedStages []image.StageID

‎pkg/storage/stages_storage.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ func IsErrBrokenImage(err error) bool {
2424
}
2525

2626
type StagesStorage interface {
27-
GetStagesIDs(ctx context.Context, projectName string) ([]image.StageID, error)
28-
GetStagesIDsByDigest(ctx context.Context, projectName, digest string) ([]image.StageID, error)
27+
GetStagesIDs(ctx context.Context, projectName string, opts ...Option) ([]image.StageID, error)
28+
GetStagesIDsByDigest(ctx context.Context, projectName, digest string, opts ...Option) ([]image.StageID, error)
2929
GetStageDescription(ctx context.Context, projectName, digest string, uniqueID int64) (*image.StageDescription, error)
3030
ExportStage(ctx context.Context, stageDescription *image.StageDescription, destinationReference string) error
3131
DeleteStage(ctx context.Context, stageDescription *image.StageDescription, options DeleteImageOptions) error

0 commit comments

Comments
 (0)