Skip to content

Commit

Permalink
feat(export): export images concurrently(#6395)
Browse files Browse the repository at this point in the history
Signed-off-by: Yaroslav Pershin <62902094+iapershin@users.noreply.github.com>
  • Loading branch information
iapershin authored and alexey-igrychev committed Nov 15, 2024
1 parent 1752e91 commit ddd804f
Show file tree
Hide file tree
Showing 3 changed files with 29 additions and 6 deletions.
7 changes: 6 additions & 1 deletion pkg/build/build_phase.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/werf/werf/pkg/storage"
"github.com/werf/werf/pkg/storage/manager"
"github.com/werf/werf/pkg/util"
"github.com/werf/werf/pkg/util/parallel"
"github.com/werf/werf/pkg/werf"
)

Expand Down Expand Up @@ -156,7 +157,7 @@ func GenerateImageEnv(werfImageName, imageName string) string {
imageEnvName = "WERF_DOCKER_IMAGE_NAME"
} else {
werfImageName := strings.ToUpper(werfImageName)
for _, l := range []string{"/", "-"} {
for _, l := range []string{"/", "-", "."} {
werfImageName = strings.ReplaceAll(werfImageName, l, "_")
}

Expand Down Expand Up @@ -285,6 +286,10 @@ func (phase *BuildPhase) AfterImages(ctx context.Context) error {
}
}
}

return nil
}); err != nil {
return err
}

return phase.createReport(ctx)
Expand Down
20 changes: 15 additions & 5 deletions pkg/build/export_phase.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/werf/werf/pkg/image"
"github.com/werf/werf/pkg/storage"
"github.com/werf/werf/pkg/util"
"github.com/werf/werf/pkg/util/parallel"
)

type Exporter struct {
Expand All @@ -39,10 +40,16 @@ func (e *Exporter) Run(ctx context.Context) error {
return nil
}

for _, desc := range e.Conveyor.imagesTree.GetImagesByName(true) {
name, images := desc.Unpair()
if !slices.Contains(e.ExportImageNameList, name) {
continue
imageList := util.SliceToMapWithValue(e.ExportImageNameList, struct{}{})
images := e.Conveyor.imagesTree.GetImagesByName(true)

if err := parallel.DoTasks(ctx, len(e.ExportImageNameList), parallel.DoTasksOptions{
MaxNumberOfWorkers: int(e.Conveyor.ParallelTasksLimit),
}, func(ctx context.Context, taskId int) error {
pair := images[taskId]
name, images := pair.Unpair()
if _, ok := imageList[name]; !ok {
return nil
}

targetPlatforms := util.MapFuncToSlice(images, func(img *build_image.Image) string { return img.TargetPlatform })
Expand All @@ -55,7 +62,7 @@ func (e *Exporter) Run(ctx context.Context) error {
// FIXME(multiarch): Support multiplatform manifest by pushing local images to repo first, then create manifest list.
// FIXME(multiarch): Also support multiplatform manifest in werf build command in local mode with enabled final-repo.
if _, isLocal := e.Conveyor.StorageManager.GetStagesStorage().(*storage.LocalStagesStorage); isLocal {
return fmt.Errorf("export command is not supported in multiplatform mode")
return fmt.Errorf("export command in multiplatform mode should be used with remote stages storage")
}

// multiplatform mode
Expand All @@ -64,6 +71,9 @@ func (e *Exporter) Run(ctx context.Context) error {
return fmt.Errorf("unable to export multiplatform image %q: %w", img.Name, err)
}
}
return nil
}); err != nil {
return fmt.Errorf("export failed: %w", err)
}

return nil
Expand Down
8 changes: 8 additions & 0 deletions pkg/util/slice.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,11 @@ func MapFuncToSlice[T, RT any, FT func(T) RT](arr []T, f FT) (res []RT) {
}
return
}

func SliceToMapWithValue[K comparable, V any](keys []K, value V) map[K]V {
resultMap := make(map[K]V)
for _, key := range keys {
resultMap[key] = value
}
return resultMap
}

0 comments on commit ddd804f

Please sign in to comment.