Merge pull request #2627 from werf/parallel_image_builds

Concurrent builds of images defined in werf.yaml

.github/workflows/daily_tests.yml

@@ -48,12 +48,6 @@ jobs:
 #      shell: bash
 #      if: matrix.os == 'windows-latest'
 
-    - uses: actions/cache@v1
-      with:
-        path: ~/go/pkg/mod
-        key: go-${{ hashFiles('**/go.sum') }}
-        restore-keys: go-
-
     - name: Compile tests binaries
       run: |
         # unit tests binaries

.github/workflows/tests.yml

@@ -39,12 +39,6 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v2
 
-    - uses: actions/cache@v1
-      with:
-        path: ~/go/pkg/mod
-        key: go-${{ hashFiles('**/go.sum') }}
-        restore-keys: go-
-
     - name: Compile tests binaries
       run: |
         # unit tests binaries

README.md

@@ -60,7 +60,7 @@ werf is not a complete CI/CD solution, but a tool for creating pipelines that ca
 - Developing applications locally with werf [#1940](https://github.com/werf/werf/issues/1940).
 - ~Content-based tagging~ [#1184](https://github.com/werf/werf/issues/1184).
 - ~Support for the most Docker registry implementations~ [#2199](https://github.com/werf/werf/issues/2199).
-- Parallel image builds [#2200](https://github.com/werf/werf/issues/2200).
+- ~Parallel image builds~ [#2200](https://github.com/werf/werf/issues/2200).
 - Proven approaches and recipes for the most popular CI systems [#1617](https://github.com/werf/werf/issues/1617).
 - ~Distributed builds with the shared Docker registry~ [#1614](https://github.com/werf/werf/issues/1614).
 - Support for Helm 3 [#1606](https://github.com/werf/werf/issues/1606).
@@ -73,7 +73,7 @@ werf is not a complete CI/CD solution, but a tool for creating pipelines that ca
 - Effortlessly build as many images as you like in one project.
 - Build images using Dockerfiles or Stapel builder instructions.
 - Build images concurrently on a single host (using file locks).
-- Build images simultaneously (coming soon) [#2200](https://github.com/werf/werf/issues/2200).
+- Build images simultaneously.
 - Build images distributedly.
 - Advanced building process with Stapel:
   - Incremental rebuilds based on git history.

README_ru.md

@@ -63,7 +63,7 @@ werf — не CI/CD-система, а инструмент для постро
 - Локальная разработка приложений с werf [#1940](https://github.com/werf/werf/issues/1940).
 - ~Тегирование, основанное на контенте~ [#1184](https://github.com/werf/werf/issues/1184).
 - ~Поддержка большинства имлементаций Docker registry~ [#2199](https://github.com/werf/werf/issues/2199).
-- Параллельная сборка образов [#2200](https://github.com/werf/werf/issues/2200).
+- ~Параллельная сборка образов~ [#2200](https://github.com/werf/werf/issues/2200).
 - Лучшие практики и рецепты для наиболее популярных CI-систем [#1617](https://github.com/werf/werf/issues/1617).
 - ~Распределенная сборка с общим Docker registry~ [#1614](https://github.com/werf/werf/issues/1614).
 - Поддержка Helm 3 [#1606](https://github.com/werf/werf/issues/1606).
@@ -76,8 +76,8 @@ werf — не CI/CD-система, а инструмент для постро
 - Удобная сборка произвольного числа образов в одном проекте.
 - Сборка образов как из Dockerfile, так и из инструкций сборщика Stapel.
 - Параллельные сборки на одном хосте (с использованием файловых блокировок).
-- Распределенная сборка (скоро) [#1614](https://github.com/werf/werf/issues/1614).
-- Параллельная сборка образов (скоро) [#2200](https://github.com/werf/werf/issues/2200).
+- Распределенная сборка.
+- Параллельная сборка описанных в werf.yaml образов.
 - Расширенная сборка со сборщиком Stapel:
   - Инкрементальная пересборка на основе истории изменений Git.
   - Сборка образов с Shell-инструкциями и Ansible-заданиями.

cmd/werf/build_and_publish/main.go

@@ -23,9 +23,6 @@ import (
 var cmdData struct {
 	PullUsername string
 	PullPassword string
-
-	IntrospectBeforeError bool
-	IntrospectAfterError  bool
 }
 
 var commonCmdData common.CmdData
@@ -94,6 +91,8 @@ If one or more IMAGE_NAME parameters specified, werf will build images stages an
 	common.SetupLogOptions(&commonCmdData, cmd)
 	common.SetupLogProjectDir(&commonCmdData, cmd)
 
+	common.SetupIntrospectAfterError(&commonCmdData, cmd)
+	common.SetupIntrospectBeforeError(&commonCmdData, cmd)
 	common.SetupIntrospectStage(&commonCmdData, cmd)
 
 	common.SetupSynchronization(&commonCmdData, cmd)
@@ -110,9 +109,7 @@ If one or more IMAGE_NAME parameters specified, werf will build images stages an
 
 	common.SetupGitUnshallow(&commonCmdData, cmd)
 	common.SetupAllowGitShallowClone(&commonCmdData, cmd)
-
-	cmd.Flags().BoolVarP(&cmdData.IntrospectAfterError, "introspect-error", "", false, "Introspect failed stage in the state, right after running failed assembly instruction")
-	cmd.Flags().BoolVarP(&cmdData.IntrospectBeforeError, "introspect-before-error", "", false, "Introspect failed stage in the clean state, before running all assembly instructions of the stage")
+	common.SetupParallelOptions(&commonCmdData, cmd)
 
 	return cmd
 }
@@ -211,7 +208,7 @@ func runBuildAndPublish(imagesToProcess []string) error {
 		}
 	}()
 
-	introspectOptions, err := common.GetIntrospectOptions(&commonCmdData, werfConfig)
+	buildStagesOptions, err := common.GetBuildStagesOptions(&commonCmdData, werfConfig)
 	if err != nil {
 		return err
 	}
@@ -221,14 +218,8 @@ func runBuildAndPublish(imagesToProcess []string) error {
 		return err
 	}
 
-	opts := build.BuildAndPublishOptions{
-		BuildStagesOptions: build.BuildStagesOptions{
-			ImageBuildOptions: container_runtime.BuildOptions{
-				IntrospectAfterError:  cmdData.IntrospectAfterError,
-				IntrospectBeforeError: cmdData.IntrospectBeforeError,
-			},
-			IntrospectOptions: introspectOptions,
-		},
+	buildAndPublishOptions := build.BuildAndPublishOptions{
+		BuildStagesOptions: buildStagesOptions,
 		PublishImagesOptions: build.PublishImagesOptions{
 			ImagesToPublish:     imagesToProcess,
 			TagOptions:          tagOpts,
@@ -237,13 +228,18 @@ func runBuildAndPublish(imagesToProcess []string) error {
 		},
 	}
 
+	conveyorOptions, err := common.GetConveyorOptionsWithParallel(&commonCmdData, buildAndPublishOptions.BuildStagesOptions)
+	if err != nil {
+		return err
+	}
+
 	logboek.LogOptionalLn()
 
-	conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, imagesToProcess, projectDir, projectTmpDir, ssh_agent.SSHAuthSock, containerRuntime, stagesManager, imagesRepo, storageLockManager, common.GetConveyorOptions(&commonCmdData))
+	conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, imagesToProcess, projectDir, projectTmpDir, ssh_agent.SSHAuthSock, containerRuntime, stagesManager, imagesRepo, storageLockManager, conveyorOptions)
 	defer conveyorWithRetry.Terminate()
 
 	if err := conveyorWithRetry.WithRetryBlock(ctx, func(c *build.Conveyor) error {
-		return c.BuildAndPublish(ctx, opts)
+		return c.BuildAndPublish(ctx, buildAndPublishOptions)
 	}); err != nil {
 		return err
 	}

cmd/werf/ci_env/ci_env.go

@@ -15,6 +15,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/werf/logboek"
+	"github.com/werf/logboek/pkg/level"
 
 	"github.com/werf/werf/cmd/werf/common"
 	"github.com/werf/werf/pkg/docker"
@@ -75,7 +76,7 @@ Currently supported only GitLab (gitlab) and GitHub (github) CI systems`,
 }
 
 func runCIEnv(cmd *cobra.Command, args []string) error {
-	logboek.Streams().Mute()
+	logboek.SetAcceptedLevel(level.Error)
 
 	if err := werf.Init(*commonCmdData.TmpDir, *commonCmdData.HomeDir); err != nil {
 		return fmt.Errorf("initialization error: %s", err)

cmd/werf/common/common.go

@@ -80,6 +80,8 @@ type CmdData struct {
 	GitHistorySynchronization *bool
 	GitUnshallow              *bool
 	AllowGitShallowClone      *bool
+	Parallel                  *bool
+	ParallelTasksLimit        *int64
 
 	DockerConfig          *string
 	InsecureRegistry      *bool
@@ -95,7 +97,9 @@ type CmdData struct {
 
 	WithoutKube *bool
 
-	StagesToIntrospect *[]string
+	IntrospectBeforeError *bool
+	IntrospectAfterError  *bool
+	StagesToIntrospect    *[]string
 
 	LogDebug         *bool
 	LogPretty        *bool
@@ -731,6 +735,21 @@ func SetupAllowGitShallowClone(cmdData *CmdData, cmd *cobra.Command) {
 	cmd.Flags().BoolVarP(cmdData.AllowGitShallowClone, "allow-git-shallow-clone", "", GetBoolEnvironmentDefaultFalse("WERF_ALLOW_GIT_SHALLOW_CLONE"), "Sign the intention of using shallow clone despite restrictions (default $WERF_ALLOW_GIT_SHALLOW_CLONE)")
 }
 
+func SetupParallelOptions(cmdData *CmdData, cmd *cobra.Command) {
+	SetupParallel(cmdData, cmd)
+	SetupParallelTasksLimit(cmdData, cmd)
+}
+
+func SetupParallel(cmdData *CmdData, cmd *cobra.Command) {
+	cmdData.Parallel = new(bool)
+	cmd.Flags().BoolVarP(cmdData.Parallel, "parallel", "p", GetBoolEnvironmentDefaultFalse("WERF_PARALLEL"), "Run in parallel (default $WERF_PARALLEL)")
+}
+
+func SetupParallelTasksLimit(cmdData *CmdData, cmd *cobra.Command) {
+	cmdData.ParallelTasksLimit = new(int64)
+	cmd.Flags().Int64VarP(cmdData.ParallelTasksLimit, "parallel-tasks-limit", "", -1, "Parallel tasks limit (default $WERF_PARALLEL_TASKS_LIMIT or without limit)")
+}
+
 func SetupGitUnshallow(cmdData *CmdData, cmd *cobra.Command) {
 	cmdData.GitUnshallow = new(bool)
 	cmd.Flags().BoolVarP(cmdData.GitUnshallow, "git-unshallow", "", GetBoolEnvironmentDefaultFalse("WERF_GIT_UNSHALLOW"), "Convert project git clone to full one (default $WERF_GIT_UNSHALLOW)")
@@ -746,6 +765,16 @@ func SetupLogProjectDir(cmdData *CmdData, cmd *cobra.Command) {
 	cmd.Flags().BoolVarP(cmdData.LogProjectDir, "log-project-dir", "", GetBoolEnvironmentDefaultFalse("WERF_LOG_PROJECT_DIR"), `Print current project directory path (default $WERF_LOG_PROJECT_DIR)`)
 }
 
+func SetupIntrospectAfterError(cmdData *CmdData, cmd *cobra.Command) {
+	cmdData.IntrospectAfterError = new(bool)
+	cmd.Flags().BoolVarP(cmdData.IntrospectAfterError, "introspect-error", "", false, "Introspect failed stage in the state, right after running failed assembly instruction")
+}
+
+func SetupIntrospectBeforeError(cmdData *CmdData, cmd *cobra.Command) {
+	cmdData.IntrospectBeforeError = new(bool)
+	cmd.Flags().BoolVarP(cmdData.IntrospectBeforeError, "introspect-before-error", "", false, "Introspect failed stage in the clean state, before running all assembly instructions of the stage")
+}
+
 func SetupIntrospectStage(cmdData *CmdData, cmd *cobra.Command) {
 	cmdData.StagesToIntrospect = new([]string)
 	cmd.Flags().StringArrayVarP(cmdData.StagesToIntrospect, "introspect-stage", "", []string{}, `Introspect a specific stage. The option can be used multiple times to introspect several stages.
@@ -856,6 +885,21 @@ func getIntEnvVar(varName string) (*int64, error) {
 	return nil, nil
 }
 
+func GetParallelTasksLimit(cmdData *CmdData) (int64, error) {
+	v, err := getInt64EnvVar("WERF_PARALLEL_TASKS_LIMIT")
+	if err != nil {
+		return 0, err
+	}
+	if v == nil {
+		v = cmdData.ParallelTasksLimit
+	}
+	if *v <= 0 {
+		return -1, nil
+	} else {
+		return *v, nil
+	}
+}
+
 func GetGitTagStrategyLimit(cmdData *CmdData) (int64, error) {
 	v, err := getInt64EnvVar("WERF_GIT_TAG_STRATEGY_LIMIT")
 	if err != nil {

cmd/werf/common/conveyor_options.go

@@ -1,8 +1,12 @@
 package common
 
 import (
+	"fmt"
+
 	"github.com/werf/werf/pkg/build"
 	"github.com/werf/werf/pkg/build/stage"
+	"github.com/werf/werf/pkg/config"
+	"github.com/werf/werf/pkg/container_runtime"
 )
 
 func GetConveyorOptions(commonCmdData *CmdData) build.ConveyorOptions {
@@ -16,3 +20,34 @@ func GetConveyorOptions(commonCmdData *CmdData) build.ConveyorOptions {
 		AllowGitShallowClone: *commonCmdData.AllowGitShallowClone,
 	}
 }
+
+func GetConveyorOptionsWithParallel(commonCmdData *CmdData, buildStagesOptions build.BuildStagesOptions) (build.ConveyorOptions, error) {
+	conveyorOptions := GetConveyorOptions(commonCmdData)
+	conveyorOptions.Parallel = !(buildStagesOptions.ImageBuildOptions.IntrospectAfterError || buildStagesOptions.ImageBuildOptions.IntrospectBeforeError || len(buildStagesOptions.Targets) != 0) && *commonCmdData.Parallel
+
+	parallelTasksLimit, err := GetParallelTasksLimit(commonCmdData)
+	if err != nil {
+		return conveyorOptions, fmt.Errorf("getting parallel tasks limit failed: %s", err)
+	}
+
+	conveyorOptions.ParallelTasksLimit = parallelTasksLimit
+
+	return conveyorOptions, nil
+}
+
+func GetBuildStagesOptions(commonCmdData *CmdData, werfConfig *config.WerfConfig) (build.BuildStagesOptions, error) {
+	introspectOptions, err := GetIntrospectOptions(commonCmdData, werfConfig)
+	if err != nil {
+		return build.BuildStagesOptions{}, err
+	}
+
+	options := build.BuildStagesOptions{
+		ImageBuildOptions: container_runtime.BuildOptions{
+			IntrospectAfterError:  *commonCmdData.IntrospectAfterError,
+			IntrospectBeforeError: *commonCmdData.IntrospectBeforeError,
+		},
+		IntrospectOptions: introspectOptions,
+	}
+
+	return options, nil
+}

cmd/werf/converge/converge.go

@@ -115,6 +115,7 @@ werf converge --stages-storage registry.mydomain.com/web/back/stages --images-re
 
 	common.SetupGitUnshallow(&commonCmdData, cmd)
 	common.SetupAllowGitShallowClone(&commonCmdData, cmd)
+	common.SetupParallelOptions(&commonCmdData, cmd)
 
 	cmd.Flags().IntVarP(&cmdData.Timeout, "timeout", "t", 0, "Resources tracking timeout in seconds")
 
@@ -233,7 +234,7 @@ func runConverge() error {
 		return fmt.Errorf("cannot init kubedog: %s", err)
 	}
 
-	opts := build.BuildAndPublishOptions{
+	buildAndPublishOptions := build.BuildAndPublishOptions{
 		BuildStagesOptions: build.BuildStagesOptions{
 			ImageBuildOptions: container_runtime.BuildOptions{},
 		},
@@ -277,11 +278,16 @@ func runConverge() error {
 		}
 		imagesRepository = imagesRepo.String()
 
-		conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, nil, projectDir, projectTmpDir, ssh_agent.SSHAuthSock, containerRuntime, stagesManager, imagesRepo, storageLockManager, common.GetConveyorOptions(&commonCmdData))
+		conveyorOptions, err := common.GetConveyorOptionsWithParallel(&commonCmdData, buildAndPublishOptions.BuildStagesOptions)
+		if err != nil {
+			return err
+		}
+
+		conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, nil, projectDir, projectTmpDir, ssh_agent.SSHAuthSock, containerRuntime, stagesManager, imagesRepo, storageLockManager, conveyorOptions)
 		defer conveyorWithRetry.Terminate()
 
 		if err := conveyorWithRetry.WithRetryBlock(ctx, func(c *build.Conveyor) error {
-			if err := c.BuildAndPublish(ctx, opts); err != nil {
+			if err := c.BuildAndPublish(ctx, buildAndPublishOptions); err != nil {
 				return err
 			}
 

cmd/werf/helm/get_autogenerated_values/main.go

@@ -6,6 +6,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/werf/logboek"
+	"github.com/werf/logboek/pkg/level"
 
 	"github.com/werf/werf/cmd/werf/common"
 	helm_common "github.com/werf/werf/cmd/werf/helm/common"
@@ -65,8 +66,7 @@ These values includes project name, docker images ids and other`),
 }
 
 func runGetServiceValues() error {
-	logboek.Streams().Mute()
-	defer logboek.Streams().Unmute()
+	logboek.SetAcceptedLevel(level.Error)
 
 	ctx := common.BackgroundContext()
 
@@ -139,7 +139,7 @@ func runGetServiceValues() error {
 	defer func() {
 		err := ssh_agent.Terminate()
 		if err != nil {
-			logboek.Warn().LogF("WARNING: ssh agent termination failed: %s\n", err)
+			logboek.Error().LogF("WARNING: ssh agent termination failed: %s\n", err)
 		}
 	}()
 

cmd/werf/helm/secret/common/common.go

@@ -8,6 +8,7 @@ import (
 
 	"golang.org/x/crypto/ssh/terminal"
 
+	"github.com/werf/logboek"
 	"github.com/werf/logboek/pkg/style"
 
 	"github.com/werf/werf/pkg/util"
@@ -45,7 +46,7 @@ func InputFromInteractiveStdin() ([]byte, error) {
 
 	isStdoutTerminal := terminal.IsTerminal(int(os.Stdout.Fd()))
 	if isStdoutTerminal {
-		fmt.Printf(style.Highlight().Colorize("Enter secret: "))
+		fmt.Printf(logboek.Colorize(style.Highlight(), "Enter secret: "))
 	}
 
 	data, err = terminal.ReadPassword(int(os.Stdin.Fd()))

cmd/werf/helm/secret/common/edit.go

@@ -148,7 +148,7 @@ func readEditedFile(m secret.Manager, filePath string, values bool) ([]byte, []b
 func askForConfirmation() (bool, error) {
 	r := os.Stdin
 
-	fmt.Println(style.Highlight().Colorize("Do you want to continue editing the file (Y/n)?"))
+	fmt.Println(logboek.Colorize(style.Highlight(), "Do you want to continue editing the file (Y/n)?"))
 
 	isTerminal := terminal.IsTerminal(int(r.Fd()))
 	if isTerminal {