Skip to content
Permalink
Browse files
feat(build): support using only specific images from werf.yaml or dis…
…abling images for all werf commands

* Specific images enabled by specifying positional arguments in all related commands:
    * werf build
    * werf converge
    * werf render
    * werf bundle publish
    * werf export
    * werf config graph
    * werf helm get-autogenerated-values
    * werf compose
* Support --without-images option for all related commands except werf-build, werf-export and werf-config-graph commands.
* Do not connect to repository and synchronization service when --without-images is used.

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
distorhead committed Sep 20, 2022
1 parent b089f2d commit c618043fd5c2f53c88e60287c9b7f161b61f8901
Show file tree
Hide file tree
Showing 19 changed files with 235 additions and 149 deletions.
@@ -63,7 +63,7 @@ If one or more IMAGE_NAME parameters specified, werf will build only these image
common.LogVersion()

return common.LogRunningTime(func() error {
return runMain(ctx, args)
return runMain(ctx, common.GetImagesToProcess(args, false))
})
},
}))
@@ -123,7 +123,7 @@ If one or more IMAGE_NAME parameters specified, werf will build only these image
return cmd
}

func runMain(ctx context.Context, args []string) error {
func runMain(ctx context.Context, imagesToProcess build.ImagesToProcess) error {
global_warnings.PostponeMultiwerfNotUpToDateWarning()

if err := werf.Init(*commonCmdData.TmpDir, *commonCmdData.HomeDir); err != nil {
@@ -187,25 +187,24 @@ func runMain(ctx context.Context, args []string) error {
if *commonCmdData.Follow {
logboek.LogOptionalLn()
return common.FollowGitHead(ctx, &commonCmdData, func(ctx context.Context, headCommitGiterminismManager giterminism_manager.Interface) error {
return run(ctx, containerBackend, headCommitGiterminismManager, args)
return run(ctx, containerBackend, headCommitGiterminismManager, imagesToProcess)
})
} else {
return run(ctx, containerBackend, giterminismManager, args)
return run(ctx, containerBackend, giterminismManager, imagesToProcess)
}
}

func run(ctx context.Context, containerBackend container_backend.ContainerBackend, giterminismManager giterminism_manager.Interface, imagesToProcess []string) error {
func run(ctx context.Context, containerBackend container_backend.ContainerBackend, giterminismManager giterminism_manager.Interface, imagesToProcess build.ImagesToProcess) error {
_, werfConfig, err := common.GetRequiredWerfConfig(ctx, &commonCmdData, giterminismManager, common.GetWerfConfigOptions(&commonCmdData, true))
if err != nil {
return fmt.Errorf("unable to load werf config: %w", err)
}

projectName := werfConfig.Meta.Project

if err := werfConfig.CheckThatImagesExist(imagesToProcess); err != nil {
if err := werfConfig.CheckThatImagesExist(imagesToProcess.OnlyImages); err != nil {
return err
}

projectName := werfConfig.Meta.Project

projectTmpDir, err := tmp_manager.CreateProjectDir(ctx)
if err != nil {
return fmt.Errorf("getting project tmp dir failed: %w", err)
@@ -245,14 +244,14 @@ func run(ctx context.Context, containerBackend container_backend.ContainerBacken
return err
}

conveyorOptions, err := common.GetConveyorOptionsWithParallel(&commonCmdData, buildOptions)
conveyorOptions, err := common.GetConveyorOptionsWithParallel(&commonCmdData, imagesToProcess, buildOptions)
if err != nil {
return err
}

logboek.LogOptionalLn()

conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, imagesToProcess, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, conveyorOptions)
conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, conveyorOptions)
defer conveyorWithRetry.Terminate()

if err := conveyorWithRetry.WithRetryBlock(ctx, func(c *build.Conveyor) error {
@@ -40,7 +40,7 @@ var commonCmdData common.CmdData
func NewCmd(ctx context.Context) *cobra.Command {
ctx = common.NewContextWithCmdData(ctx, &commonCmdData)
cmd := common.SetCommandContext(ctx, &cobra.Command{
Use: "export",
Use: "export [IMAGE_NAME...]",
Short: "Export bundle",
Hidden: true, // Deprecated command
Long: common.GetLongCommandDescription(`Export bundle into the provided directory (or into directory named as a resulting chart in the current working directory). werf bundle contains built images defined in the werf.yaml, helm chart, service values which contain built images tags, any custom values and set values params provided during publish invocation, werf service templates and values.`),
@@ -76,11 +76,13 @@ func NewCmd(ctx context.Context) *cobra.Command {
common.LogVersion()

return common.LogRunningTime(func() error {
return runExport(ctx)
return runExport(ctx, common.GetImagesToProcess(args, *commonCmdData.WithoutImages))
})
},
})

commonCmdData.SetupWithoutImages(cmd)

common.SetupDir(&commonCmdData, cmd)
common.SetupGitWorkTree(&commonCmdData, cmd)
common.SetupGiterminismOptions(&commonCmdData, cmd)
@@ -142,7 +144,7 @@ func NewCmd(ctx context.Context) *cobra.Command {
return cmd
}

func runExport(ctx context.Context) error {
func runExport(ctx context.Context, imagesToProcess build.ImagesToProcess) error {
if err := werf.Init(*commonCmdData.TmpDir, *commonCmdData.HomeDir); err != nil {
return fmt.Errorf("initialization error: %w", err)
}
@@ -195,6 +197,9 @@ func runExport(ctx context.Context) error {
if err != nil {
return fmt.Errorf("unable to load werf config: %w", err)
}
if err := werfConfig.CheckThatImagesExist(imagesToProcess.OnlyImages); err != nil {
return err
}

projectName := werfConfig.Meta.Project

@@ -239,7 +244,7 @@ func runExport(ctx context.Context) error {
var imagesInfoGetters []*image.InfoGetter
var imagesRepo string

if len(werfConfig.StapelImages) != 0 || len(werfConfig.ImagesFromDockerfile) != 0 {
if !imagesToProcess.WithoutImages && (len(werfConfig.StapelImages)+len(werfConfig.ImagesFromDockerfile) > 0) {
stagesStorage, err := common.GetStagesStorage(ctx, containerBackend, &commonCmdData)
if err != nil {
return err
@@ -273,12 +278,12 @@ func runExport(ctx context.Context) error {

imagesRepo = storageManager.GetServiceValuesRepo()

conveyorOptions, err := common.GetConveyorOptionsWithParallel(&commonCmdData, buildOptions)
conveyorOptions, err := common.GetConveyorOptionsWithParallel(&commonCmdData, imagesToProcess, buildOptions)
if err != nil {
return err
}

conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, nil, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, conveyorOptions)
conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, conveyorOptions)
defer conveyorWithRetry.Terminate()

if err := conveyorWithRetry.WithRetryBlock(ctx, func(c *build.Conveyor) error {
@@ -43,7 +43,7 @@ var commonCmdData common.CmdData
func NewCmd(ctx context.Context) *cobra.Command {
ctx = common.NewContextWithCmdData(ctx, &commonCmdData)
cmd := common.SetCommandContext(ctx, &cobra.Command{
Use: "publish",
Use: "publish [IMAGE_NAME...]",
Short: "Publish bundle",
Long: common.GetLongCommandDescription(`Publish bundle into the container registry. werf bundle contains built images defined in the werf.yaml, helm chart, service values which contain built images tags, any custom values and set values params provided during publish invocation, werf addon templates (like werf_image).
@@ -65,10 +65,12 @@ Published into container registry bundle can be rolled out by the "werf bundle"

common.LogVersion()

return common.LogRunningTime(func() error { return runPublish(ctx) })
return common.LogRunningTime(func() error { return runPublish(ctx, common.GetImagesToProcess(args, *commonCmdData.WithoutImages)) })
},
})

commonCmdData.SetupWithoutImages(cmd)

common.SetupDir(&commonCmdData, cmd)
common.SetupGitWorkTree(&commonCmdData, cmd)
common.SetupGiterminismOptions(&commonCmdData, cmd)
@@ -139,7 +141,7 @@ Published into container registry bundle can be rolled out by the "werf bundle"
return cmd
}

func runPublish(ctx context.Context) error {
func runPublish(ctx context.Context, imagesToProcess build.ImagesToProcess) error {
global_warnings.PostponeMultiwerfNotUpToDateWarning()

if err := werf.Init(*commonCmdData.TmpDir, *commonCmdData.HomeDir); err != nil {
@@ -194,6 +196,9 @@ func runPublish(ctx context.Context) error {
if err != nil {
return fmt.Errorf("unable to load werf config: %w", err)
}
if err := werfConfig.CheckThatImagesExist(imagesToProcess.OnlyImages); err != nil {
return err
}

projectName := werfConfig.Meta.Project

@@ -252,7 +257,7 @@ func runPublish(ctx context.Context) error {
var imagesInfoGetters []*image.InfoGetter
var imagesRepo string

if len(werfConfig.StapelImages) != 0 || len(werfConfig.ImagesFromDockerfile) != 0 {
if !imagesToProcess.WithoutImages && (len(werfConfig.StapelImages)+len(werfConfig.ImagesFromDockerfile) > 0) {
synchronization, err := common.GetSynchronization(ctx, &commonCmdData, projectName, stagesStorage)
if err != nil {
return err
@@ -278,12 +283,12 @@ func runPublish(ctx context.Context) error {

imagesRepo = storageManager.GetServiceValuesRepo()

conveyorOptions, err := common.GetConveyorOptionsWithParallel(&commonCmdData, buildOptions)
conveyorOptions, err := common.GetConveyorOptionsWithParallel(&commonCmdData, imagesToProcess, buildOptions)
if err != nil {
return err
}

conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, nil, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, conveyorOptions)
conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, conveyorOptions)
defer conveyorWithRetry.Terminate()

if err := conveyorWithRetry.WithRetryBlock(ctx, func(c *build.Conveyor) error {
@@ -1,5 +1,11 @@
package common

import (
"github.com/spf13/cobra"

"github.com/werf/werf/pkg/util"
)

type CmdData struct {
GitWorkTree *string
ProjectName *string
@@ -32,8 +38,9 @@ type CmdData struct {
SecretValues *[]string
IgnoreSecretKey *bool

Repo *RepoData
FinalRepo *RepoData
WithoutImages *bool
Repo *RepoData
FinalRepo *RepoData

SecondaryStagesStorage *[]string
CacheStagesStorage *[]string
@@ -92,3 +99,8 @@ type CmdData struct {

Platform *string
}

func (cmdData *CmdData) SetupWithoutImages(cmd *cobra.Command) {
cmdData.WithoutImages = new(bool)
cmd.Flags().BoolVarP(cmdData.WithoutImages, "without-images", "", util.GetBoolEnvironmentDefaultFalse("WERF_WITHOUT_IMAGES"), "Disable building of images defined in the werf.yaml (if any) and usage of such images in the .helm/templates ($WERF_WITHOUT_IMAGES or false by default — e.g. enable all images defined in the werf.yaml by default)")
}
@@ -16,16 +16,17 @@ import (
"github.com/werf/werf/pkg/storage"
)

func GetConveyorOptions(commonCmdData *CmdData) build.ConveyorOptions {
func GetConveyorOptions(commonCmdData *CmdData, imagesToProcess build.ImagesToProcess) build.ConveyorOptions {
return build.ConveyorOptions{
LocalGitRepoVirtualMergeOptions: stage.VirtualMergeOptions{
VirtualMerge: *commonCmdData.VirtualMerge,
},
ImagesToProcess: imagesToProcess,
}
}

func GetConveyorOptionsWithParallel(commonCmdData *CmdData, buildStagesOptions build.BuildOptions) (build.ConveyorOptions, error) {
conveyorOptions := GetConveyorOptions(commonCmdData)
func GetConveyorOptionsWithParallel(commonCmdData *CmdData, imagesToProcess build.ImagesToProcess, buildStagesOptions build.BuildOptions) (build.ConveyorOptions, error) {
conveyorOptions := GetConveyorOptions(commonCmdData, imagesToProcess)
conveyorOptions.Parallel = !(buildStagesOptions.ImageBuildOptions.IntrospectAfterError || buildStagesOptions.ImageBuildOptions.IntrospectBeforeError || len(buildStagesOptions.Targets) != 0) && *commonCmdData.Parallel

parallelTasksLimit, err := GetParallelTasksLimit(commonCmdData)
@@ -0,0 +1,12 @@
package common

import "github.com/werf/werf/pkg/build"

func GetImagesToProcess(onlyImages []string, withoutImages bool) build.ImagesToProcess {
if withoutImages {
return build.NewImagesToProcess(nil, true)
} else if len(onlyImages) > 0 {
return build.NewImagesToProcess(onlyImages, false)
}
return build.NewImagesToProcess(nil, false)
}

0 comments on commit c618043

Please sign in to comment.