Merge pull request #1633 from flant/introspect_stage

[introspection] --introspect-stage [IMAGE_NAME/]STAGE_NAME option

cmd/werf/build_and_publish/main.go

@@ -2,7 +2,6 @@ package build_and_publish
 
 import (
 	"fmt"
-
 	"github.com/spf13/cobra"
 
 	"github.com/flant/logboek"
@@ -87,6 +86,8 @@ If one or more IMAGE_NAME parameters specified, werf will build images stages an
 	common.SetupLogOptions(&CommonCmdData, cmd)
 	common.SetupLogProjectDir(&CommonCmdData, cmd)
 
+	common.SetupIntrospectStage(&CommonCmdData, cmd)
+
 	cmd.Flags().BoolVarP(&CmdData.IntrospectAfterError, "introspect-error", "", false, "Introspect failed stage in the state, right after running failed assembly instruction")
 	cmd.Flags().BoolVarP(&CmdData.IntrospectBeforeError, "introspect-before-error", "", false, "Introspect failed stage in the clean state, before running all assembly instructions of the stage")
 
@@ -165,12 +166,18 @@ func runBuildAndPublish(imagesToProcess []string) error {
 		}
 	}()
 
+	introspectOptions, err := common.GetIntrospectOptions(&CommonCmdData, werfConfig)
+	if err != nil {
+		return err
+	}
+
 	opts := build.BuildAndPublishOptions{
 		BuildStagesOptions: build.BuildStagesOptions{
 			ImageBuildOptions: image.BuildOptions{
 				IntrospectAfterError:  CmdData.IntrospectAfterError,
 				IntrospectBeforeError: CmdData.IntrospectBeforeError,
 			},
+			IntrospectOptions: introspectOptions,
 		},
 		PublishImagesOptions: build.PublishImagesOptions{
 			TagOptions: tagOpts,

cmd/werf/common/common.go

@@ -14,6 +14,9 @@ import (
 
 	"github.com/flant/kubedog/pkg/kube"
 	"github.com/flant/logboek"
+
+	"github.com/flant/werf/pkg/build"
+	"github.com/flant/werf/pkg/build/stage"
 	cleanup "github.com/flant/werf/pkg/cleaning"
 	"github.com/flant/werf/pkg/config"
 	"github.com/flant/werf/pkg/deploy/helm"
@@ -61,6 +64,8 @@ type CmdData struct {
 	GitCommitStrategyLimit      *int64
 	GitCommitStrategyExpiryDays *int64
 
+	StagesToIntrospect *[]string
+
 	LogPretty        *bool
 	LogColorMode     *string
 	LogProjectDir    *bool
@@ -299,6 +304,27 @@ func SetupLogProjectDir(cmdData *CmdData, cmd *cobra.Command) {
 	cmd.Flags().BoolVarP(cmdData.LogProjectDir, "log-project-dir", "", getBoolEnvironment("WERF_LOG_PROJECT_DIR"), `Print current project directory path (default $WERF_LOG_PROJECT_DIR)`)
 }
 
+func SetupIntrospectStage(cmdData *CmdData, cmd *cobra.Command) {
+	cmdData.StagesToIntrospect = new([]string)
+	cmd.Flags().StringArrayVarP(cmdData.StagesToIntrospect, "introspect-stage", "", []string{}, `Introspect a specific stage. The option can be used multiple times to introspect several stages.
+
+There are the following formats to use:
+* specify IMAGE_NAME/STAGE_NAME to introspect stage STAGE_NAME of either image or artifact IMAGE_NAME
+* specify STAGE_NAME or */STAGE_NAME for the introspection of all existing stages with name STAGE_NAME
+
+IMAGE_NAME is the name of an image or artifact described in werf.yaml, the nameless image specified with ~.
+STAGE_NAME should be one of the following: `+strings.Join(allStagesNames(), ", "))
+}
+
+func allStagesNames() []string {
+	var stageNames []string
+	for _, stageName := range stage.AllStages {
+		stageNames = append(stageNames, string(stageName))
+	}
+
+	return stageNames
+}
+
 func getBoolEnvironment(environmentName string) bool {
 	switch os.Getenv(environmentName) {
 	case "1", "true", "yes":
@@ -481,6 +507,48 @@ func GetNamespace(namespaceOption string) string {
 	return namespaceOption
 }
 
+func GetIntrospectOptions(cmdData *CmdData, werfConfig *config.WerfConfig) (build.IntrospectOptions, error) {
+	isStageExist := func(sName string) bool {
+		for _, stageName := range allStagesNames() {
+			if sName == stageName {
+				return true
+			}
+		}
+
+		return false
+	}
+
+	introspectOptions := build.IntrospectOptions{}
+	for _, imageAndStage := range *cmdData.StagesToIntrospect {
+		var imageName, stageName string
+
+		parts := strings.SplitN(imageAndStage, "/", 2)
+		if len(parts) == 1 {
+			imageName = "*"
+			stageName = parts[0]
+		} else {
+			if parts[0] != "~" {
+				imageName = parts[0]
+			}
+
+			stageName = parts[1]
+		}
+
+		if imageName != "*" && !werfConfig.HasImageOrArtifact(imageName) {
+			return introspectOptions, fmt.Errorf("specified image %s (%s) is not defined in werf.yaml", imageName, imageAndStage)
+		}
+
+		if !isStageExist(stageName) {
+			return introspectOptions, fmt.Errorf("specified stage name %s (%s) is not exist", stageName, imageAndStage)
+		}
+
+		introspectTarget := build.IntrospectTarget{ImageName: imageName, StageName: stageName}
+		introspectOptions.Targets = append(introspectOptions.Targets, introspectTarget)
+	}
+
+	return introspectOptions, nil
+}
+
 func LogKubeContext(kubeContext string) {
 	if kubeContext != "" {
 		logboek.LogF("Using kube context: %s\n", kubeContext)

cmd/werf/main.go

@@ -45,8 +45,8 @@ import (
 	helm_deploy_chart "github.com/flant/werf/cmd/werf/helm/deploy_chart"
 	helm_generate_chart "github.com/flant/werf/cmd/werf/helm/generate_chart"
 	helm_get_autogenerated_values "github.com/flant/werf/cmd/werf/helm/get_autogenerated_values"
-	helm_get_release "github.com/flant/werf/cmd/werf/helm/get_release"
 	helm_get_namespace "github.com/flant/werf/cmd/werf/helm/get_namespace"
+	helm_get_release "github.com/flant/werf/cmd/werf/helm/get_release"
 	helm_lint "github.com/flant/werf/cmd/werf/helm/lint"
 	helm_render "github.com/flant/werf/cmd/werf/helm/render"
 

cmd/werf/stages/build/cmd_factory/main.go

@@ -2,7 +2,6 @@ package cmd_factory
 
 import (
 	"fmt"
-
 	"github.com/spf13/cobra"
 
 	"github.com/flant/logboek"
@@ -75,6 +74,8 @@ If one or more IMAGE_NAME parameters specified, werf will build only these image
 	common.SetupDockerConfig(commonCmdData, cmd, "Command needs granted permissions to read, pull and push images into the specified stages storage, to pull base images")
 	common.SetupInsecureRepo(commonCmdData, cmd)
 
+	common.SetupIntrospectStage(commonCmdData, cmd)
+
 	common.SetupLogOptions(commonCmdData, cmd)
 	common.SetupLogProjectDir(commonCmdData, cmd)
 
@@ -144,11 +145,17 @@ func runStagesBuild(cmdData *CmdData, commonCmdData *common.CmdData, imagesToPro
 		}
 	}()
 
+	introspectOptions, err := common.GetIntrospectOptions(commonCmdData, werfConfig)
+	if err != nil {
+		return err
+	}
+
 	opts := build.BuildStagesOptions{
 		ImageBuildOptions: image.BuildOptions{
 			IntrospectAfterError:  cmdData.IntrospectAfterError,
 			IntrospectBeforeError: cmdData.IntrospectBeforeError,
 		},
+		IntrospectOptions: introspectOptions,
 	}
 
 	c := build.NewConveyor(werfConfig, imagesToProcess, projectDir, projectTmpDir, ssh_agent.SSHAuthSock)

docs/_includes/cli/werf_build.md

@@ -61,6 +61,21 @@ werf build [IMAGE_NAME...] [options]
             the stage
       --introspect-error=false:
             Introspect failed stage in the state, right after running failed assembly instruction
+      --introspect-stage=[]:
+            Introspect a specific stage. The option can be used multiple times to introspect        
+            several stages.
+
+            There are the following formats to use:
+            * specify IMAGE_NAME/STAGE_NAME to introspect stage STAGE_NAME of either image or       
+            artifact IMAGE_NAME
+            * specify STAGE_NAME or */STAGE_NAME for the introspection of all existing stages with  
+            name STAGE_NAME
+
+            IMAGE_NAME is the name of an image or artifact described in werf.yaml, the nameless     
+            image specified with ~.
+            STAGE_NAME should be one of the following: from, beforeInstall, importsBeforeInstall,   
+            gitArchive, install, importsAfterInstall, beforeSetup, importsBeforeSetup, setup,       
+            importsAfterSetup, gitCache, gitLatestPatch, dockerInstructions, dockerfile
       --log-color-mode='auto':
             Set log color mode.
             Supported on, off and auto (based on the stdout's file descriptor referring to a        

docs/_includes/cli/werf_build_and_publish.md

@@ -72,6 +72,21 @@ werf build-and-publish [IMAGE_NAME...] [options]
             the stage
       --introspect-error=false:
             Introspect failed stage in the state, right after running failed assembly instruction
+      --introspect-stage=[]:
+            Introspect a specific stage. The option can be used multiple times to introspect        
+            several stages.
+
+            There are the following formats to use:
+            * specify IMAGE_NAME/STAGE_NAME to introspect stage STAGE_NAME of either image or       
+            artifact IMAGE_NAME
+            * specify STAGE_NAME or */STAGE_NAME for the introspection of all existing stages with  
+            name STAGE_NAME
+
+            IMAGE_NAME is the name of an image or artifact described in werf.yaml, the nameless     
+            image specified with ~.
+            STAGE_NAME should be one of the following: from, beforeInstall, importsBeforeInstall,   
+            gitArchive, install, importsAfterInstall, beforeSetup, importsBeforeSetup, setup,       
+            importsAfterSetup, gitCache, gitLatestPatch, dockerInstructions, dockerfile
       --log-color-mode='auto':
             Set log color mode.
             Supported on, off and auto (based on the stdout's file descriptor referring to a        

docs/_includes/cli/werf_stages_build.md

@@ -61,6 +61,21 @@ werf stages build [IMAGE_NAME...] [options]
             the stage
       --introspect-error=false:
             Introspect failed stage in the state, right after running failed assembly instruction
+      --introspect-stage=[]:
+            Introspect a specific stage. The option can be used multiple times to introspect        
+            several stages.
+
+            There are the following formats to use:
+            * specify IMAGE_NAME/STAGE_NAME to introspect stage STAGE_NAME of either image or       
+            artifact IMAGE_NAME
+            * specify STAGE_NAME or */STAGE_NAME for the introspection of all existing stages with  
+            name STAGE_NAME
+
+            IMAGE_NAME is the name of an image or artifact described in werf.yaml, the nameless     
+            image specified with ~.
+            STAGE_NAME should be one of the following: from, beforeInstall, importsBeforeInstall,   
+            gitArchive, install, importsAfterInstall, beforeSetup, importsBeforeSetup, setup,       
+            importsAfterSetup, gitCache, gitLatestPatch, dockerInstructions, dockerfile
       --log-color-mode='auto':
             Set log color mode.
             Supported on, off and auto (based on the stdout's file descriptor referring to a        

docs/pages/reference/local_development/stage_introspection.md

@@ -7,6 +7,7 @@ summary: |
   <div class="language-bash highlighter-rouge"><pre class="highlight"><code><span class="c"># introspection before and after execution of a dysfunctional set of instructions</span>
   werf build --introspect-error
   werf build --introspect-before-error
+  werf build --introspect-stage [IMAGE_NAME/]STAGE_NAME
   </code></pre>
   </div>
 ---

pkg/build/build_stages_phase.go

@@ -7,6 +7,8 @@ import (
 	"github.com/docker/docker/pkg/stringid"
 
 	"github.com/flant/logboek"
+
+	"github.com/flant/werf/pkg/build/stage"
 	imagePkg "github.com/flant/werf/pkg/image"
 )
 
@@ -16,6 +18,26 @@ func NewBuildStagesPhase(stagesRepo string, opts BuildStagesOptions) *BuildStage
 
 type BuildStagesOptions struct {
 	ImageBuildOptions imagePkg.BuildOptions
+	IntrospectOptions
+}
+
+type IntrospectOptions struct {
+	Targets []IntrospectTarget
+}
+
+type IntrospectTarget struct {
+	ImageName string
+	StageName string
+}
+
+func (opts *IntrospectOptions) ImageStageShouldBeIntrospected(imageName, stageName string) bool {
+	for _, stage := range opts.Targets {
+		if (stage.ImageName == "*" || stage.ImageName == imageName) && stage.StageName == stageName {
+			return true
+		}
+	}
+
+	return false
 }
 
 type BuildStagesPhase struct {
@@ -66,6 +88,12 @@ func (p *BuildStagesPhase) runImage(image *Image, c *Conveyor) error {
 
 			prevStageImageSize = img.Inspect().Size
 
+			if p.IntrospectOptions.ImageStageShouldBeIntrospected(image.GetName(), string(s.Name())) {
+				if err := introspectStage(s); err != nil {
+					return err
+				}
+			}
+
 			continue
 		}
 
@@ -107,11 +135,29 @@ func (p *BuildStagesPhase) runImage(image *Image, c *Conveyor) error {
 		}
 
 		prevStageImageSize = img.Inspect().Size
+
+		if p.IntrospectOptions.ImageStageShouldBeIntrospected(image.GetName(), string(s.Name())) {
+			if err := introspectStage(s); err != nil {
+				return err
+			}
+		}
 	}
 
 	return nil
 }
 
+func introspectStage(s stage.Interface) error {
+	logProcessMessage := fmt.Sprintf("Introspecting stage %s", s.Name())
+	logProcessOptions := logboek.LogProcessOptions{ColorizeMsgFunc: logboek.ColorizeHighlight}
+	return logboek.LogProcess(logProcessMessage, logProcessOptions, func() error {
+		if err := logboek.WithRawStreamsOutputModeOn(s.GetImage().Introspect); err != nil {
+			return fmt.Errorf("introspect error failed: %s", err)
+		}
+
+		return nil
+	})
+}
+
 var (
 	logImageInfoLeftPartWidth = 12
 	logImageInfoFormat        = fmt.Sprintf("  %%%ds: %%s\n", logImageInfoLeftPartWidth)

pkg/build/stage/base.go

@@ -33,6 +33,26 @@ const (
 	Dockerfile StageName = "dockerfile"
 )
 
+var (
+	AllStages = []StageName{
+		From,
+		BeforeInstall,
+		ImportsBeforeInstall,
+		GitArchive,
+		Install,
+		ImportsAfterInstall,
+		BeforeSetup,
+		ImportsBeforeSetup,
+		Setup,
+		ImportsAfterSetup,
+		GitCache,
+		GitLatestPatch,
+		DockerInstructions,
+
+		Dockerfile,
+	}
+)
+
 type NewBaseStageOptions struct {
 	ImageName        string
 	ConfigMounts     []*config.Mount

pkg/config/werf.go

@@ -13,6 +13,14 @@ type WerfConfig struct {
 	Artifacts            []*StapelImageArtifact
 }
 
+func (c *WerfConfig) HasImageOrArtifact(imageName string) bool {
+	if c.HasImage(imageName) || c.GetArtifact(imageName) != nil {
+		return true
+	}
+
+	return false
+}
+
 func (c *WerfConfig) HasImage(imageName string) bool {
 	if c.GetImage(imageName) != nil {
 		return true