config.go

/*
Ginkgo accepts a number of configuration options.
These are documented [here](http://onsi.github.io/ginkgo/#the-ginkgo-cli)
*/

package types

import (
	"flag"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"
)

// Configuration controlling how an individual test suite is run
type SuiteConfig struct {
	RandomSeed            int64
	RandomizeAllSpecs     bool
	FocusStrings          []string
	SkipStrings           []string
	FocusFiles            []string
	SkipFiles             []string
	LabelFilter           string
	FailOnPending         bool
	FailFast              bool
	FlakeAttempts         int
	DryRun                bool
	PollProgressAfter     time.Duration
	PollProgressInterval  time.Duration
	Timeout               time.Duration
	EmitSpecProgress      bool // this is deprecated but its removal is causing compile issue for some users that were setting it manually
	OutputInterceptorMode string
	SourceRoots           []string
	GracePeriod           time.Duration

	ParallelProcess int
	ParallelTotal   int
	ParallelHost    string
}

func NewDefaultSuiteConfig() SuiteConfig {
	return SuiteConfig{
		RandomSeed:      time.Now().Unix(),
		Timeout:         time.Hour,
		ParallelProcess: 1,
		ParallelTotal:   1,
		GracePeriod:     30 * time.Second,
	}
}

type VerbosityLevel uint

const (
	VerbosityLevelSuccinct VerbosityLevel = iota
	VerbosityLevelNormal
	VerbosityLevelVerbose
	VerbosityLevelVeryVerbose
)

func (vl VerbosityLevel) GT(comp VerbosityLevel) bool {
	return vl > comp
}

func (vl VerbosityLevel) GTE(comp VerbosityLevel) bool {
	return vl >= comp
}

func (vl VerbosityLevel) Is(comp VerbosityLevel) bool {
	return vl == comp
}

func (vl VerbosityLevel) LTE(comp VerbosityLevel) bool {
	return vl <= comp
}

func (vl VerbosityLevel) LT(comp VerbosityLevel) bool {
	return vl < comp
}

// Configuration for Ginkgo's reporter
type ReporterConfig struct {
	NoColor        bool
	Succinct       bool
	Verbose        bool
	VeryVerbose    bool
	FullTrace      bool
	ShowNodeEvents bool

	JSONReport     string
	JUnitReport    string
	TeamcityReport string
}

func (rc ReporterConfig) Verbosity() VerbosityLevel {
	if rc.Succinct {
		return VerbosityLevelSuccinct
	} else if rc.Verbose {
		return VerbosityLevelVerbose
	} else if rc.VeryVerbose {
		return VerbosityLevelVeryVerbose
	}
	return VerbosityLevelNormal
}

func (rc ReporterConfig) WillGenerateReport() bool {
	return rc.JSONReport != "" || rc.JUnitReport != "" || rc.TeamcityReport != ""
}

func NewDefaultReporterConfig() ReporterConfig {
	return ReporterConfig{}
}

// Configuration for the Ginkgo CLI
type CLIConfig struct {
	//for build, run, and watch
	Recurse      bool
	SkipPackage  string
	RequireSuite bool
	NumCompilers int

	//for run and watch only
	Procs                     int
	Parallel                  bool
	AfterRunHook              string
	OutputDir                 string
	KeepSeparateCoverprofiles bool
	KeepSeparateReports       bool

	//for run only
	KeepGoing       bool
	UntilItFails    bool
	Repeat          int
	RandomizeSuites bool

	//for watch only
	Depth       int
	WatchRegExp string
}

func NewDefaultCLIConfig() CLIConfig {
	return CLIConfig{
		Depth:       1,
		WatchRegExp: `\.go$`,
	}
}

func (g CLIConfig) ComputedProcs() int {
	if g.Procs > 0 {
		return g.Procs
	}

	n := 1
	if g.Parallel {
		n = runtime.NumCPU()
		if n > 4 {
			n = n - 1
		}
	}
	return n
}

func (g CLIConfig) ComputedNumCompilers() int {
	if g.NumCompilers > 0 {
		return g.NumCompilers
	}

	return runtime.NumCPU()
}

// Configuration for the Ginkgo CLI capturing available go flags
// A subset of Go flags are exposed by Ginkgo.  Some are available at compile time (e.g. ginkgo build) and others only at run time (e.g. ginkgo run - which has both build and run time flags).
// More details can be found at:
// https://docs.google.com/spreadsheets/d/1zkp-DS4hU4sAJl5eHh1UmgwxCPQhf3s5a8fbiOI8tJU/
type GoFlagsConfig struct {
	//build-time flags for code-and-performance analysis
	Race      bool
	Cover     bool
	CoverMode string
	CoverPkg  string
	Vet       string

	//run-time flags for code-and-performance analysis
	BlockProfile         string
	BlockProfileRate     int
	CoverProfile         string
	CPUProfile           string
	MemProfile           string
	MemProfileRate       int
	MutexProfile         string
	MutexProfileFraction int
	Trace                string

	//build-time flags for building
	A             bool
	ASMFlags      string
	BuildMode     string
	Compiler      string
	GCCGoFlags    string
	GCFlags       string
	InstallSuffix string
	LDFlags       string
	LinkShared    bool
	Mod           string
	N             bool
	ModFile       string
	ModCacheRW    bool
	MSan          bool
	PkgDir        string
	Tags          string
	TrimPath      bool
	ToolExec      string
	Work          bool
	X             bool
}

func NewDefaultGoFlagsConfig() GoFlagsConfig {
	return GoFlagsConfig{}
}

func (g GoFlagsConfig) BinaryMustBePreserved() bool {
	return g.BlockProfile != "" || g.CPUProfile != "" || g.MemProfile != "" || g.MutexProfile != ""
}

// Configuration that were deprecated in 2.0
type deprecatedConfig struct {
	DebugParallel                   bool
	NoisySkippings                  bool
	NoisyPendings                   bool
	RegexScansFilePath              bool
	SlowSpecThresholdWithFLoatUnits float64
	Stream                          bool
	Notify                          bool
	EmitSpecProgress                bool
	SlowSpecThreshold               time.Duration
	AlwaysEmitGinkgoWriter          bool
}

// Flags

// Flags sections used by both the CLI and the Ginkgo test process
var FlagSections = GinkgoFlagSections{
	{Key: "multiple-suites", Style: "{{dark-green}}", Heading: "Running Multiple Test Suites"},
	{Key: "order", Style: "{{green}}", Heading: "Controlling Test Order"},
	{Key: "parallel", Style: "{{yellow}}", Heading: "Controlling Test Parallelism"},
	{Key: "low-level-parallel", Style: "{{yellow}}", Heading: "Controlling Test Parallelism",
		Description: "These are set by the Ginkgo CLI, {{red}}{{bold}}do not set them manually{{/}} via go test.\nUse ginkgo -p or ginkgo -procs=N instead."},
	{Key: "filter", Style: "{{cyan}}", Heading: "Filtering Tests"},
	{Key: "failure", Style: "{{red}}", Heading: "Failure Handling"},
	{Key: "output", Style: "{{magenta}}", Heading: "Controlling Output Formatting"},
	{Key: "code-and-coverage-analysis", Style: "{{orange}}", Heading: "Code and Coverage Analysis"},
	{Key: "performance-analysis", Style: "{{coral}}", Heading: "Performance Analysis"},
	{Key: "debug", Style: "{{blue}}", Heading: "Debugging Tests",
		Description: "In addition to these flags, Ginkgo supports a few debugging environment variables.  To change the parallel server protocol set {{blue}}GINKGO_PARALLEL_PROTOCOL{{/}} to {{bold}}HTTP{{/}}.  To avoid pruning callstacks set {{blue}}GINKGO_PRUNE_STACK{{/}} to {{bold}}FALSE{{/}}."},
	{Key: "watch", Style: "{{light-yellow}}", Heading: "Controlling Ginkgo Watch"},
	{Key: "misc", Style: "{{light-gray}}", Heading: "Miscellaneous"},
	{Key: "go-build", Style: "{{light-gray}}", Heading: "Go Build Flags", Succinct: true,
		Description: "These flags are inherited from go build.  Run {{bold}}ginkgo help build{{/}} for more detailed flag documentation."},
}

// SuiteConfigFlags provides flags for the Ginkgo test process, and CLI
var SuiteConfigFlags = GinkgoFlags{
	{KeyPath: "S.RandomSeed", Name: "seed", SectionKey: "order", UsageDefaultValue: "randomly generated by Ginkgo",
		Usage: "The seed used to randomize the spec suite."},
	{KeyPath: "S.RandomizeAllSpecs", Name: "randomize-all", SectionKey: "order", DeprecatedName: "randomizeAllSpecs", DeprecatedDocLink: "changed-command-line-flags",
		Usage: "If set, ginkgo will randomize all specs together.  By default, ginkgo only randomizes the top level Describe, Context and When containers."},

	{KeyPath: "S.FailOnPending", Name: "fail-on-pending", SectionKey: "failure", DeprecatedName: "failOnPending", DeprecatedDocLink: "changed-command-line-flags",
		Usage: "If set, ginkgo will mark the test suite as failed if any specs are pending."},
	{KeyPath: "S.FailFast", Name: "fail-fast", SectionKey: "failure", DeprecatedName: "failFast", DeprecatedDocLink: "changed-command-line-flags",
		Usage: "If set, ginkgo will stop running a test suite after a failure occurs."},
	{KeyPath: "S.FlakeAttempts", Name: "flake-attempts", SectionKey: "failure", UsageDefaultValue: "0 - failed tests are not retried", DeprecatedName: "flakeAttempts", DeprecatedDocLink: "changed-command-line-flags",
		Usage: "Make up to this many attempts to run each spec. If any of the attempts succeed, the suite will not be failed."},

	{KeyPath: "S.DryRun", Name: "dry-run", SectionKey: "debug", DeprecatedName: "dryRun", DeprecatedDocLink: "changed-command-line-flags",
		Usage: "If set, ginkgo will walk the test hierarchy without actually running anything.  Best paired with -v."},
	{KeyPath: "S.PollProgressAfter", Name: "poll-progress-after", SectionKey: "debug", UsageDefaultValue: "0",
		Usage: "Emit node progress reports periodically if node hasn't completed after this duration."},
	{KeyPath: "S.PollProgressInterval", Name: "poll-progress-interval", SectionKey: "debug", UsageDefaultValue: "10s",
		Usage: "The rate at which to emit node progress reports after poll-progress-after has elapsed."},
	{KeyPath: "S.SourceRoots", Name: "source-root", SectionKey: "debug",
		Usage: "The location to look for source code when generating progress reports.  You can pass multiple --source-root flags."},
	{KeyPath: "S.Timeout", Name: "timeout", SectionKey: "debug", UsageDefaultValue: "1h",
		Usage: "Test suite fails if it does not complete within the specified timeout."},
	{KeyPath: "S.GracePeriod", Name: "grace-period", SectionKey: "debug", UsageDefaultValue: "30s",
		Usage: "When interrupted, Ginkgo will wait for GracePeriod for the current running node to exit before moving on to the next one."},
	{KeyPath: "S.OutputInterceptorMode", Name: "output-interceptor-mode", SectionKey: "debug", UsageArgument: "dup, swap, or none",
		Usage: "If set, ginkgo will use the specified output interception strategy when running in parallel.  Defaults to dup on unix and swap on windows."},

	{KeyPath: "S.LabelFilter", Name: "label-filter", SectionKey: "filter", UsageArgument: "expression",
		Usage: "If set, ginkgo will only run specs with labels that match the label-filter.  The passed-in expression can include boolean operations (!, &&, ||, ','), groupings via '()', and regular expressions '/regexp/'.  e.g. '(cat || dog) && !fruit'"},
	{KeyPath: "S.FocusStrings", Name: "focus", SectionKey: "filter",
		Usage: "If set, ginkgo will only run specs that match this regular expression. Can be specified multiple times, values are ORed."},
	{KeyPath: "S.SkipStrings", Name: "skip", SectionKey: "filter",
		Usage: "If set, ginkgo will only run specs that do not match this regular expression. Can be specified multiple times, values are ORed."},
	{KeyPath: "S.FocusFiles", Name: "focus-file", SectionKey: "filter", UsageArgument: "file (regexp) | file:line | file:lineA-lineB | file:line,line,line",
		Usage: "If set, ginkgo will only run specs in matching files. Can be specified multiple times, values are ORed."},
	{KeyPath: "S.SkipFiles", Name: "skip-file", SectionKey: "filter", UsageArgument: "file (regexp) | file:line | file:lineA-lineB | file:line,line,line",
		Usage: "If set, ginkgo will skip specs in matching files. Can be specified multiple times, values are ORed."},

	{KeyPath: "D.RegexScansFilePath", DeprecatedName: "regexScansFilePath", DeprecatedDocLink: "removed--regexscansfilepath", DeprecatedVersion: "2.0.0"},
	{KeyPath: "D.DebugParallel", DeprecatedName: "debug", DeprecatedDocLink: "removed--debug", DeprecatedVersion: "2.0.0"},
	{KeyPath: "D.EmitSpecProgress", DeprecatedName: "progress", SectionKey: "debug",
		DeprecatedVersion: "2.5.0", Usage: ".  The functionality provided by --progress was confusing and is no longer needed.  Use --show-node-events instead to see node entry and exit events included in the timeline of failed and verbose specs.  Or you can run with -vv to always see all node events.  Lastly, --poll-progress-after and the PollProgressAfter decorator now provide a better mechanism for debugging specs that tend to get stuck."},
}

// ParallelConfigFlags provides flags for the Ginkgo test process (not the CLI)
var ParallelConfigFlags = GinkgoFlags{
	{KeyPath: "S.ParallelProcess", Name: "parallel.process", SectionKey: "low-level-parallel", UsageDefaultValue: "1",
		Usage: "This worker process's (one-indexed) process number.  For running specs in parallel."},
	{KeyPath: "S.ParallelTotal", Name: "parallel.total", SectionKey: "low-level-parallel", UsageDefaultValue: "1",
		Usage: "The total number of worker processes.  For running specs in parallel."},
	{KeyPath: "S.ParallelHost", Name: "parallel.host", SectionKey: "low-level-parallel", UsageDefaultValue: "set by Ginkgo CLI",
		Usage: "The address for the server that will synchronize the processes."},
}

// ReporterConfigFlags provides flags for the Ginkgo test process, and CLI
var ReporterConfigFlags = GinkgoFlags{
	{KeyPath: "R.NoColor", Name: "no-color", SectionKey: "output", DeprecatedName: "noColor", DeprecatedDocLink: "changed-command-line-flags",
		Usage: "If set, suppress color output in default reporter."},
	{KeyPath: "R.Verbose", Name: "v", SectionKey: "output",
		Usage: "If set, emits more output including GinkgoWriter contents."},
	{KeyPath: "R.VeryVerbose", Name: "vv", SectionKey: "output",
		Usage: "If set, emits with maximal verbosity - includes skipped and pending tests."},
	{KeyPath: "R.Succinct", Name: "succinct", SectionKey: "output",
		Usage: "If set, default reporter prints out a very succinct report"},
	{KeyPath: "R.FullTrace", Name: "trace", SectionKey: "output",
		Usage: "If set, default reporter prints out the full stack trace when a failure occurs"},
	{KeyPath: "R.ShowNodeEvents", Name: "show-node-events", SectionKey: "output",
		Usage: "If set, default reporter prints node > Enter and < Exit events when specs fail"},

	{KeyPath: "R.JSONReport", Name: "json-report", UsageArgument: "filename.json", SectionKey: "output",
		Usage: "If set, Ginkgo will generate a JSON-formatted test report at the specified location."},
	{KeyPath: "R.JUnitReport", Name: "junit-report", UsageArgument: "filename.xml", SectionKey: "output", DeprecatedName: "reportFile", DeprecatedDocLink: "improved-reporting-infrastructure",
		Usage: "If set, Ginkgo will generate a conformant junit test report in the specified file."},
	{KeyPath: "R.TeamcityReport", Name: "teamcity-report", UsageArgument: "filename", SectionKey: "output",
		Usage: "If set, Ginkgo will generate a Teamcity-formatted test report at the specified location."},

	{KeyPath: "D.SlowSpecThresholdWithFLoatUnits", DeprecatedName: "slowSpecThreshold", DeprecatedDocLink: "changed--slowspecthreshold",
		Usage: "use --slow-spec-threshold instead and pass in a duration string (e.g. '5s', not '5.0')"},
	{KeyPath: "D.NoisyPendings", DeprecatedName: "noisyPendings", DeprecatedDocLink: "removed--noisypendings-and--noisyskippings", DeprecatedVersion: "2.0.0"},
	{KeyPath: "D.NoisySkippings", DeprecatedName: "noisySkippings", DeprecatedDocLink: "removed--noisypendings-and--noisyskippings", DeprecatedVersion: "2.0.0"},
	{KeyPath: "D.SlowSpecThreshold", DeprecatedName: "slow-spec-threshold", SectionKey: "output", Usage: "--slow-spec-threshold has been deprecated and will be removed in a future version of Ginkgo.  This feature has proved to be more noisy than useful.  You can use --poll-progress-after, instead, to get more actionable feedback about potentially slow specs and understand where they might be getting stuck.", DeprecatedVersion: "2.5.0"},
	{KeyPath: "D.AlwaysEmitGinkgoWriter", DeprecatedName: "always-emit-ginkgo-writer", SectionKey: "output", Usage: " - use -v instead, or one of Ginkgo's machine-readable report formats to get GinkgoWriter output for passing specs."},
}

// BuildTestSuiteFlagSet attaches to the CommandLine flagset and provides flags for the Ginkgo test process
func BuildTestSuiteFlagSet(suiteConfig *SuiteConfig, reporterConfig *ReporterConfig) (GinkgoFlagSet, error) {
	flags := SuiteConfigFlags.CopyAppend(ParallelConfigFlags...).CopyAppend(ReporterConfigFlags...)
	flags = flags.WithPrefix("ginkgo")
	bindings := map[string]interface{}{
		"S": suiteConfig,
		"R": reporterConfig,
		"D": &deprecatedConfig{},
	}
	extraGoFlagsSection := GinkgoFlagSection{Style: "{{gray}}", Heading: "Go test flags"}

	return NewAttachedGinkgoFlagSet(flag.CommandLine, flags, bindings, FlagSections, extraGoFlagsSection)
}

// VetConfig validates that the Ginkgo test process' configuration is sound
func VetConfig(flagSet GinkgoFlagSet, suiteConfig SuiteConfig, reporterConfig ReporterConfig) []error {
	errors := []error{}

	if flagSet.WasSet("count") || flagSet.WasSet("test.count") {
		flag := flagSet.Lookup("count")
		if flag == nil {
			flag = flagSet.Lookup("test.count")
		}
		count, err := strconv.Atoi(flag.Value.String())
		if err != nil || count != 1 {
			errors = append(errors, GinkgoErrors.InvalidGoFlagCount())
		}
	}

	if flagSet.WasSet("parallel") || flagSet.WasSet("test.parallel") {
		errors = append(errors, GinkgoErrors.InvalidGoFlagParallel())
	}

	if suiteConfig.ParallelTotal < 1 {
		errors = append(errors, GinkgoErrors.InvalidParallelTotalConfiguration())
	}

	if suiteConfig.ParallelProcess > suiteConfig.ParallelTotal || suiteConfig.ParallelProcess < 1 {
		errors = append(errors, GinkgoErrors.InvalidParallelProcessConfiguration())
	}

	if suiteConfig.ParallelTotal > 1 && suiteConfig.ParallelHost == "" {
		errors = append(errors, GinkgoErrors.MissingParallelHostConfiguration())
	}

	if suiteConfig.DryRun && suiteConfig.ParallelTotal > 1 {
		errors = append(errors, GinkgoErrors.DryRunInParallelConfiguration())
	}

	if suiteConfig.GracePeriod <= 0 {
		errors = append(errors, GinkgoErrors.GracePeriodCannotBeZero())
	}

	if len(suiteConfig.FocusFiles) > 0 {
		_, err := ParseFileFilters(suiteConfig.FocusFiles)
		if err != nil {
			errors = append(errors, err)
		}
	}

	if len(suiteConfig.SkipFiles) > 0 {
		_, err := ParseFileFilters(suiteConfig.SkipFiles)
		if err != nil {
			errors = append(errors, err)
		}
	}

	if suiteConfig.LabelFilter != "" {
		_, err := ParseLabelFilter(suiteConfig.LabelFilter)
		if err != nil {
			errors = append(errors, err)
		}
	}

	switch strings.ToLower(suiteConfig.OutputInterceptorMode) {
	case "", "dup", "swap", "none":
	default:
		errors = append(errors, GinkgoErrors.InvalidOutputInterceptorModeConfiguration(suiteConfig.OutputInterceptorMode))
	}

	numVerbosity := 0
	for _, v := range []bool{reporterConfig.Succinct, reporterConfig.Verbose, reporterConfig.VeryVerbose} {
		if v {
			numVerbosity++
		}
	}
	if numVerbosity > 1 {
		errors = append(errors, GinkgoErrors.ConflictingVerbosityConfiguration())
	}

	return errors
}

// GinkgoCLISharedFlags provides flags shared by the Ginkgo CLI's build, watch, and run commands
var GinkgoCLISharedFlags = GinkgoFlags{
	{KeyPath: "C.Recurse", Name: "r", SectionKey: "multiple-suites",
		Usage: "If set, ginkgo finds and runs test suites under the current directory recursively."},
	{KeyPath: "C.SkipPackage", Name: "skip-package", SectionKey: "multiple-suites", DeprecatedName: "skipPackage", DeprecatedDocLink: "changed-command-line-flags",
		UsageArgument: "comma-separated list of packages",
		Usage:         "A comma-separated list of package names to be skipped.  If any part of the package's path matches, that package is ignored."},
	{KeyPath: "C.RequireSuite", Name: "require-suite", SectionKey: "failure", DeprecatedName: "requireSuite", DeprecatedDocLink: "changed-command-line-flags",
		Usage: "If set, Ginkgo fails if there are ginkgo tests in a directory but no invocation of RunSpecs."},
	{KeyPath: "C.NumCompilers", Name: "compilers", SectionKey: "multiple-suites", UsageDefaultValue: "0 (will autodetect)",
		Usage: "When running multiple packages, the number of concurrent compilations to perform."},
}

// GinkgoCLIRunAndWatchFlags provides flags shared by the Ginkgo CLI's build and watch commands (but not run)
var GinkgoCLIRunAndWatchFlags = GinkgoFlags{
	{KeyPath: "C.Procs", Name: "procs", SectionKey: "parallel", UsageDefaultValue: "1 (run in series)",
		Usage: "The number of parallel test nodes to run."},
	{KeyPath: "C.Procs", Name: "nodes", SectionKey: "parallel", UsageDefaultValue: "1 (run in series)",
		Usage: "--nodes is an alias for --procs"},
	{KeyPath: "C.Parallel", Name: "p", SectionKey: "parallel",
		Usage: "If set, ginkgo will run in parallel with an auto-detected number of nodes."},
	{KeyPath: "C.AfterRunHook", Name: "after-run-hook", SectionKey: "misc", DeprecatedName: "afterSuiteHook", DeprecatedDocLink: "changed-command-line-flags",
		Usage: "Command to run when a test suite completes."},
	{KeyPath: "C.OutputDir", Name: "output-dir", SectionKey: "output", UsageArgument: "directory", DeprecatedName: "outputdir", DeprecatedDocLink: "improved-profiling-support",
		Usage: "A location to place all generated profiles and reports."},
	{KeyPath: "C.KeepSeparateCoverprofiles", Name: "keep-separate-coverprofiles", SectionKey: "code-and-coverage-analysis",
		Usage: "If set, Ginkgo does not merge coverprofiles into one monolithic coverprofile.  The coverprofiles will remain in their respective package directories or in -output-dir if set."},
	{KeyPath: "C.KeepSeparateReports", Name: "keep-separate-reports", SectionKey: "output",
		Usage: "If set, Ginkgo does not merge per-suite reports (e.g. -json-report) into one monolithic report for the entire testrun.  The reports will remain in their respective package directories or in -output-dir if set."},

	{KeyPath: "D.Stream", DeprecatedName: "stream", DeprecatedDocLink: "removed--stream", DeprecatedVersion: "2.0.0"},
	{KeyPath: "D.Notify", DeprecatedName: "notify", DeprecatedDocLink: "removed--notify", DeprecatedVersion: "2.0.0"},
}

// GinkgoCLIRunFlags provides flags for Ginkgo CLI's run command that aren't shared by any other commands
var GinkgoCLIRunFlags = GinkgoFlags{
	{KeyPath: "C.KeepGoing", Name: "keep-going", SectionKey: "multiple-suites", DeprecatedName: "keepGoing", DeprecatedDocLink: "changed-command-line-flags",
		Usage: "If set, failures from earlier test suites do not prevent later test suites from running."},
	{KeyPath: "C.UntilItFails", Name: "until-it-fails", SectionKey: "debug", DeprecatedName: "untilItFails", DeprecatedDocLink: "changed-command-line-flags",
		Usage: "If set, ginkgo will keep rerunning test suites until a failure occurs."},
	{KeyPath: "C.Repeat", Name: "repeat", SectionKey: "debug", UsageArgument: "n", UsageDefaultValue: "0 - i.e. no repetition, run only once",
		Usage: "The number of times to re-run a test-suite.  Useful for debugging flaky tests.  If set to N the suite will be run N+1 times and will be required to pass each time."},
	{KeyPath: "C.RandomizeSuites", Name: "randomize-suites", SectionKey: "order", DeprecatedName: "randomizeSuites", DeprecatedDocLink: "changed-command-line-flags",
		Usage: "If set, ginkgo will randomize the order in which test suites run."},
}

// GinkgoCLIRunFlags provides flags for Ginkgo CLI's watch command that aren't shared by any other commands
var GinkgoCLIWatchFlags = GinkgoFlags{
	{KeyPath: "C.Depth", Name: "depth", SectionKey: "watch",
		Usage: "Ginkgo will watch dependencies down to this depth in the dependency tree."},
	{KeyPath: "C.WatchRegExp", Name: "watch-regexp", SectionKey: "watch", DeprecatedName: "watchRegExp", DeprecatedDocLink: "changed-command-line-flags",
		UsageArgument:     "Regular Expression",
		UsageDefaultValue: `\.go$`,
		Usage:             "Only files matching this regular expression will be watched for changes."},
}

// GoBuildFlags provides flags for the Ginkgo CLI build, run, and watch commands that capture go's build-time flags.  These are passed to go test -c by the ginkgo CLI
var GoBuildFlags = GinkgoFlags{
	{KeyPath: "Go.Race", Name: "race", SectionKey: "code-and-coverage-analysis",
		Usage: "enable data race detection. Supported only on linux/amd64, freebsd/amd64, darwin/amd64, windows/amd64, linux/ppc64le and linux/arm64 (only for 48-bit VMA)."},
	{KeyPath: "Go.Vet", Name: "vet", UsageArgument: "list", SectionKey: "code-and-coverage-analysis",
		Usage: `Configure the invocation of "go vet" during "go test" to use the comma-separated list of vet checks.  If list is empty, "go test" runs "go vet" with a curated list of checks believed to be always worth addressing.  If list is "off", "go test" does not run "go vet" at all.  Available checks can be found by running 'go doc cmd/vet'`},
	{KeyPath: "Go.Cover", Name: "cover", SectionKey: "code-and-coverage-analysis",
		Usage: "Enable coverage analysis.	Note that because coverage works by annotating the source code before compilation, compilation and test failures with coverage enabled may report line numbers that don't correspond to the original sources."},
	{KeyPath: "Go.CoverMode", Name: "covermode", UsageArgument: "set,count,atomic", SectionKey: "code-and-coverage-analysis",
		Usage: `Set the mode for coverage analysis for the package[s] being tested. 'set': does this statement run? 'count': how many times does this statement run? 'atomic': like count, but correct in multithreaded tests and more expensive (must use atomic with -race). Sets -cover`},
	{KeyPath: "Go.CoverPkg", Name: "coverpkg", UsageArgument: "pattern1,pattern2,pattern3", SectionKey: "code-and-coverage-analysis",
		Usage: "Apply coverage analysis in each test to packages matching the patterns. 	The default is for each test to analyze only the package being tested. See 'go help packages' for a description of package patterns. Sets -cover."},

	{KeyPath: "Go.A", Name: "a", SectionKey: "go-build",
		Usage: "force rebuilding of packages that are already up-to-date."},
	{KeyPath: "Go.ASMFlags", Name: "asmflags", UsageArgument: "'[pattern=]arg list'", SectionKey: "go-build",
		Usage: "arguments to pass on each go tool asm invocation."},
	{KeyPath: "Go.BuildMode", Name: "buildmode", UsageArgument: "mode", SectionKey: "go-build",
		Usage: "build mode to use. See 'go help buildmode' for more."},
	{KeyPath: "Go.Compiler", Name: "compiler", UsageArgument: "name", SectionKey: "go-build",
		Usage: "name of compiler to use, as in runtime.Compiler (gccgo or gc)."},
	{KeyPath: "Go.GCCGoFlags", Name: "gccgoflags", UsageArgument: "'[pattern=]arg list'", SectionKey: "go-build",
		Usage: "arguments to pass on each gccgo compiler/linker invocation."},
	{KeyPath: "Go.GCFlags", Name: "gcflags", UsageArgument: "'[pattern=]arg list'", SectionKey: "go-build",
		Usage: "arguments to pass on each go tool compile invocation."},
	{KeyPath: "Go.InstallSuffix", Name: "installsuffix", SectionKey: "go-build",
		Usage: "a suffix to use in the name of the package installation directory, in order to keep output separate from default builds. If using the -race flag, the install suffix is automatically set to raceor, if set explicitly, has _race appended to it. Likewise for the -msan flag.  Using a -buildmode option that requires non-default compile flags has a similar effect."},
	{KeyPath: "Go.LDFlags", Name: "ldflags", UsageArgument: "'[pattern=]arg list'", SectionKey: "go-build",
		Usage: "arguments to pass on each go tool link invocation."},
	{KeyPath: "Go.LinkShared", Name: "linkshared", SectionKey: "go-build",
		Usage: "build code that will be linked against shared libraries previously created with -buildmode=shared."},
	{KeyPath: "Go.Mod", Name: "mod", UsageArgument: "mode (readonly, vendor, or mod)", SectionKey: "go-build",
		Usage: "module download mode to use: readonly, vendor, or mod.  See 'go help modules' for more."},
	{KeyPath: "Go.ModCacheRW", Name: "modcacherw", SectionKey: "go-build",
		Usage: "leave newly-created directories in the module cache read-write instead of making them read-only."},
	{KeyPath: "Go.ModFile", Name: "modfile", UsageArgument: "file", SectionKey: "go-build",
		Usage: `in module aware mode, read (and possibly write) an alternate go.mod file instead of the one in the module root directory. A file named go.mod must still be present in order to determine the module root directory, but it is not accessed. When -modfile is specified, an alternate go.sum file is also used: its path is derived from the -modfile flag by trimming the ".mod" extension and appending ".sum".`},
	{KeyPath: "Go.MSan", Name: "msan", SectionKey: "go-build",
		Usage: "enable interoperation with memory sanitizer. Supported only on linux/amd64, linux/arm64 and only with Clang/LLVM as the host C compiler. On linux/arm64, pie build mode will be used."},
	{KeyPath: "Go.N", Name: "n", SectionKey: "go-build",
		Usage: "print the commands but do not run them."},
	{KeyPath: "Go.PkgDir", Name: "pkgdir", UsageArgument: "dir", SectionKey: "go-build",
		Usage: "install and load all packages from dir instead of the usual locations. For example, when building with a non-standard configuration, use -pkgdir to keep generated packages in a separate location."},
	{KeyPath: "Go.Tags", Name: "tags", UsageArgument: "tag,list", SectionKey: "go-build",
		Usage: "a comma-separated list of build tags to consider satisfied during the build. For more information about build tags, see the description of build constraints in the documentation for the go/build package. (Earlier versions of Go used a space-separated list, and that form is deprecated but still recognized.)"},
	{KeyPath: "Go.TrimPath", Name: "trimpath", SectionKey: "go-build",
		Usage: `remove all file system paths from the resulting executable. Instead of absolute file system paths, the recorded file names will begin with either "go" (for the standard library), or a module path@version (when using modules), or a plain import path (when using GOPATH).`},
	{KeyPath: "Go.ToolExec", Name: "toolexec", UsageArgument: "'cmd args'", SectionKey: "go-build",
		Usage: "a program to use to invoke toolchain programs like vet and asm. For example, instead of running asm, the go command will run cmd args /path/to/asm <arguments for asm>'."},
	{KeyPath: "Go.Work", Name: "work", SectionKey: "go-build",
		Usage: "print the name of the temporary work directory and do not delete it when exiting."},
	{KeyPath: "Go.X", Name: "x", SectionKey: "go-build",
		Usage: "print the commands."},
}

// GoRunFlags provides flags for the Ginkgo CLI  run, and watch commands that capture go's run-time flags.  These are passed to the compiled test binary by the ginkgo CLI
var GoRunFlags = GinkgoFlags{
	{KeyPath: "Go.CoverProfile", Name: "coverprofile", UsageArgument: "file", SectionKey: "code-and-coverage-analysis",
		Usage: `Write a coverage profile to the file after all tests have passed. Sets -cover.`},
	{KeyPath: "Go.BlockProfile", Name: "blockprofile", UsageArgument: "file", SectionKey: "performance-analysis",
		Usage: `Write a goroutine blocking profile to the specified file when all tests are complete. Preserves test binary.`},
	{KeyPath: "Go.BlockProfileRate", Name: "blockprofilerate", UsageArgument: "rate", SectionKey: "performance-analysis",
		Usage: `Control the detail provided in goroutine blocking profiles by calling runtime.SetBlockProfileRate with rate. See 'go doc runtime.SetBlockProfileRate'. The profiler aims to sample, on average, one blocking event every n nanoseconds the program spends blocked. By default, if -test.blockprofile is set without this flag, all blocking events are recorded, equivalent to -test.blockprofilerate=1.`},
	{KeyPath: "Go.CPUProfile", Name: "cpuprofile", UsageArgument: "file", SectionKey: "performance-analysis",
		Usage: `Write a CPU profile to the specified file before exiting. Preserves test binary.`},
	{KeyPath: "Go.MemProfile", Name: "memprofile", UsageArgument: "file", SectionKey: "performance-analysis",
		Usage: `Write an allocation profile to the file after all tests have passed. Preserves test binary.`},
	{KeyPath: "Go.MemProfileRate", Name: "memprofilerate", UsageArgument: "rate", SectionKey: "performance-analysis",
		Usage: `Enable more precise (and expensive) memory allocation profiles by setting runtime.MemProfileRate. See 'go doc runtime.MemProfileRate'. To profile all memory allocations, use -test.memprofilerate=1.`},
	{KeyPath: "Go.MutexProfile", Name: "mutexprofile", UsageArgument: "file", SectionKey: "performance-analysis",
		Usage: `Write a mutex contention profile to the specified file when all tests are complete. Preserves test binary.`},
	{KeyPath: "Go.MutexProfileFraction", Name: "mutexprofilefraction", UsageArgument: "n", SectionKey: "performance-analysis",
		Usage: `if >= 0, calls runtime.SetMutexProfileFraction()	Sample 1 in n stack traces of goroutines holding a contended mutex.`},
	{KeyPath: "Go.Trace", Name: "execution-trace", UsageArgument: "file", ExportAs: "trace", SectionKey: "performance-analysis",
		Usage: `Write an execution trace to the specified file before exiting.`},
}

// VetAndInitializeCLIAndGoConfig validates that the Ginkgo CLI's configuration is sound
// It returns a potentially mutated copy of the config that rationalizes the configuration to ensure consistency for downstream consumers
func VetAndInitializeCLIAndGoConfig(cliConfig CLIConfig, goFlagsConfig GoFlagsConfig) (CLIConfig, GoFlagsConfig, []error) {
	errors := []error{}

	if cliConfig.Repeat > 0 && cliConfig.UntilItFails {
		errors = append(errors, GinkgoErrors.BothRepeatAndUntilItFails())
	}

	//initialize the output directory
	if cliConfig.OutputDir != "" {
		err := os.MkdirAll(cliConfig.OutputDir, 0777)
		if err != nil {
			errors = append(errors, err)
		}
	}

	//ensure cover mode is configured appropriately
	if goFlagsConfig.CoverMode != "" || goFlagsConfig.CoverPkg != "" || goFlagsConfig.CoverProfile != "" {
		goFlagsConfig.Cover = true
	}
	if goFlagsConfig.Cover && goFlagsConfig.CoverProfile == "" {
		goFlagsConfig.CoverProfile = "coverprofile.out"
	}

	return cliConfig, goFlagsConfig, errors
}

// GenerateGoTestCompileArgs is used by the Ginkgo CLI to generate command line arguments to pass to the go test -c command when compiling the test
func GenerateGoTestCompileArgs(goFlagsConfig GoFlagsConfig, destination string, packageToBuild string, pathToInvocationPath string) ([]string, error) {
	// if the user has set the CoverProfile run-time flag make sure to set the build-time cover flag to make sure
	// the built test binary can generate a coverprofile
	if goFlagsConfig.CoverProfile != "" {
		goFlagsConfig.Cover = true
	}

	if goFlagsConfig.CoverPkg != "" {
		coverPkgs := strings.Split(goFlagsConfig.CoverPkg, ",")
		adjustedCoverPkgs := make([]string, len(coverPkgs))
		for i, coverPkg := range coverPkgs {
			coverPkg = strings.Trim(coverPkg, " ")
			if strings.HasPrefix(coverPkg, "./") {
				// this is a relative coverPkg - we need to reroot it
				adjustedCoverPkgs[i] = "./" + filepath.Join(pathToInvocationPath, strings.TrimPrefix(coverPkg, "./"))
			} else {
				// this is a package name - don't touch it
				adjustedCoverPkgs[i] = coverPkg
			}
		}
		goFlagsConfig.CoverPkg = strings.Join(adjustedCoverPkgs, ",")
	}

	args := []string{"test", "-c", "-o", destination, packageToBuild}
	goArgs, err := GenerateFlagArgs(
		GoBuildFlags,
		map[string]interface{}{
			"Go": &goFlagsConfig,
		},
	)

	if err != nil {
		return []string{}, err
	}
	args = append(args, goArgs...)
	return args, nil
}

// GenerateGinkgoTestRunArgs is used by the Ginkgo CLI to generate command line arguments to pass to the compiled Ginkgo test binary
func GenerateGinkgoTestRunArgs(suiteConfig SuiteConfig, reporterConfig ReporterConfig, goFlagsConfig GoFlagsConfig) ([]string, error) {
	var flags GinkgoFlags
	flags = SuiteConfigFlags.WithPrefix("ginkgo")
	flags = flags.CopyAppend(ParallelConfigFlags.WithPrefix("ginkgo")...)
	flags = flags.CopyAppend(ReporterConfigFlags.WithPrefix("ginkgo")...)
	flags = flags.CopyAppend(GoRunFlags.WithPrefix("test")...)
	bindings := map[string]interface{}{
		"S":  &suiteConfig,
		"R":  &reporterConfig,
		"Go": &goFlagsConfig,
	}

	return GenerateFlagArgs(flags, bindings)
}

// GenerateGoTestRunArgs is used by the Ginkgo CLI to generate command line arguments to pass to the compiled non-Ginkgo test binary
func GenerateGoTestRunArgs(goFlagsConfig GoFlagsConfig) ([]string, error) {
	flags := GoRunFlags.WithPrefix("test")
	bindings := map[string]interface{}{
		"Go": &goFlagsConfig,
	}

	args, err := GenerateFlagArgs(flags, bindings)
	if err != nil {
		return args, err
	}
	args = append(args, "--test.v")
	return args, nil
}

// BuildRunCommandFlagSet builds the FlagSet for the `ginkgo run` command
func BuildRunCommandFlagSet(suiteConfig *SuiteConfig, reporterConfig *ReporterConfig, cliConfig *CLIConfig, goFlagsConfig *GoFlagsConfig) (GinkgoFlagSet, error) {
	flags := SuiteConfigFlags
	flags = flags.CopyAppend(ReporterConfigFlags...)
	flags = flags.CopyAppend(GinkgoCLISharedFlags...)
	flags = flags.CopyAppend(GinkgoCLIRunAndWatchFlags...)
	flags = flags.CopyAppend(GinkgoCLIRunFlags...)
	flags = flags.CopyAppend(GoBuildFlags...)
	flags = flags.CopyAppend(GoRunFlags...)

	bindings := map[string]interface{}{
		"S":  suiteConfig,
		"R":  reporterConfig,
		"C":  cliConfig,
		"Go": goFlagsConfig,
		"D":  &deprecatedConfig{},
	}

	return NewGinkgoFlagSet(flags, bindings, FlagSections)
}

// BuildWatchCommandFlagSet builds the FlagSet for the `ginkgo watch` command
func BuildWatchCommandFlagSet(suiteConfig *SuiteConfig, reporterConfig *ReporterConfig, cliConfig *CLIConfig, goFlagsConfig *GoFlagsConfig) (GinkgoFlagSet, error) {
	flags := SuiteConfigFlags
	flags = flags.CopyAppend(ReporterConfigFlags...)
	flags = flags.CopyAppend(GinkgoCLISharedFlags...)
	flags = flags.CopyAppend(GinkgoCLIRunAndWatchFlags...)
	flags = flags.CopyAppend(GinkgoCLIWatchFlags...)
	flags = flags.CopyAppend(GoBuildFlags...)
	flags = flags.CopyAppend(GoRunFlags...)

	bindings := map[string]interface{}{
		"S":  suiteConfig,
		"R":  reporterConfig,
		"C":  cliConfig,
		"Go": goFlagsConfig,
		"D":  &deprecatedConfig{},
	}

	return NewGinkgoFlagSet(flags, bindings, FlagSections)
}

// BuildBuildCommandFlagSet builds the FlagSet for the `ginkgo build` command
func BuildBuildCommandFlagSet(cliConfig *CLIConfig, goFlagsConfig *GoFlagsConfig) (GinkgoFlagSet, error) {
	flags := GinkgoCLISharedFlags
	flags = flags.CopyAppend(GoBuildFlags...)

	bindings := map[string]interface{}{
		"C":  cliConfig,
		"Go": goFlagsConfig,
		"D":  &deprecatedConfig{},
	}

	flagSections := make(GinkgoFlagSections, len(FlagSections))
	copy(flagSections, FlagSections)
	for i := range flagSections {
		if flagSections[i].Key == "multiple-suites" {
			flagSections[i].Heading = "Building Multiple Suites"
		}
		if flagSections[i].Key == "go-build" {
			flagSections[i] = GinkgoFlagSection{Key: "go-build", Style: "{{/}}", Heading: "Go Build Flags",
				Description: "These flags are inherited from go build."}
		}
	}

	return NewGinkgoFlagSet(flags, bindings, flagSections)
}

func BuildLabelsCommandFlagSet(cliConfig *CLIConfig) (GinkgoFlagSet, error) {
	flags := GinkgoCLISharedFlags.SubsetWithNames("r", "skip-package")

	bindings := map[string]interface{}{
		"C": cliConfig,
	}

	flagSections := make(GinkgoFlagSections, len(FlagSections))
	copy(flagSections, FlagSections)
	for i := range flagSections {
		if flagSections[i].Key == "multiple-suites" {
			flagSections[i].Heading = "Fetching Labels from Multiple Suites"
		}
	}

	return NewGinkgoFlagSet(flags, bindings, flagSections)
}