feat: some config fields are not loading (#185)

* feat: add writing policies to config * feat: use exlucsively config from config file * feat: add mapstructure tags * feat: fix worker flags * chore: remove debug logs * feat: add mapstructure to flags * feat: add init command * feat: update detect presence * feat: update test init command * fix: add mapstructure to missed settings * fix: add mapstructure to patterns * fix: rebuild snapshot
Bearer · Dec 2, 2022 · 0174c2c · 0174c2c
1 parent ffcb0b1
commit 0174c2c
Show file tree

Hide file tree

Showing 10 changed files with 351 additions and 85 deletions.
diff --git a/integration/flags/.snapshots/TestInitCommand-init b/integration/flags/.snapshots/TestInitCommand-init
diff --git a/pkg/commands/init.go b/pkg/commands/init.go
@@ -27,6 +27,7 @@ func NewInitCommand() *cobra.Command {
 				return err
 			}
 			viper.Set(settings.CustomDetectorKey, globalSettings.CustomDetector)
+			viper.Set(settings.PoliciesKey, globalSettings.Policies)
 
 			viper.SetConfigFile("./curio.yml")
 			err = viper.WriteConfig()

diff --git a/pkg/commands/process/settings/settings.go b/pkg/commands/process/settings/settings.go
@@ -13,12 +13,12 @@ import (
 )
 
 type Config struct {
-	Worker         flag.WorkerOptions `json:"worker" yaml:"worker"`
-	Scan           flag.ScanOptions   `json:"scan" yaml:"scan"`
-	Report         flag.ReportOptions `json:"report" yaml:"report"`
-	CustomDetector map[string]Rule    `json:"custom_detector" yaml:"custom_detector"`
-	Policies       map[string]*Policy `json:"policies" yaml:"policies"`
-	Target         string             `json:"target" yaml:"target"`
+	Worker         flag.WorkerOptions `mapstructure:"worker" json:"worker" yaml:"worker"`
+	Scan           flag.ScanOptions   `mapstructure:"scan" json:"scan" yaml:"scan"`
+	Report         flag.ReportOptions `mapstructure:"report" json:"report" yaml:"report"`
+	CustomDetector map[string]Rule    `mapstructure:"custom_detector" json:"custom_detector" yaml:"custom_detector"`
+	Policies       map[string]*Policy `mapstructure:"policies" json:"policies" yaml:"policies"`
+	Target         string             `mapstructure:"target" json:"target" yaml:"target"`
 }
 
 type PolicyLevel string
@@ -31,18 +31,18 @@ var LevelLow = "low"
 type Modules []*PolicyModule
 
 type Policy struct {
-	Query       string
-	Id          string
-	Name        string
-	Description string
-	Level       PolicyLevel
-	Modules     Modules
+	Query       string      `mapstructure:"query" json:"query" yaml:"query"`
+	Id          string      `mapstructure:"id" json:"id" yaml:"id"`
+	Name        string      `mapstructure:"name" json:"name" yaml:"name"`
+	Description string      `mapstructure:"description" json:"description" yaml:"description"`
+	Level       PolicyLevel `mapstructure:"level" json:"level" yaml:"level"`
+	Modules     Modules     `mapstructure:"modules" json:"modules" yaml:"modules"`
 }
 
 type PolicyModule struct {
-	Path    string `yaml:"path,omitempty"`
-	Name    string
-	Content string
+	Path    string `mapstructure:"path" json:"path,omitempty" yaml:"path,omitempty"`
+	Name    string `mapstructure:"name" json:"name" yaml:"name"`
+	Content string `mapstructure:"content" json:"content" yaml:"content"`
 }
 
 func (modules Modules) ToRegoModules() (output []rego.Module) {
@@ -66,30 +66,30 @@ type RulePattern struct {
 }
 
 type Rule struct {
-	Disabled       bool
-	Type           string
-	Languages      []string
-	Patterns       []RulePattern
-	ParamParenting bool `yaml:"param_parenting"`
-	Processors     []Processor
-
-	RootSingularize bool `yaml:"root_singularize"`
-	RootLowercase   bool `yaml:"root_lowercase"`
-
-	Metavars map[string]MetaVar
-	Stored   bool
-	DetectPresence bool `yaml:"detect_presence"`
+	Disabled       bool          `mapstructure:"disabled" json:"disabled" yaml:"disabled"`
+	Type           string        `mapstructure:"type" json:"type" yaml:"type"`
+	Languages      []string      `mapstructure:"languages" json:"languages" yaml:"languages"`
+	ParamParenting bool          `mapstructure:"param_parenting" json:"param_parenting" yaml:"param_parenting"`
+	Processors     []Processor   `mapstructure:"processors" json:"processors" yaml:"processors"`
+	Patterns       []RulePattern `mapstructure:"patterns" json:"patterns" yaml:"patterns"`
+
+	RootSingularize bool `mapstructure:"root_singularize" yaml:"root_singularize" `
+	RootLowercase   bool `mapstructure:"root_lowercase" yaml:"root_lowercase"`
+
+	Metavars       map[string]MetaVar `mapstructure:"metavars" json:"metavars" yaml:"metavars"`
+	Stored         bool               `mapstructure:"stored" json:"stored" yaml:"stored"`
+	DetectPresence bool               `mapstructure:"detect_presence" json:"detect_presence" yaml:"detect_presence"`
 }
 
 type Processor struct {
-	Query   string
-	Modules Modules
+	Query   string  `mapstructure:"query" json:"query" yaml:"query"`
+	Modules Modules `mapstructure:"modules" json:"modules" yaml:"modules"`
 }
 
 type MetaVar struct {
-	Input  string
-	Output int
-	Regex  string
+	Input  string `mapstructure:"input" json:"input" yaml:"input"`
+	Output int    `mapstructure:"output" json:"output" yaml:"output"`
+	Regex  string `mapstructure:"regex" json:"regex" yaml:"regex"`
 }
 
 //go:embed custom_detector.yml
@@ -108,12 +108,14 @@ var CustomDetectorKey string = "scan.custom_detector"
 var PoliciesKey string = "scan.policies"
 
 func FromOptions(opts flag.Options) (Config, error) {
-	rules := DefaultCustomDetector()
+	var rules map[string]Rule
 	if viper.IsSet(CustomDetectorKey) {
 		err := viper.UnmarshalKey(CustomDetectorKey, &rules)
 		if err != nil {
 			return Config{}, err
 		}
+	} else {
+		rules = DefaultCustomDetector()
 	}
 
 	for _, customDetector := range rules {
@@ -131,12 +133,14 @@ func FromOptions(opts flag.Options) (Config, error) {
 		}
 	}
 
-	policies := DefaultPolicies()
+	var policies map[string]*Policy
 	if viper.IsSet(PoliciesKey) {
 		err := viper.UnmarshalKey(PoliciesKey, &policies)
 		if err != nil {
 			return Config{}, err
 		}
+	} else {
+		policies = DefaultPolicies()
 	}
 
 	for key := range policies {

diff --git a/pkg/commands/scan.go b/pkg/commands/scan.go
@@ -89,8 +89,8 @@ func NewScanCommand() *cobra.Command {
 }
 
 func readConfig(configFile string) error {
-	viper.SetConfigFile(configFile)
 	viper.SetConfigType("yaml")
+	viper.SetConfigFile(configFile)
 	if err := viper.ReadInConfig(); err != nil {
 		if _, ok := err.(viper.ConfigFileNotFoundError); ok {
 			return nil

diff --git a/pkg/flag/policy_flags.go b/pkg/flag/policy_flags.go
@@ -21,8 +21,8 @@ type PolicyFlagGroup struct {
 }
 
 type PolicyOptions struct {
-	SkipPolicy map[string]bool `json:"skip_policy" yaml:"skip_policy"`
-	OnlyPolicy map[string]bool `json:"only_policy" yaml:"only_policy"`
+	SkipPolicy map[string]bool `mapstructure:"skip-policy" json:"skip-policy" yaml:"skip-policy"`
+	OnlyPolicy map[string]bool `mapstructure:"only-policy" json:"only-policy" yaml:"only-policy"`
 }
 
 func NewPolicyFlagGroup() *PolicyFlagGroup {

diff --git a/pkg/flag/process_flags.go b/pkg/flag/process_flags.go
@@ -15,7 +15,7 @@ type ProcessFlagGroup struct {
 }
 
 type ProcessOptions struct {
-	Port string
+	Port string `mapstructure:"port" json:"port" yaml:"port"`
 }
 
 func NewProcessGroup() *ProcessFlagGroup {

diff --git a/pkg/flag/report_flags.go b/pkg/flag/report_flags.go
@@ -41,9 +41,9 @@ type ReportFlagGroup struct {
 }
 
 type ReportOptions struct {
-	Format string
-	Report string
-	Output string
+	Format string `mapstructure:"format" json:"format" yaml:"format"`
+	Report string `mapstructure:"report" json:"report" yaml:"report"`
+	Output string `mapstructure:"output" json:"output" yaml:"output"`
 }
 
 func NewReportFlagGroup() *ReportFlagGroup {

diff --git a/pkg/flag/repository_flags.go b/pkg/flag/repository_flags.go
@@ -28,9 +28,9 @@ type RepoFlagGroup struct {
 }
 
 type RepoOptions struct {
-	RepoBranch string
-	RepoCommit string
-	RepoTag    string
+	RepoBranch string `mapstructure:"branch" json:"branch" yaml:"branch"`
+	RepoCommit string `mapstructure:"commit" json:"commit" yaml:"commit"`
+	RepoTag    string `mapstructure:"tag" json:"tag" yaml:"tag"`
 }
 
 func NewRepoFlagGroup() *RepoFlagGroup {

diff --git a/pkg/flag/scan_flags.go b/pkg/flag/scan_flags.go
@@ -67,14 +67,14 @@ type ScanFlagGroup struct {
 }
 
 type ScanOptions struct {
-	Target                  string        `json:"target" yaml:"target"`
-	SkipPath                []string      `json:"skip_path" yaml:"skip_path"`
-	Debug                   bool          `json:"debug" yaml:"debug"`
-	DisableDomainResolution bool          `json:"disable_domain_resolution" yaml:"disable_domain_resolution"`
-	DomainResolutionTimeout time.Duration `json:"domain_resolution_timeout" yaml:"domain_resolution_timeout"`
-	InternalDomains         []string      `json:"internal_domains" yaml:"internal_domains"`
-	Context                 Context       `json:"context" yaml:"context"`
-	Quiet                   bool          `json:"quiet" yaml:"quiet"`
+	Target                  string        `mapstructure:"target" json:"target" yaml:"target"`
+	SkipPath                []string      `mapstructure:"skip-path" json:"skip-path" yaml:"skip-path"`
+	Debug                   bool          `mapstructure:"debug" json:"debug" yaml:"debug"`
+	DisableDomainResolution bool          `mapstructure:"disable-domain-resolution" json:"disable-domain-resolution" yaml:"disable-domain-resolution"`
+	DomainResolutionTimeout time.Duration `mapstructure:"domain-resolution-timeout" json:"domain-resolution-timeout" yaml:"domain-resolution-timeout"`
+	InternalDomains         []string      `mapstructure:"internal-domains" json:"internal-domains" yaml:"internal-domains"`
+	Context                 Context       `mapstructure:"context" json:"context" yaml:"context"`
+	Quiet                   bool          `mapstructure:"quiet" json:"quiet" yaml:"quiet"`
 }
 
 func NewScanFlagGroup() *ScanFlagGroup {

diff --git a/pkg/flag/worker_flags.go b/pkg/flag/worker_flags.go
@@ -82,16 +82,16 @@ type WorkerFlagGroup struct {
 
 // GlobalOptions defines flags and other configuration parameters for all the subcommands
 type WorkerOptions struct {
-	Workers                   int           `json:"workers" yaml:"workers"`
-	Timeout                   time.Duration `json:"timeout" yaml:"timeout"`
-	TimeoutFileMinimum        time.Duration `json:"timeout_file_minimum" yaml:"timeout_file_minimum"`
-	TimeoutFileMaximum        time.Duration `json:"timeout_file_maximum" yaml:"timeout_file_maximum"`
-	TimeoutFileSecondPerBytes int           `json:"timeout_file_second_per_bytes" yaml:"timeout_file_second_per_bytes"`
-	TimeoutWorkerOnline       time.Duration `json:"timeout_worker_online" yaml:"timeout_worker_online"`
-	FileSizeMaximum           int           `json:"file_size_maximum" yaml:"file_size_maximum"`
-	FilesToBatch              int           `json:"files_to_batch" yaml:"files_to_batch"`
-	MemoryMaximum             int           `json:"memory_maximum" yaml:"memory_maximum"`
-	ExistingWorker            string        `json:"existing_worker" yaml:"existing_worker"`
+	Workers                   int           `mapstructure:"workers" json:"workers" yaml:"workers"`
+	Timeout                   time.Duration `mapstructure:"timeout" json:"timeout" yaml:"timeout"`
+	TimeoutFileMinimum        time.Duration `mapstructure:"timeout-file-min" json:"timeout-file-min" yaml:"timeout-file-min"`
+	TimeoutFileMaximum        time.Duration `mapstructure:"timeout-file-max"  json:"timeout-file-max" yaml:"timeout-file-max"`
+	TimeoutFileSecondPerBytes int           `mapstructure:"timeout-file-second-per-bytes" json:"timeout-file-second-per-bytes" yaml:"timeout-file-second-per-bytes"`
+	TimeoutWorkerOnline       time.Duration `mapstructure:"timeout-worker-online" json:"timeout-worker-online" yaml:"timeout-worker-online"`
+	FileSizeMaximum           int           `mapstructure:"file-size-max" json:"file-size-max" yaml:"file-size-max"`
+	FilesToBatch              int           `mapstructure:"files-to-batch" json:"files-to-batch" yaml:"files-to-batch"`
+	MemoryMaximum             int           `mapstructure:"memory-max" json:"memory-max" yaml:"memory-max"`
+	ExistingWorker            string        `mapstructure:"existing-worker" json:"existing-worker" yaml:"existing-worker"`
 }
 
 func NewWorkerFlagGroup() *WorkerFlagGroup {