Refactor detect, add entropy to all findings (#804)

Refactor `detect`, add `entropy` to all findings
gitleaks · Mar 12, 2022 · 6e72472 · 6e72472
1 parent 9326f35
commit 6e72472
Show file tree

Hide file tree

Showing 24 changed files with 1,086 additions and 913 deletions.
diff --git a/cmd/detect.go b/cmd/detect.go
@@ -11,7 +11,6 @@ import (
 
 	"github.com/zricethezav/gitleaks/v8/config"
 	"github.com/zricethezav/gitleaks/v8/detect"
-	"github.com/zricethezav/gitleaks/v8/git"
 	"github.com/zricethezav/gitleaks/v8/report"
 )
 
@@ -35,56 +34,92 @@ func runDetect(cmd *cobra.Command, args []string) {
 		err      error
 	)
 
-	viper.Unmarshal(&vc)
+	// Load config
+	if err = viper.Unmarshal(&vc); err != nil {
+		log.Fatal().Err(err).Msg("Failed to load config")
+	}
 	cfg, err := vc.Translate()
 	if err != nil {
 		log.Fatal().Err(err).Msg("Failed to load config")
 	}
-
 	cfg.Path, _ = cmd.Flags().GetString("config")
-	source, _ := cmd.Flags().GetString("source")
-	logOpts, _ := cmd.Flags().GetString("log-opts")
-	verbose, _ := cmd.Flags().GetBool("verbose")
-	redact, _ := cmd.Flags().GetBool("redact")
-	noGit, _ := cmd.Flags().GetBool("no-git")
-	exitCode, _ := cmd.Flags().GetInt("exit-code")
-	if cfg.Path == "" {
-		cfg.Path = filepath.Join(source, ".gitleaks.toml")
-	}
+
+	// start timer
 	start := time.Now()
 
+	// Setup detector
+	detector := detect.NewDetector(cfg)
+	detector.Config.Path, err = cmd.Flags().GetString("config")
+	if err != nil {
+		log.Fatal().Err(err)
+	}
+	source, err := cmd.Flags().GetString("source")
+	if err != nil {
+		log.Fatal().Err(err)
+	}
+	// if config path is not set, then use the {source}/.gitleaks.toml path.
+	// note that there may not be a `{source}/.gitleaks.toml` file, this is ok.
+	if detector.Config.Path == "" {
+		detector.Config.Path = filepath.Join(source, ".gitleaks.toml")
+	}
+	// set verbose flag
+	if detector.Verbose, err = cmd.Flags().GetBool("verbose"); err != nil {
+		log.Fatal().Err(err)
+	}
+	// set redact flag
+	if detector.Redact, err = cmd.Flags().GetBool("redact"); err != nil {
+		log.Fatal().Err(err)
+	}
+
+	// set exit code
+	exitCode, err := cmd.Flags().GetInt("exit-code")
+	if err != nil {
+		log.Fatal().Err(err)
+	}
+
+	// determine what type of scan:
+	// - git: scan the history of the repo
+	// - no-git: scan files by treating the repo as a plain directory
+	noGit, err := cmd.Flags().GetBool("no-git")
+	if err != nil {
+		log.Fatal().Err(err)
+	}
+
+	// start the detector scan
 	if noGit {
-		if logOpts != "" {
-			log.Fatal().Err(err).Msg("--log-opts cannot be used with --no-git")
-		}
-		findings, err = detect.FromFiles(source, cfg, detect.Options{
-			Verbose: verbose,
-			Redact:  redact,
-		})
+		findings, err = detector.DetectFiles(source)
 		if err != nil {
-			log.Fatal().Err(err).Msg("Failed to scan files")
+			// don't exit on error, just log it
+			log.Error().Err(err)
 		}
+
 	} else {
-		files, err := git.GitLog(source, logOpts)
+		logOpts, err := cmd.Flags().GetString("log-opts")
 		if err != nil {
-			log.Fatal().Err(err).Msg("Failed to get git log")
+			log.Fatal().Err(err)
+		}
+		findings, err = detector.DetectGit(source, logOpts, detect.DetectType)
+		if err != nil {
+			// don't exit on error, just log it
+			log.Error().Err(err)
 		}
-
-		findings = detect.FromGit(files, cfg, detect.Options{Verbose: verbose, Redact: redact})
 	}
 
+	// log info about the scan
+	log.Info().Msgf("scan completed in %s", time.Since(start))
 	if len(findings) != 0 {
 		log.Warn().Msgf("leaks found: %d", len(findings))
 	} else {
 		log.Info().Msg("no leaks found")
 	}
 
-	log.Info().Msgf("scan completed in %s", time.Since(start))
-
+	// write report if desired
 	reportPath, _ := cmd.Flags().GetString("report-path")
 	ext, _ := cmd.Flags().GetString("report-format")
 	if reportPath != "" {
-		report.Write(findings, cfg, ext, reportPath)
+		if err = report.Write(findings, cfg, ext, reportPath); err != nil {
+			log.Fatal().Err(err)
+		}
 	}
 
 	if len(findings) != 0 {

diff --git a/cmd/protect.go b/cmd/protect.go
@@ -11,7 +11,6 @@ import (
 
 	"github.com/zricethezav/gitleaks/v8/config"
 	"github.com/zricethezav/gitleaks/v8/detect"
-	"github.com/zricethezav/gitleaks/v8/git"
 	"github.com/zricethezav/gitleaks/v8/report"
 )
 
@@ -30,41 +29,75 @@ func runProtect(cmd *cobra.Command, args []string) {
 	initConfig()
 	var vc config.ViperConfig
 
-	viper.Unmarshal(&vc)
+	if err := viper.Unmarshal(&vc); err != nil {
+		log.Fatal().Err(err).Msg("Failed to load config")
+	}
 	cfg, err := vc.Translate()
 	if err != nil {
 		log.Fatal().Err(err).Msg("Failed to load config")
 	}
 
 	cfg.Path, _ = cmd.Flags().GetString("config")
-	source, _ := cmd.Flags().GetString("source")
-	verbose, _ := cmd.Flags().GetBool("verbose")
-	redact, _ := cmd.Flags().GetBool("redact")
 	exitCode, _ := cmd.Flags().GetInt("exit-code")
 	staged, _ := cmd.Flags().GetBool("staged")
-	if cfg.Path == "" {
-		cfg.Path = filepath.Join(source, ".gitleaks.toml")
-	}
 	start := time.Now()
 
-	files, err := git.GitDiff(source, staged)
+	// Setup detector
+	detector := detect.NewDetector(cfg)
+	detector.Config.Path, err = cmd.Flags().GetString("config")
+	if err != nil {
+		log.Fatal().Err(err)
+	}
+	source, err := cmd.Flags().GetString("source")
 	if err != nil {
-		log.Fatal().Err(err).Msg("Failed to get git log")
+		log.Fatal().Err(err)
+	}
+	// if config path is not set, then use the {source}/.gitleaks.toml path.
+	// note that there may not be a `{source}/.gitleaks.toml` file, this is ok.
+	if detector.Config.Path == "" {
+		detector.Config.Path = filepath.Join(source, ".gitleaks.toml")
+	}
+	// set verbose flag
+	if detector.Verbose, err = cmd.Flags().GetBool("verbose"); err != nil {
+		log.Fatal().Err(err)
+	}
+	// set redact flag
+	if detector.Redact, err = cmd.Flags().GetBool("redact"); err != nil {
+		log.Fatal().Err(err)
 	}
 
-	findings := detect.FromGit(files, cfg, detect.Options{Verbose: verbose, Redact: redact})
+	// get log options for git scan
+	logOpts, err := cmd.Flags().GetString("log-opts")
+	if err != nil {
+		log.Fatal().Err(err)
+	}
+
+	// start git scan
+	var findings []report.Finding
+	if staged {
+		findings, err = detector.DetectGit(source, logOpts, detect.ProtectStagedType)
+	} else {
+		findings, err = detector.DetectGit(source, logOpts, detect.ProtectType)
+	}
+	if err != nil {
+		// don't exit on error, just log it
+		log.Error().Err(err)
+	}
+
+	// log info about the scan
+	log.Info().Msgf("scan completed in %s", time.Since(start))
 	if len(findings) != 0 {
 		log.Warn().Msgf("leaks found: %d", len(findings))
 	} else {
 		log.Info().Msg("no leaks found")
 	}
 
-	log.Info().Msgf("scan duration: %s", time.Since(start))
-
 	reportPath, _ := cmd.Flags().GetString("report-path")
 	ext, _ := cmd.Flags().GetString("report-format")
 	if reportPath != "" {
-		report.Write(findings, cfg, ext, reportPath)
+		if err = report.Write(findings, cfg, ext, reportPath); err != nil {
+			log.Fatal().Err(err)
+		}
 	}
 	if len(findings) != 0 {
 		os.Exit(exitCode)

diff --git a/cmd/root.go b/cmd/root.go
@@ -45,7 +45,10 @@ func init() {
 	rootCmd.PersistentFlags().StringP("log-level", "l", "info", "log level (debug, info, warn, error, fatal)")
 	rootCmd.PersistentFlags().BoolP("verbose", "v", false, "show verbose output from scan")
 	rootCmd.PersistentFlags().Bool("redact", false, "redact secrets from logs and stdout")
-	viper.BindPFlag("config", rootCmd.PersistentFlags().Lookup("config"))
+	err := viper.BindPFlag("config", rootCmd.PersistentFlags().Lookup("config"))
+	if err != nil {
+		log.Fatal().Msgf("err binding config %s", err.Error())
+	}
 }
 
 func initLog() {
@@ -71,7 +74,7 @@ func initLog() {
 }
 
 func initConfig() {
-	fmt.Fprintf(os.Stderr, banner)
+	fmt.Fprint(os.Stderr, banner)
 	cfgPath, err := rootCmd.Flags().GetString("config")
 	if err != nil {
 		log.Fatal().Msg(err.Error())
@@ -97,14 +100,18 @@ func initConfig() {
 			log.Debug().Msgf("Unable to load gitleaks config from %s since --source=%s is a file, using default config",
 				filepath.Join(source, ".gitleaks.toml"), source)
 			viper.SetConfigType("toml")
-			viper.ReadConfig(strings.NewReader(config.DefaultConfig))
+			if err = viper.ReadConfig(strings.NewReader(config.DefaultConfig)); err != nil {
+				log.Fatal().Msgf("err reading toml %s", err.Error())
+			}
 			return
 		}
 
 		if _, err := os.Stat(filepath.Join(source, ".gitleaks.toml")); os.IsNotExist(err) {
 			log.Debug().Msgf("No gitleaks config found in path %s, using default gitleaks config", filepath.Join(source, ".gitleaks.toml"))
 			viper.SetConfigType("toml")
-			viper.ReadConfig(strings.NewReader(config.DefaultConfig))
+			if err = viper.ReadConfig(strings.NewReader(config.DefaultConfig)); err != nil {
+				log.Fatal().Msgf("err reading default config toml %s", err.Error())
+			}
 			return
 		} else {
 			log.Debug().Msgf("Using existing gitleaks config %s from `(--source)/.gitleaks.toml`", filepath.Join(source, ".gitleaks.toml"))

diff --git a/config/allowlist.go b/config/allowlist.go
@@ -2,13 +2,23 @@ package config
 
 import "regexp"
 
+// Allowlist allows a rule to be ignored for specific
+// regexes, paths, and/or commits
 type Allowlist struct {
+	// Short human readable description of the allowlist.
 	Description string
-	Regexes     []*regexp.Regexp
-	Paths       []*regexp.Regexp
-	Commits     []string
+
+	// Regexes is slice of content regular expressions that are allowed to be ignored.
+	Regexes []*regexp.Regexp
+
+	// Paths is a slice of path regular expressions that are allowed to be ignored.
+	Paths []*regexp.Regexp
+
+	// Commits is a slice of commit SHAs that are allowed to be ignored.
+	Commits []string
 }
 
+// CommitAllowed returns true if the commit is allowed to be ignored.
 func (a *Allowlist) CommitAllowed(c string) bool {
 	if c == "" {
 		return false
@@ -21,16 +31,12 @@ func (a *Allowlist) CommitAllowed(c string) bool {
 	return false
 }
 
+// PathAllowed returns true if the path is allowed to be ignored.
 func (a *Allowlist) PathAllowed(path string) bool {
-	if anyRegexMatch(path, a.Paths) {
-		return true
-	}
-	return false
+	return anyRegexMatch(path, a.Paths)
 }
 
+// RegexAllowed returns true if the regex is allowed to be ignored.
 func (a *Allowlist) RegexAllowed(s string) bool {
-	if anyRegexMatch(s, a.Regexes) {
-		return true
-	}
-	return false
+	return anyRegexMatch(s, a.Regexes)
 }
diff --git a/config/config_test.go b/config/config_test.go
@@ -103,7 +103,10 @@ func TestTranslate(t *testing.T) {
 		}
 
 		var vc ViperConfig
-		viper.Unmarshal(&vc)
+		err = viper.Unmarshal(&vc)
+		if err != nil {
+			t.Error(err)
+		}
 		cfg, err := vc.Translate()
 		if tt.wantError != nil {
 			if err == nil {
@@ -115,53 +118,3 @@ func TestTranslate(t *testing.T) {
 		assert.Equal(t, cfg.Rules, tt.cfg.Rules)
 	}
 }
-
-func TestIncludeEntropy(t *testing.T) {
-	tests := []struct {
-		rule    Rule
-		secret  string
-		entropy float32
-		include bool
-	}{
-		{
-			rule: Rule{
-				RuleID:      "generic-api-key",
-				SecretGroup: 4,
-				Entropy:     3.5,
-				Regex:       regexp.MustCompile(`(?i)((key|api|token|secret|password)[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9a-zA-Z\-_=]{8,64})['\"]`),
-			},
-			secret:  `e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5`,
-			entropy: 3.7906235872459746,
-			include: true,
-		},
-		{
-			rule: Rule{
-				RuleID:      "generic-api-key",
-				SecretGroup: 4,
-				Entropy:     4,
-				Regex:       regexp.MustCompile(`(?i)((key|api|token|secret|password)[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9a-zA-Z\-_=]{8,64})['\"]`),
-			},
-			secret:  `e7322523fb86ed64c836a979cf8465fbd436378c653c1db38f9ae87bc62a6fd5`,
-			entropy: 3.7906235872459746,
-			include: false,
-		},
-		{
-			rule: Rule{
-				RuleID:      "generic-api-key",
-				SecretGroup: 4,
-				Entropy:     3.0,
-				Regex:       regexp.MustCompile(`(?i)((key|api|token|secret|password)[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9a-zA-Z\-_=]{8,64})['\"]`),
-			},
-			secret:  `ssh-keyboard-interactive`,
-			entropy: 0,
-			include: false,
-		},
-	}
-
-	for _, tt := range tests {
-		include, entropy := tt.rule.IncludeEntropy(tt.secret)
-		assert.Equal(t, true, tt.rule.EntropySet())
-		assert.Equal(t, tt.entropy, float32(entropy))
-		assert.Equal(t, tt.include, include)
-	}
-}