diff --git a/CHANGELOG.md b/CHANGELOG.md index 358ee315a..3635a346a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ CHANGELOG ========= +1.14.0 +---- +- Entropy Range support in gitleaks config + 1.13.0 ---- - Github PR support diff --git a/gitleaks_test.go b/gitleaks_test.go index 6c93a3685..856081821 100644 --- a/gitleaks_test.go +++ b/gitleaks_test.go @@ -67,6 +67,26 @@ repos = [ ] ` +const testEntropyRange = ` +[misc] +entropy = [ + "7.5-8.0", + "3.3-3.4", +] +` +const testBadEntropyRange = ` +[misc] +entropy = [ + "8.0-3.0", +] +` +const testBadEntropyRange2 = ` +[misc] +entropy = [ + "8.0-8.9", +] +` + var benchmarkRepo *RepoDescriptor var benchmarkLeaksRepo *RepoDescriptor @@ -427,6 +447,9 @@ func testTomlLoader() string { ioutil.WriteFile(path.Join(tmpDir, "commit"), []byte(testWhitelistCommit), 0644) ioutil.WriteFile(path.Join(tmpDir, "file"), []byte(testWhitelistFile), 0644) ioutil.WriteFile(path.Join(tmpDir, "repo"), []byte(testWhitelistRepo), 0644) + ioutil.WriteFile(path.Join(tmpDir, "entropy"), []byte(testEntropyRange), 0644) + ioutil.WriteFile(path.Join(tmpDir, "badEntropy"), []byte(testBadEntropyRange), 0644) + ioutil.WriteFile(path.Join(tmpDir, "badEntropy2"), []byte(testBadEntropyRange2), 0644) return tmpDir } @@ -637,8 +660,27 @@ func TestAuditRepo(t *testing.T) { Depth: 2, }, }, + { + repo: leaksRepo, + description: "toml entropy range", + numLeaks: 422, + configPath: path.Join(configsDir, "entropy"), + }, + { + repo: leaksRepo, + description: "toml bad entropy range", + numLeaks: 0, + configPath: path.Join(configsDir, "badEntropy"), + expectedErrMsg: "entropy range must be ascending", + }, + { + repo: leaksRepo, + description: "toml bad entropy2 range", + numLeaks: 0, + configPath: path.Join(configsDir, "badEntropy2"), + expectedErrMsg: "invalid entropy ranges, must be within 0.0-8.0", + }, } - whiteListCommits = make(map[string]bool) g := goblin.Goblin(t) for _, test := range tests { @@ -671,19 +713,24 @@ func TestAuditRepo(t *testing.T) { } else { whiteListRepos = nil } - + skip := false // config paths if test.configPath != "" { os.Setenv("GITLEAKS_CONFIG", test.configPath) - loadToml() + err := loadToml() + if err != nil { + g.Assert(err.Error()).Equal(test.expectedErrMsg) + skip = true + } } + if !skip { + leaks, err = auditGitRepo(test.repo) - leaks, err = auditGitRepo(test.repo) - - if opts.Redact { - g.Assert(leaks[0].Offender).Equal("REDACTED") + if opts.Redact { + g.Assert(leaks[0].Offender).Equal("REDACTED") + } + g.Assert(len(leaks)).Equal(test.numLeaks) } - g.Assert(len(leaks)).Equal(test.numLeaks) }) }) } diff --git a/main.go b/main.go index edbff2958..11a1bfef5 100644 --- a/main.go +++ b/main.go @@ -16,6 +16,7 @@ import ( "path/filepath" "regexp" "runtime" + "strconv" "strings" "sync" "time" @@ -113,6 +114,9 @@ type Config struct { Branches []string Repos []string } + Misc struct { + Entropy []string + } } type gitDiff struct { @@ -127,8 +131,13 @@ type gitDiff struct { author string } +type entropyRange struct { + v1 float64 + v2 float64 +} + const defaultGithubURL = "https://api.github.com/" -const version = "1.13.0" +const version = "1.14.0" const errExit = 2 const leakExit = 1 const defaultConfig = ` @@ -177,11 +186,17 @@ regex = '''(?i)twitter.*['\"][0-9a-zA-Z]{35,44}['\"]''' #] #branches = [ -# "dev/STUPDIFKNFEATURE" +# "dev/goodrepo" #] #repos = [ -# "someYugeRepoWeKnowIsCLEAR" +# "mygoodrepo" +#] + +[misc] +#entropy = [ +# "3.3-4.30" +# "6.0-8.0 #] ` @@ -194,6 +209,7 @@ var ( whiteListCommits map[string]bool whiteListBranches []string whiteListRepos []string + entropyRanges []entropyRange fileDiffRegex *regexp.Regexp sshAuth *ssh.PublicKeys dir string @@ -622,11 +638,24 @@ func inspect(diff gitDiff) []Leak { leaks = addLeak(leaks, line, match, leakType, diff) } - if opts.Entropy > 0 { + if opts.Entropy > 0 || len(entropyRanges) != 0 { + entropyLeak := false words := strings.Fields(line) for _, word := range words { - if getShannonEntropy(word) >= opts.Entropy { - leaks = addLeak(leaks, line, word, "High Entropy", diff) + entropy := getShannonEntropy(word) + if entropy >= opts.Entropy && len(entropyRanges) == 0 { + entropyLeak = true + } + if len(entropyRanges) != 0 { + for _, eR := range entropyRanges { + if entropy > eR.v1 && entropy < eR.v2 { + entropyLeak = true + } + } + } + if entropyLeak { + leaks = addLeak(leaks, line, word, fmt.Sprintf("Entropy: %.2f", entropy), diff) + entropyLeak = false } } } @@ -813,6 +842,13 @@ func loadToml() error { } } + if len(config.Misc.Entropy) != 0 { + err := entropyLimits(config.Misc.Entropy) + if err != nil { + return err + } + } + if singleSearchRegex != nil { regexes["singleSearch"] = singleSearchRegex } else { @@ -836,6 +872,33 @@ func loadToml() error { return nil } +// entropyLimits hydrates entropyRanges which allows for fine tuning entropy checking +func entropyLimits(entropyLimitStr []string) error { + for _, span := range entropyLimitStr { + split := strings.Split(span, "-") + v1, err := strconv.ParseFloat(split[0], 64) + if err != nil { + return err + } + v2, err := strconv.ParseFloat(split[1], 64) + if err != nil { + return err + } + if v1 > v2 { + return fmt.Errorf("entropy range must be ascending") + } + r := entropyRange{ + v1: v1, + v2: v2, + } + if r.v1 > 8.0 || r.v1 < 0.0 || r.v2 > 8.0 || r.v2 < 0.0 { + return fmt.Errorf("invalid entropy ranges, must be within 0.0-8.0") + } + entropyRanges = append(entropyRanges, r) + } + return nil +} + // getSSHAuth return an ssh auth use by go-git to clone repos behind authentication. // If --ssh-key is set then it will attempt to load the key from that path. If not, // gitleaks will use the default $HOME/.ssh/id_rsa key