Merge f1a2d5e into 54a328d

thoughtworks · May 28, 2020 · 42e5b33 · 42e5b33
2 parents 54a328d + f1a2d5e
commit 42e5b33
Show file tree

Hide file tree

Showing 6 changed files with 53 additions and 34 deletions.
diff --git a/detector/base64_aggressive_detector_test.go b/detector/base64_aggressive_detector_test.go
@@ -18,7 +18,7 @@ func TestShouldFlagPotentialAWSAccessKeysInAggressiveMode(t *testing.T) {
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().AggressiveMode().Test(additions, talismanRC, results)
+	NewFileContentDetector(talismanRC).AggressiveMode().Test(additions, talismanRC, results)
 	assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 }
 
@@ -29,7 +29,7 @@ func TestShouldFlagPotentialAWSAccessKeysAtPropertyDefinitionInAggressiveMode(t
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().AggressiveMode().Test(additions, talismanRC, results)
+	NewFileContentDetector(talismanRC).AggressiveMode().Test(additions, talismanRC, results)
 	assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 }
 
@@ -40,7 +40,7 @@ func TestShouldNotFlagPotentialSecretsWithinSafeJavaCodeEvenInAggressiveMode(t *
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().AggressiveMode().Test(additions, talismanRC, results)
+	NewFileContentDetector(talismanRC).AggressiveMode().Test(additions, talismanRC, results)
 	if results == nil {
 		additions = nil
 	}

diff --git a/detector/base64_detector.go b/detector/base64_detector.go
@@ -1,20 +1,33 @@
 package detector
 
+import (
+	"talisman/talismanrc"
+
+	log "github.com/Sirupsen/logrus"
+)
+
 const BASE64_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
 const BASE64_ENTROPY_THRESHOLD = 4.5
 const MIN_BASE64_SECRET_LENGTH = 20
 
 type Base64Detector struct {
-	base64Map map[string]bool
-	aggressiveDetector *Base64AggressiveDetector
-	entropy *Entropy
-	wordCheck *WordCheck
+	base64Map              map[string]bool
+	aggressiveDetector     *Base64AggressiveDetector
+	entropy                *Entropy
+	wordCheck              *WordCheck
+	base64EntropyThreshold float64
 }
 
-func NewBase64Detector() *Base64Detector {
+func NewBase64Detector(tRC *talismanrc.TalismanRC) *Base64Detector {
 	bd := Base64Detector{}
 	bd.initBase64Map()
 	bd.aggressiveDetector = nil
+
+	bd.base64EntropyThreshold = BASE64_ENTROPY_THRESHOLD
+	if tRC.Experimental.Base64EntropyThreshold > 0.0 {
+		bd.base64EntropyThreshold = tRC.Experimental.Base64EntropyThreshold
+		log.Debugf("Setting b64 entropy threshold to %f", bd.base64EntropyThreshold)
+	}
 	bd.entropy = &Entropy{}
 	return &bd
 }
@@ -30,7 +43,8 @@ func (bd *Base64Detector) checkBase64Encoding(word string) string {
 	entropyCandidates := bd.entropy.GetEntropyCandidatesWithinWord(word, MIN_BASE64_SECRET_LENGTH, bd.base64Map)
 	for _, candidate := range entropyCandidates {
 		entropy := bd.entropy.GetShannonEntropy(candidate, BASE64_CHARS)
-		if entropy > BASE64_ENTROPY_THRESHOLD && !bd.wordCheck.containsWordsOnly(candidate) {
+		log.Debugf("Detected entropy for word %s = %f", candidate, entropy)
+		if entropy > bd.base64EntropyThreshold && !bd.wordCheck.containsWordsOnly(candidate) {
 			return word
 		}
 	}
@@ -39,4 +53,3 @@ func (bd *Base64Detector) checkBase64Encoding(word string) string {
 	}
 	return ""
 }
-
diff --git a/detector/detector.go b/detector/detector.go
@@ -30,7 +30,7 @@ func NewChain() *Chain {
 func DefaultChain(tRC *talismanrc.TalismanRC) *Chain {
 	result := NewChain()
 	result.AddDetector(DefaultFileNameDetector())
-	result.AddDetector(NewFileContentDetector())
+	result.AddDetector(NewFileContentDetector(tRC))
 	result.AddDetector(NewPatternDetector(tRC.CustomPatterns))
 	return result
 }

diff --git a/detector/filecontent_detector.go b/detector/filecontent_detector.go
@@ -14,14 +14,15 @@ import (
 type fn func(fc *FileContentDetector, word string) string
 
 type FileContentDetector struct {
-	base64Detector     *Base64Detector
-	hexDetector        *HexDetector
-	creditCardDetector *CreditCardDetector
+	base64Detector         *Base64Detector
+	hexDetector            *HexDetector
+	creditCardDetector     *CreditCardDetector
+	base64EntropyThreshold float64
 }
 
-func NewFileContentDetector() *FileContentDetector {
+func NewFileContentDetector(tRC *talismanrc.TalismanRC) *FileContentDetector {
 	fc := FileContentDetector{}
-	fc.base64Detector = NewBase64Detector()
+	fc.base64Detector = NewBase64Detector(tRC)
 	fc.hexDetector = NewHexDetector()
 	fc.creditCardDetector = NewCreditCardDetector()
 	return &fc

diff --git a/detector/filecontent_detector_test.go b/detector/filecontent_detector_test.go
@@ -21,7 +21,7 @@ func TestShouldNotFlagSafeText(t *testing.T) {
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().Test(additions, &talismanrc.TalismanRC{}, results)
+	NewFileContentDetector(talismanRC).Test(additions, &talismanrc.TalismanRC{}, results)
 	assert.False(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 }
 
@@ -31,7 +31,7 @@ func TestShouldIgnoreFileIfNeeded(t *testing.T) {
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().Test(additions, talismanrc.NewTalismanRC([]byte(talismanRCContents)), results)
+	NewFileContentDetector(talismanRC).Test(additions, talismanrc.NewTalismanRC([]byte(talismanRCContents)), results)
 	assert.True(t, results.Successful(), "Expected file %s to be ignored by pattern", filename)
 }
 
@@ -45,7 +45,7 @@ func TestShouldNotFlag4CharSafeText(t *testing.T) {
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector(talismanRC).Test(additions, talismanRC, results)
 	assert.False(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 }
 
@@ -56,7 +56,7 @@ func TestShouldNotFlagLowEntropyBase64Text(t *testing.T) {
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector(talismanRC).Test(additions, talismanRC, results)
 	assert.False(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 }
 
@@ -68,7 +68,7 @@ func TestShouldFlagPotentialAWSSecretKeys(t *testing.T) {
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 	filePath := additions[0].Path
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector(talismanRC).Test(additions, talismanRC, results)
 	expectedMessage := fmt.Sprintf("Expected file to not to contain base64 encoded texts such as: %s", awsSecretAccessKey)
 	assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 	assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0])
@@ -83,7 +83,7 @@ func TestShouldFlagPotentialSecretWithoutTrimmingWhenLengthLessThan50Characters(
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 	filePath := additions[0].Path
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector(talismanRC).Test(additions, talismanRC, results)
 	expectedMessage := fmt.Sprintf("Expected file to not to contain base64 encoded texts such as: %s", secret)
 	assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 	assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0])
@@ -98,7 +98,7 @@ func TestShouldFlagPotentialJWT(t *testing.T) {
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 	filePath := additions[0].Path
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector(talismanRC).Test(additions, talismanRC, results)
 	expectedMessage := fmt.Sprintf("Expected file to not to contain base64 encoded texts such as: %s", jwt[:47]+"...")
 	assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 	assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0])
@@ -113,7 +113,7 @@ func TestShouldFlagPotentialSecretsWithinJavaCode(t *testing.T) {
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 	filePath := additions[0].Path
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector(talismanRC).Test(additions, talismanRC, results)
 	expectedMessage := "Expected file to not to contain base64 encoded texts such as: accessKey=\"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPL..."
 	assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 	assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0])
@@ -127,7 +127,7 @@ func TestShouldNotFlagPotentialSecretsWithinSafeJavaCode(t *testing.T) {
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector(talismanRC).Test(additions, talismanRC, results)
 	assert.False(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 }
 
@@ -138,7 +138,7 @@ func TestShouldNotFlagPotentialSecretsWithinSafeLongMethodName(t *testing.T) {
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector(talismanRC).Test(additions, talismanRC, results)
 	assert.False(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 }
 
@@ -150,7 +150,7 @@ func TestShouldFlagPotentialSecretsEncodedInHex(t *testing.T) {
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 	filePath := additions[0].Path
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector(talismanRC).Test(additions, talismanRC, results)
 	expectedMessage := "Expected file to not to contain hex encoded texts such as: " + hex
 	assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0])
 	assert.Len(t, results.Results, 1)
@@ -166,7 +166,7 @@ func TestResultsShouldContainHexTextsIfHexAndBase64ExistInFile(t *testing.T) {
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 	filePath := additions[0].Path
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector(talismanRC).Test(additions, talismanRC, results)
 	expectedMessage := "Expected file to not to contain hex encoded texts such as: " + hex
 	messageReceived := strings.Join(getFailureMessages(results, filePath), " ")
 	assert.Regexp(t, expectedMessage, messageReceived, "Should contain hex detection message")
@@ -183,7 +183,7 @@ func TestResultsShouldContainBase64TextsIfHexAndBase64ExistInFile(t *testing.T)
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 	filePath := additions[0].Path
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector(talismanRC).Test(additions, talismanRC, results)
 	expectedMessage := "Expected file to not to contain base64 encoded texts such as: " + base64
 	messageReceived := strings.Join(getFailureMessages(results, filePath), " ")
 	assert.Regexp(t, expectedMessage, messageReceived, "Should contain base64 detection message")
@@ -198,7 +198,7 @@ func TestResultsShouldContainCreditCardNumberIfCreditCardNumberExistInFile(t *te
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 	filePath := additions[0].Path
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector(talismanRC).Test(additions, talismanRC, results)
 	expectedMessage := "Expected file to not to contain credit card numbers such as: " + creditCardNumber
 	assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0])
 	assert.Len(t, results.Results, 1)

diff --git a/talismanrc/talismanrc.go b/talismanrc/talismanrc.go
@@ -1,15 +1,16 @@
 package talismanrc
 
 import (
-	logr "github.com/Sirupsen/logrus"
-	"github.com/spf13/afero"
-	"gopkg.in/yaml.v2"
 	"log"
 	"os"
 	"reflect"
 	"regexp"
 	"sort"
 
+	logr "github.com/Sirupsen/logrus"
+	"github.com/spf13/afero"
+	"gopkg.in/yaml.v2"
+
 	"talisman/gitrepo"
 )
 
@@ -24,7 +25,6 @@ var (
 	currentRCFileName  = DefaultRCFileName
 )
 
-
 type FileIgnoreConfig struct {
 	FileName        string   `yaml:"filename"`
 	Checksum        string   `yaml:"checksum"`
@@ -35,12 +35,17 @@ type ScopeConfig struct {
 	ScopeName string `yaml:"scope"`
 }
 
+type ExperimentalConfig struct {
+	Base64EntropyThreshold float64 `yaml:"base64EntropyThreshold,omitempty"`
+}
+
 type PatternString string
 
 type TalismanRC struct {
 	FileIgnoreConfig []FileIgnoreConfig `yaml:"fileignoreconfig,omitempty"`
 	ScopeConfig      []ScopeConfig      `yaml:"scopeconfig,omitempty"`
 	CustomPatterns   []PatternString    `yaml:"custom_patterns,omitempty"`
+	Experimental     ExperimentalConfig `yaml:"experimental,omitempty"`
 }
 
 func SetFs(_fs afero.Fs) {