Check scan required based on wildcard in talismancr

detector/base64_aggressive_detector_test.go

@@ -18,7 +18,7 @@ func TestShouldFlagPotentialAWSAccessKeysInAggressiveMode(t *testing.T) {
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().AggressiveMode().Test(additions, talismanRC, results)
+	NewFileContentDetector().AggressiveMode().Test([]gitrepo.Addition{}, additions, talismanRC, results)
 	assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 }
 
@@ -29,7 +29,7 @@ func TestShouldFlagPotentialAWSAccessKeysAtPropertyDefinitionInAggressiveMode(t
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().AggressiveMode().Test(additions, talismanRC, results)
+	NewFileContentDetector().AggressiveMode().Test([]gitrepo.Addition{}, additions, talismanRC, results)
 	assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 }
 
@@ -40,7 +40,7 @@ func TestShouldNotFlagPotentialSecretsWithinSafeJavaCodeEvenInAggressiveMode(t *
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().AggressiveMode().Test(additions, talismanRC, results)
+	NewFileContentDetector().AggressiveMode().Test([]gitrepo.Addition{}, additions, talismanRC, results)
 	if results == nil {
 		additions = nil
 	}

detector/checksum_compare.go

@@ -1,6 +1,7 @@
 package detector
 
 import (
+	"talisman/checksumcalculator"
 	"talisman/gitrepo"
 	"talisman/talismanrc"
 	"talisman/utility"
@@ -9,11 +10,12 @@ import (
 type ChecksumCompare struct {
 	additions    []gitrepo.Addition
 	ignoreConfig *talismanrc.TalismanRC
+	allAdditions []gitrepo.Addition
 }
 
 //NewChecksumCompare returns new instance of the ChecksumCompare
-func NewChecksumCompare(gitAdditions []gitrepo.Addition, talismanRCConfig *talismanrc.TalismanRC) *ChecksumCompare {
-	cc := ChecksumCompare{additions: gitAdditions, ignoreConfig: talismanRCConfig}
+func NewChecksumCompare(allAdditions []gitrepo.Addition, gitAdditions []gitrepo.Addition, talismanRCConfig *talismanrc.TalismanRC) *ChecksumCompare {
+	cc := ChecksumCompare{allAdditions: allAdditions, additions: gitAdditions, ignoreConfig: talismanRCConfig}
 	return &cc
 }
 
@@ -22,7 +24,8 @@ func (cc *ChecksumCompare) IsScanNotRequired(addition gitrepo.Addition) bool {
 	declaredCheckSum := ""
 	for _, ignore := range cc.ignoreConfig.FileIgnoreConfig {
 		if addition.Matches(ignore.FileName) {
-			currentCollectiveChecksum = utility.CollectiveSHA256Hash([]string{ignore.FileName})
+			calculator := checksumcalculator.NewChecksumCalculator(cc.allAdditions)
+			currentCollectiveChecksum = calculator.CalculateCollectiveChecksumForPattern(ignore.FileName)
 			declaredCheckSum = ignore.Checksum
 		}
 	}

detector/checksum_compare_test.go

@@ -1,20 +1,30 @@
 package detector
 
 import (
-	"talisman/utility"
-	"testing"
-
 	"github.com/stretchr/testify/assert"
+	"talisman/gitrepo"
+	"talisman/talismanrc"
+	"testing"
 )
 
-func TestShouldReturnCorrectFileHash(t *testing.T) {
-	checksumSomeFile := utility.CollectiveSHA256Hash([]string{"some_file.pem"})
-	checksumTestSomeFile := utility.CollectiveSHA256Hash([]string{"test/some_file.pem"})
-	assert.Equal(t, checksumSomeFile, "87139cc4d975333b25b6275f97680604add51b84eb8f4a3b9dcbbc652e6f27ac", "Should be equal to some_file.pem hash value")
-	assert.Equal(t, checksumTestSomeFile, "25bd31a28bf9d4e06327f1c4a5cab2260574ae508803f66adcc393350e994866", "Should be equal to test/some_file.pem hash value")
-}
+func TestChecksumCompare_IsScanNotRequired(t *testing.T) {
+
+	t.Run("should return false if talismanrc is empty", func(t *testing.T) {
+		ignoreConfig := talismanrc.NewTalismanRC(nil)
+		cc := NewChecksumCompare([]gitrepo.Addition{}, []gitrepo.Addition{}, ignoreConfig)
+
+		required := cc.IsScanNotRequired(gitrepo.Addition{})
+
+		assert.False(t, required)
+	})
+
+	t.Run("should return false if talismanrc is empty", func(t *testing.T) {
+		ignoreConfig := talismanrc.NewTalismanRC(nil)
+		cc := NewChecksumCompare([]gitrepo.Addition{}, []gitrepo.Addition{}, ignoreConfig)
+
+		required := cc.IsScanNotRequired(gitrepo.Addition{})
+
+		assert.False(t, required)
+	})
 
-func TestShouldReturnEmptyFileHashWhenNoPathsPassed(t *testing.T) {
-	checksum := utility.CollectiveSHA256Hash([]string{})
-	assert.Equal(t, checksum, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "Should be equal to empty hash value when no paths passed")
 }

detector/detector.go

@@ -1,6 +1,7 @@
 package detector
 
 import (
+	"os"
 	"talisman/gitrepo"
 	"talisman/talismanrc"
 )
@@ -9,7 +10,7 @@ import (
 //Detectors are expected to honor the ignores that are passed in and log them in the results
 //Detectors are expected to signal any errors to the results
 type Detector interface {
-	Test(additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *DetectionResults)
+	Test(allAdditions []gitrepo.Addition, additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *DetectionResults)
 }
 
 //Chain represents a chain of Detectors.
@@ -43,7 +44,10 @@ func (dc *Chain) AddDetector(d Detector) *Chain {
 //Test validates the additions against each detector in the chain.
 //The results are passed in from detector to detector and thus collect all errors from all detectors
 func (dc *Chain) Test(additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *DetectionResults) {
+	wd, _ := os.Getwd()
+	repo := gitrepo.RepoLocatedAt(wd)
+	allAdditions := repo.TrackedFilesAsAdditions()
 	for _, v := range dc.detectors {
-		v.Test(additions, ignoreConfig, result)
+		v.Test(allAdditions, additions, ignoreConfig, result)
 	}
 }

detector/detector_test.go

@@ -28,11 +28,11 @@ func TestValidationChainWithFailingValidationAlwaysFails(t *testing.T) {
 
 type FailingDetection struct{}
 
-func (v FailingDetection) Test(additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *DetectionResults) {
+func (v FailingDetection) Test(allAdditions []gitrepo.Addition, additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *DetectionResults) {
 	result.Fail("some_file", "filecontent", "FAILED BY DESIGN", []string{})
 }
 
 type PassingDetection struct{}
 
-func (p PassingDetection) Test(additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *DetectionResults) {
+func (p PassingDetection) Test(allAdditions []gitrepo.Addition, additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *DetectionResults) {
 }

detector/filecontent_detector.go

@@ -72,7 +72,7 @@ type content struct {
 	results     []string
 }
 
-func (fc *FileContentDetector) Test(additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *DetectionResults) {
+func (fc *FileContentDetector) Test(allAdditions []gitrepo.Addition, additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *DetectionResults) {
 	contentTypes := []struct {
 		contentType
 		fn
@@ -90,7 +90,7 @@ func (fc *FileContentDetector) Test(additions []gitrepo.Addition, ignoreConfig *
 			fn:          checkCreditCardNumber,
 		},
 	}
-	cc := NewChecksumCompare(additions, ignoreConfig)
+	cc := NewChecksumCompare(allAdditions, additions, ignoreConfig)
 	re := regexp.MustCompile(`(?i)checksum[ \t]*:[ \t]*[0-9a-fA-F]+`)
 
 	contents := make(chan content, 512)

detector/filecontent_detector_test.go

@@ -21,7 +21,7 @@ func TestShouldNotFlagSafeText(t *testing.T) {
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().Test(additions, &talismanrc.TalismanRC{}, results)
+	NewFileContentDetector().Test([]gitrepo.Addition{}, additions, &talismanrc.TalismanRC{}, results)
 	assert.False(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 }
 
@@ -31,7 +31,7 @@ func TestShouldIgnoreFileIfNeeded(t *testing.T) {
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().Test(additions, talismanrc.NewTalismanRC([]byte(talismanRCContents)), results)
+	NewFileContentDetector().Test([]gitrepo.Addition{}, additions, talismanrc.NewTalismanRC([]byte(talismanRCContents)), results)
 	assert.True(t, results.Successful(), "Expected file %s to be ignored by pattern", filename)
 }
 
@@ -45,7 +45,7 @@ func TestShouldNotFlag4CharSafeText(t *testing.T) {
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector().Test([]gitrepo.Addition{}, additions, talismanRC, results)
 	assert.False(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 }
 
@@ -56,7 +56,7 @@ func TestShouldNotFlagLowEntropyBase64Text(t *testing.T) {
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector().Test([]gitrepo.Addition{}, additions, talismanRC, results)
 	assert.False(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 }
 
@@ -68,7 +68,7 @@ func TestShouldFlagPotentialAWSSecretKeys(t *testing.T) {
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 	filePath := additions[0].Path
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector().Test([]gitrepo.Addition{}, additions, talismanRC, results)
 	expectedMessage := fmt.Sprintf("Expected file to not to contain base64 encoded texts such as: %s", awsSecretAccessKey)
 	assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 	assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0])
@@ -83,7 +83,7 @@ func TestShouldFlagPotentialSecretWithoutTrimmingWhenLengthLessThan50Characters(
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 	filePath := additions[0].Path
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector().Test([]gitrepo.Addition{}, additions, talismanRC, results)
 	expectedMessage := fmt.Sprintf("Expected file to not to contain base64 encoded texts such as: %s", secret)
 	assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 	assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0])
@@ -98,7 +98,7 @@ func TestShouldFlagPotentialJWT(t *testing.T) {
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 	filePath := additions[0].Path
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector().Test([]gitrepo.Addition{}, additions, talismanRC, results)
 	expectedMessage := fmt.Sprintf("Expected file to not to contain base64 encoded texts such as: %s", jwt[:47]+"...")
 	assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 	assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0])
@@ -113,7 +113,7 @@ func TestShouldFlagPotentialSecretsWithinJavaCode(t *testing.T) {
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 	filePath := additions[0].Path
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector().Test([]gitrepo.Addition{}, additions, talismanRC, results)
 	expectedMessage := "Expected file to not to contain base64 encoded texts such as: accessKey=\"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPL..."
 	assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 	assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0])
@@ -127,7 +127,7 @@ func TestShouldNotFlagPotentialSecretsWithinSafeJavaCode(t *testing.T) {
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector().Test([]gitrepo.Addition{}, additions, talismanRC, results)
 	assert.False(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 }
 
@@ -138,7 +138,7 @@ func TestShouldNotFlagPotentialSecretsWithinSafeLongMethodName(t *testing.T) {
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector().Test([]gitrepo.Addition{}, additions, talismanRC, results)
 	assert.False(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 }
 
@@ -150,7 +150,7 @@ func TestShouldFlagPotentialSecretsEncodedInHex(t *testing.T) {
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 	filePath := additions[0].Path
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector().Test([]gitrepo.Addition{}, additions, talismanRC, results)
 	expectedMessage := "Expected file to not to contain hex encoded texts such as: " + hex
 	assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0])
 	assert.Len(t, results.Results, 1)
@@ -166,7 +166,7 @@ func TestResultsShouldContainHexTextsIfHexAndBase64ExistInFile(t *testing.T) {
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 	filePath := additions[0].Path
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector().Test([]gitrepo.Addition{}, additions, talismanRC, results)
 	expectedMessage := "Expected file to not to contain hex encoded texts such as: " + hex
 	messageReceived := strings.Join(getFailureMessages(results, filePath), " ")
 	assert.Regexp(t, expectedMessage, messageReceived, "Should contain hex detection message")
@@ -183,7 +183,7 @@ func TestResultsShouldContainBase64TextsIfHexAndBase64ExistInFile(t *testing.T)
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 	filePath := additions[0].Path
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector().Test([]gitrepo.Addition{}, additions, talismanRC, results)
 	expectedMessage := "Expected file to not to contain base64 encoded texts such as: " + base64
 	messageReceived := strings.Join(getFailureMessages(results, filePath), " ")
 	assert.Regexp(t, expectedMessage, messageReceived, "Should contain base64 detection message")
@@ -198,7 +198,7 @@ func TestResultsShouldContainCreditCardNumberIfCreditCardNumberExistInFile(t *te
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 	filePath := additions[0].Path
 
-	NewFileContentDetector().Test(additions, talismanRC, results)
+	NewFileContentDetector().Test([]gitrepo.Addition{}, additions, talismanRC, results)
 	expectedMessage := "Expected file to not to contain credit card numbers such as: " + creditCardNumber
 	assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0])
 	assert.Len(t, results.Results, 1)

detector/filename_detector.go

@@ -79,8 +79,8 @@ func NewFileNameDetector(patterns []*regexp.Regexp) Detector {
 }
 
 //Test tests the fileNames of the Additions to ensure that they don't look suspicious
-func (fd FileNameDetector) Test(additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *DetectionResults) {
-	cc := NewChecksumCompare(additions, ignoreConfig)
+func (fd FileNameDetector) Test(allAdditions []gitrepo.Addition, additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *DetectionResults) {
+	cc := NewChecksumCompare(allAdditions, additions, ignoreConfig)
 	for _, addition := range additions {
 		if ignoreConfig.Deny(addition, "filename") || cc.IsScanNotRequired(addition) {
 			log.WithFields(log.Fields{

detector/filename_detector_test.go

@@ -152,20 +152,20 @@ func shouldNotFailWithDefaultDetectorAndIgnores(fileName, ignore string, t *test
 	talismanRC.FileIgnoreConfig = make([]talismanrc.FileIgnoreConfig, 1)
 	talismanRC.FileIgnoreConfig[0] = fileIgnoreConfig
 
-	DefaultFileNameDetector().Test(additionsNamed(fileName), talismanRC, results)
+	DefaultFileNameDetector().Test([]gitrepo.Addition{}, additionsNamed(fileName), talismanRC, results)
 	assert.True(t, results.Successful(), "Expected file %s to be ignored by pattern", fileName, ignore)
 }
 
 func shouldFailWithSpecificPattern(fileName, pattern string, t *testing.T) {
 	results := NewDetectionResults()
 	pt := regexp.MustCompile(pattern)
-	NewFileNameDetector([]*regexp.Regexp{pt}).Test(additionsNamed(fileName), talismanRC, results)
+	NewFileNameDetector([]*regexp.Regexp{pt}).Test([]gitrepo.Addition{}, additionsNamed(fileName), talismanRC, results)
 	assert.True(t, results.HasFailures(), "Expected file %s to fail the check against the %s pattern", fileName, pattern)
 }
 
 func shouldFailWithDefaultDetector(fileName, pattern string, t *testing.T) {
 	results := NewDetectionResults()
-	DefaultFileNameDetector().Test(additionsNamed(fileName), talismanRC, results)
+	DefaultFileNameDetector().Test([]gitrepo.Addition{}, additionsNamed(fileName), talismanRC, results)
 	assert.True(t, results.HasFailures(), "Expected file %s to fail the check against default detector. Missing pattern %s?", fileName, pattern)
 }
 

detector/filesize_detector.go

@@ -21,8 +21,8 @@ func NewFileSizeDetector(size int) Detector {
 	return FileSizeDetector{size}
 }
 
-func (fd FileSizeDetector) Test(additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *DetectionResults) {
-	cc := NewChecksumCompare(additions, ignoreConfig)
+func (fd FileSizeDetector) Test(allAdditions []gitrepo.Addition, additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *DetectionResults) {
+	cc := NewChecksumCompare(allAdditions, additions, ignoreConfig)
 	for _, addition := range additions {
 		if ignoreConfig.Deny(addition, "filesize") || cc.IsScanNotRequired(addition) {
 			log.WithFields(log.Fields{

detector/filesize_detector_test.go

@@ -13,15 +13,15 @@ func TestShouldFlagLargeFiles(t *testing.T) {
 	results := NewDetectionResults()
 	content := []byte("more than one byte")
 	additions := []gitrepo.Addition{gitrepo.NewAddition("filename", content)}
-	NewFileSizeDetector(2).Test(additions, talismanRC, results)
+	NewFileSizeDetector(2).Test([]gitrepo.Addition{}, additions, talismanRC, results)
 	assert.True(t, results.HasFailures(), "Expected file to fail the check against file size detector.")
 }
 
 func TestShouldNotFlagSmallFiles(t *testing.T) {
 	results := NewDetectionResults()
 	content := []byte("m")
 	additions := []gitrepo.Addition{gitrepo.NewAddition("filename", content)}
-	NewFileSizeDetector(2).Test(additions, talismanRC, results)
+	NewFileSizeDetector(2).Test([]gitrepo.Addition{}, additions, talismanRC, results)
 	assert.False(t, results.HasFailures(), "Expected file to not to fail the check against file size detector.")
 }
 
@@ -39,6 +39,6 @@ func TestShouldNotFlagIgnoredLargeFiles(t *testing.T) {
 	talismanRC.FileIgnoreConfig[0] = fileIgnoreConfig
 
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
-	NewFileSizeDetector(2).Test(additions, talismanRC, results)
+	NewFileSizeDetector(2).Test([]gitrepo.Addition{}, additions, talismanRC, results)
 	assert.True(t, results.Successful(), "expected file %s to be ignored by file size detector", filename)
 }

detector/pattern_detector.go

@@ -34,8 +34,8 @@ type match struct {
 }
 
 //Test tests the contents of the Additions to ensure that they don't look suspicious
-func (detector PatternDetector) Test(additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *DetectionResults) {
-	cc := NewChecksumCompare(additions, ignoreConfig)
+func (detector PatternDetector) Test(allAdditions []gitrepo.Addition, additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *DetectionResults) {
+	cc := NewChecksumCompare(allAdditions, additions, ignoreConfig)
 	matches := make(chan match, 512)
 	ignoredFilePaths := make(chan gitrepo.FilePath, 512)
 	waitGroup := &sync.WaitGroup{}