From 011f0ba755456d4a1a1a008ba307320761c60052 Mon Sep 17 00:00:00 2001 From: Dinesh Date: Wed, 14 Oct 2020 19:38:41 +0530 Subject: [PATCH] Show progress bar on scanning --- .gitignore | 1 + detector/chain.go | 11 ++++++- detector/chain_test.go | 4 +-- detector/detector/detector.go | 2 +- .../base64_aggressive_detector_test.go | 6 ++-- detector/filecontent/filecontent_detector.go | 3 +- .../filecontent/filecontent_detector_test.go | 30 +++++++++---------- detector/filename/filename_detector.go | 4 ++- detector/filename/filename_detector_test.go | 8 ++--- detector/filesize/filesize_detector.go | 4 ++- detector/filesize/filesize_detector_test.go | 8 ++--- detector/pattern/pattern_detector.go | 3 +- detector/pattern/pattern_detector_test.go | 8 ++--- go.mod | 2 +- go.sum | 11 +++++++ talisman.go | 1 - 16 files changed, 66 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index 981bbac4..65f4298b 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ coverage.out coverage.txt .talismanrc talisman_reports/ +*.iml diff --git a/detector/chain.go b/detector/chain.go index d163be12..0db649d4 100644 --- a/detector/chain.go +++ b/detector/chain.go @@ -1,6 +1,8 @@ package detector import ( + log "github.com/Sirupsen/logrus" + "github.com/cheggaaa/pb/v3" "os" "talisman/checksumcalculator" "talisman/detector/detector" @@ -50,7 +52,14 @@ func (dc *Chain) Test(currentAdditions []gitrepo.Addition, talismanRC *talismanr hasher := utility.DefaultSHA256Hasher{} calculator := checksumcalculator.NewChecksumCalculator(hasher, append(allAdditions, currentAdditions...)) cc := helpers.NewChecksumCompare(calculator, hasher, talismanRC) + log.Printf("Number of files to scan: %d\n", len(currentAdditions)) + log.Printf("Number of detectors: %d\n", len(dc.detectors)) + total := len(currentAdditions) * len(dc.detectors) + bar := pb.StartNew(total) for _, v := range dc.detectors { - v.Test(cc, currentAdditions, talismanRC, result) + v.Test(cc, currentAdditions, talismanRC, result, func() { + bar.Increment() + }) } + bar.Finish() } diff --git a/detector/chain_test.go b/detector/chain_test.go index 8bb655ec..a77214b2 100644 --- a/detector/chain_test.go +++ b/detector/chain_test.go @@ -12,13 +12,13 @@ import ( type FailingDetection struct{} -func (v FailingDetection) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults) { +func (v FailingDetection) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) { result.Fail("some_file", "filecontent", "FAILED BY DESIGN", []string{}, severity.Low()) } type PassingDetection struct{} -func (p PassingDetection) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults) { +func (p PassingDetection) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) { } func TestEmptyValidationChainPassesAllValidations(t *testing.T) { diff --git a/detector/detector/detector.go b/detector/detector/detector.go index ea17d03f..79a24ec5 100644 --- a/detector/detector/detector.go +++ b/detector/detector/detector.go @@ -10,5 +10,5 @@ import ( //Detectors are expected to honor the ignores that are passed in and log them in the results //Detectors are expected to signal any errors to the results type Detector interface { - Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults) + Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) } diff --git a/detector/filecontent/base64_aggressive_detector_test.go b/detector/filecontent/base64_aggressive_detector_test.go index c096d48a..424b8ad2 100644 --- a/detector/filecontent/base64_aggressive_detector_test.go +++ b/detector/filecontent/base64_aggressive_detector_test.go @@ -18,7 +18,7 @@ func TestShouldFlagPotentialAWSAccessKeysInAggressiveMode(t *testing.T) { filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} - NewFileContentDetector(talismanRC).AggressiveMode().Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results) + NewFileContentDetector(talismanRC).AggressiveMode().Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {}) assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts") } @@ -29,7 +29,7 @@ func TestShouldFlagPotentialAWSAccessKeysAtPropertyDefinitionInAggressiveMode(t filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} - NewFileContentDetector(talismanRC).AggressiveMode().Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results) + NewFileContentDetector(talismanRC).AggressiveMode().Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {}) assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts") } @@ -40,7 +40,7 @@ func TestShouldNotFlagPotentialSecretsWithinSafeJavaCodeEvenInAggressiveMode(t * filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} - NewFileContentDetector(talismanRC).AggressiveMode().Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results) + NewFileContentDetector(talismanRC).AggressiveMode().Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {}) if results == nil { additions = nil } diff --git a/detector/filecontent/filecontent_detector.go b/detector/filecontent/filecontent_detector.go index 04c3fed0..c5a88f03 100644 --- a/detector/filecontent/filecontent_detector.go +++ b/detector/filecontent/filecontent_detector.go @@ -76,7 +76,7 @@ type content struct { severity severity.Severity } -func (fc *FileContentDetector) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults) { +func (fc *FileContentDetector) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) { contentTypes := []struct { contentType fn @@ -108,6 +108,7 @@ func (fc *FileContentDetector) Test(comparator helpers.ChecksumCompare, currentA for _, addition := range currentAdditions { go func(addition gitrepo.Addition) { defer waitGroup.Done() + defer additionCompletionCallback() if ignoreConfig.Deny(addition, "filecontent") || comparator.IsScanNotRequired(addition) { ignoredFilePaths <- addition.Path return diff --git a/detector/filecontent/filecontent_detector_test.go b/detector/filecontent/filecontent_detector_test.go index 6a91e81b..3dc963ad 100644 --- a/detector/filecontent/filecontent_detector_test.go +++ b/detector/filecontent/filecontent_detector_test.go @@ -25,7 +25,7 @@ func TestShouldNotFlagSafeText(t *testing.T) { filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} - NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, &talismanrc.TalismanRC{}, results) + NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, &talismanrc.TalismanRC{}, results, func() {}) assert.False(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts") } @@ -35,7 +35,7 @@ func TestShouldIgnoreFileIfNeeded(t *testing.T) { filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} - NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanrc.NewTalismanRC([]byte(talismanRCContents)), results) + NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanrc.NewTalismanRC([]byte(talismanRCContents)), results, func() {}) assert.True(t, results.Successful(), "Expected file %s to be ignored by pattern", filename) } @@ -49,7 +49,7 @@ func TestShouldNotFlag4CharSafeText(t *testing.T) { filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} - NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results) + NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {}) assert.False(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts") } @@ -60,7 +60,7 @@ func TestShouldNotFlagLowEntropyBase64Text(t *testing.T) { filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} - NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results) + NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {}) assert.False(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts") } @@ -72,7 +72,7 @@ func TestShouldFlagPotentialAWSSecretKeys(t *testing.T) { additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} filePath := additions[0].Path - NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results) + NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {}) expectedMessage := fmt.Sprintf("Expected file to not to contain base64 encoded texts such as: %s", awsSecretAccessKey) assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts") assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0]) @@ -87,7 +87,7 @@ func TestShouldFlagPotentialSecretWithoutTrimmingWhenLengthLessThan50Characters( additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} filePath := additions[0].Path - NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results) + NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {}) expectedMessage := fmt.Sprintf("Expected file to not to contain base64 encoded texts such as: %s", secret) assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts") assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0]) @@ -102,7 +102,7 @@ func TestShouldFlagPotentialJWT(t *testing.T) { additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} filePath := additions[0].Path - NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results) + NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {}) expectedMessage := fmt.Sprintf("Expected file to not to contain base64 encoded texts such as: %s", jwt[:47]+"...") assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts") assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0]) @@ -117,7 +117,7 @@ func TestShouldFlagPotentialSecretsWithinJavaCode(t *testing.T) { additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} filePath := additions[0].Path - NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results) + NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {}) expectedMessage := "Expected file to not to contain base64 encoded texts such as: accessKey=\"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPL..." assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts") assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0]) @@ -131,7 +131,7 @@ func TestShouldNotFlagPotentialSecretsWithinSafeJavaCode(t *testing.T) { filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} - NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results) + NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {}) assert.False(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts") } @@ -142,7 +142,7 @@ func TestShouldNotFlagPotentialSecretsWithinSafeLongMethodName(t *testing.T) { filename := "filename" additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} - NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results) + NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {}) assert.False(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts") } @@ -154,7 +154,7 @@ func TestShouldFlagPotentialSecretsEncodedInHex(t *testing.T) { additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} filePath := additions[0].Path - NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results) + NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {}) expectedMessage := "Expected file to not to contain hex encoded texts such as: " + hex assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0]) assert.Len(t, results.Results, 1) @@ -170,7 +170,7 @@ func TestShouldNotFlagPotentialCreditCardNumberIfAboveThreshold(t *testing.T) { var talismanRCContents = "threshold: high" talismanRCWithThreshold := talismanrc.NewTalismanRC([]byte(talismanRCContents)) - NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanRCWithThreshold), additions, talismanRCWithThreshold, results) + NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanRCWithThreshold), additions, talismanRCWithThreshold, results, func() {}) assert.False(t, results.HasFailures(), "Expected file to not flag base64 encoded texts if threshold is higher") } @@ -185,7 +185,7 @@ func TestResultsShouldContainHexTextsIfHexAndBase64ExistInFile(t *testing.T) { additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} filePath := additions[0].Path - NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results) + NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {}) expectedMessage := "Expected file to not to contain hex encoded texts such as: " + hex messageReceived := strings.Join(getFailureMessages(results, filePath), " ") assert.Regexp(t, expectedMessage, messageReceived, "Should contain hex detection message") @@ -202,7 +202,7 @@ func TestResultsShouldContainBase64TextsIfHexAndBase64ExistInFile(t *testing.T) additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} filePath := additions[0].Path - NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results) + NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {}) expectedMessage := "Expected file to not to contain base64 encoded texts such as: " + base64 messageReceived := strings.Join(getFailureMessages(results, filePath), " ") assert.Regexp(t, expectedMessage, messageReceived, "Should contain base64 detection message") @@ -217,7 +217,7 @@ func TestResultsShouldContainCreditCardNumberIfCreditCardNumberExistInFile(t *te additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} filePath := additions[0].Path - NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results) + NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {}) expectedMessage := "Expected file to not to contain credit card numbers such as: " + creditCardNumber assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0]) assert.Len(t, results.Results, 1) diff --git a/detector/filename/filename_detector.go b/detector/filename/filename_detector.go index 50eab0e8..bfe4fd92 100644 --- a/detector/filename/filename_detector.go +++ b/detector/filename/filename_detector.go @@ -83,13 +83,14 @@ func NewFileNameDetector(patternsWithSeverity []*severity.PatternSeverity, thres } //Test tests the fileNames of the Additions to ensure that they don't look suspicious -func (fd FileNameDetector) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults) { +func (fd FileNameDetector) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) { for _, addition := range currentAdditions { if ignoreConfig.Deny(addition, "filename") || comparator.IsScanNotRequired(addition) { log.WithFields(log.Fields{ "filePath": addition.Path, }).Info("Ignoring addition as it was specified to be ignored.") result.Ignore(addition.Path, "filename") + additionCompletionCallback() continue } for _, patternWithSeverity := range fd.flagPatterns { @@ -106,5 +107,6 @@ func (fd FileNameDetector) Test(comparator helpers.ChecksumCompare, currentAddit } } } + additionCompletionCallback() } } diff --git a/detector/filename/filename_detector_test.go b/detector/filename/filename_detector_test.go index 66895f18..dfe55994 100644 --- a/detector/filename/filename_detector_test.go +++ b/detector/filename/filename_detector_test.go @@ -140,7 +140,7 @@ func TestShouldIgnoreIfErrorIsBelowThreshold(t *testing.T) { results := helpers.NewDetectionResults() severity := severity.HighSeverity fileName := ".bash_aliases" - DefaultFileNameDetector(severity).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additionsNamed(fileName), talismanRC, results) + DefaultFileNameDetector(severity).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additionsNamed(fileName), talismanRC, results, func() {}) assert.False(t, results.HasFailures(), "Expected file %s to not fail", fileName) assert.True(t, results.HasWarnings(), "Expected file %s to having warnings", fileName) } @@ -166,20 +166,20 @@ func shouldNotFailWithDefaultDetectorAndIgnores(fileName, ignore string, thresho talismanRC.FileIgnoreConfig = make([]talismanrc.FileIgnoreConfig, 1) talismanRC.FileIgnoreConfig[0] = fileIgnoreConfig - DefaultFileNameDetector(threshold).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additionsNamed(fileName), talismanRC, results) + DefaultFileNameDetector(threshold).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additionsNamed(fileName), talismanRC, results, func() {}) assert.True(t, results.Successful(), "Expected file %s to be ignored by pattern", fileName, ignore) } func shouldFailWithSpecificPattern(fileName, pattern string, threshold severity.SeverityValue, t *testing.T) { results := helpers.NewDetectionResults() pt := []*severity.PatternSeverity{{Pattern: regexp.MustCompile(pattern), Severity: severity.Low()}} - NewFileNameDetector(pt, threshold).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additionsNamed(fileName), talismanRC, results) + NewFileNameDetector(pt, threshold).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additionsNamed(fileName), talismanRC, results, func() {}) assert.True(t, results.HasFailures(), "Expected file %s to fail the check against the %s pattern", fileName, pattern) } func shouldFailWithDefaultDetector(fileName, pattern string, severity severity.SeverityValue, t *testing.T) { results := helpers.NewDetectionResults() - DefaultFileNameDetector(severity).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additionsNamed(fileName), talismanRC, results) + DefaultFileNameDetector(severity).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additionsNamed(fileName), talismanRC, results, func() {}) assert.True(t, results.HasFailures(), "Expected file %s to fail the check against default detector. Missing pattern %s?", fileName, pattern) } diff --git a/detector/filesize/filesize_detector.go b/detector/filesize/filesize_detector.go index a4058e39..ce00a11e 100644 --- a/detector/filesize/filesize_detector.go +++ b/detector/filesize/filesize_detector.go @@ -19,7 +19,7 @@ func NewFileSizeDetector(size int) detector.Detector { return FileSizeDetector{size} } -func (fd FileSizeDetector) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults) { +func (fd FileSizeDetector) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) { severity := severity.SeverityConfiguration["LargeFileSize"] for _, addition := range currentAdditions { if ignoreConfig.Deny(addition, "filesize") || comparator.IsScanNotRequired(addition) { @@ -27,6 +27,7 @@ func (fd FileSizeDetector) Test(comparator helpers.ChecksumCompare, currentAddit "filePath": addition.Path, }).Info("Ignoring addition as it was specified to be ignored.") result.Ignore(addition.Path, "filesize") + additionCompletionCallback() continue } size := len(addition.Data) @@ -42,5 +43,6 @@ func (fd FileSizeDetector) Test(comparator helpers.ChecksumCompare, currentAddit result.Warn(addition.Path, "filesize", fmt.Sprintf("The file name %q with file size %d is larger than max allowed file size(%d)", addition.Path, size, fd.size), addition.Commits, severity) } } + additionCompletionCallback() } } diff --git a/detector/filesize/filesize_detector_test.go b/detector/filesize/filesize_detector_test.go index 8775e0e8..e0dfe2a3 100644 --- a/detector/filesize/filesize_detector_test.go +++ b/detector/filesize/filesize_detector_test.go @@ -17,7 +17,7 @@ func TestShouldFlagLargeFiles(t *testing.T) { results := helpers.NewDetectionResults() content := []byte("more than one byte") additions := []gitrepo.Addition{gitrepo.NewAddition("filename", content)} - NewFileSizeDetector(2).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results) + NewFileSizeDetector(2).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {}) assert.True(t, results.HasFailures(), "Expected file to fail the check against file size detector.") } @@ -27,7 +27,7 @@ func TestShouldNotFlagLargeFilesIfThresholdIsBelowSeverity(t *testing.T) { var talismanRCContents = "threshold: high" talismanRCWithThreshold := talismanrc.NewTalismanRC([]byte(talismanRCContents)) additions := []gitrepo.Addition{gitrepo.NewAddition("filename", content)} - NewFileSizeDetector(2).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanRCWithThreshold), additions, talismanRCWithThreshold, results) + NewFileSizeDetector(2).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanRCWithThreshold), additions, talismanRCWithThreshold, results, func() {}) assert.False(t, results.HasFailures(), "Expected file to not to fail the check against file size detector.") assert.True(t, results.HasWarnings(), "Expected file to have warnings against file size detector.") } @@ -36,7 +36,7 @@ func TestShouldNotFlagSmallFiles(t *testing.T) { results := helpers.NewDetectionResults() content := []byte("m") additions := []gitrepo.Addition{gitrepo.NewAddition("filename", content)} - NewFileSizeDetector(2).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results) + NewFileSizeDetector(2).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {}) assert.False(t, results.HasFailures(), "Expected file to not to fail the check against file size detector.") } @@ -54,6 +54,6 @@ func TestShouldNotFlagIgnoredLargeFiles(t *testing.T) { talismanRC.FileIgnoreConfig[0] = fileIgnoreConfig additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} - NewFileSizeDetector(2).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results) + NewFileSizeDetector(2).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {}) assert.True(t, results.Successful(), "expected file %s to be ignored by file size detector", filename) } diff --git a/detector/pattern/pattern_detector.go b/detector/pattern/pattern_detector.go index c5769238..bbcb185c 100644 --- a/detector/pattern/pattern_detector.go +++ b/detector/pattern/pattern_detector.go @@ -37,7 +37,7 @@ type match struct { } //Test tests the contents of the Additions to ensure that they don't look suspicious -func (detector PatternDetector) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults) { +func (detector PatternDetector) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) { matches := make(chan match, 512) ignoredFilePaths := make(chan gitrepo.FilePath, 512) waitGroup := &sync.WaitGroup{} @@ -45,6 +45,7 @@ func (detector PatternDetector) Test(comparator helpers.ChecksumCompare, current for _, addition := range currentAdditions { go func(addition gitrepo.Addition) { defer waitGroup.Done() + defer additionCompletionCallback() if ignoreConfig.Deny(addition, "filecontent") || comparator.IsScanNotRequired(addition) { ignoredFilePaths <- addition.Path return diff --git a/detector/pattern/pattern_detector_test.go b/detector/pattern/pattern_detector_test.go index 19fed7c9..03456e4f 100644 --- a/detector/pattern/pattern_detector_test.go +++ b/detector/pattern/pattern_detector_test.go @@ -57,7 +57,7 @@ func TestShouldIgnorePasswordPatterns(t *testing.T) { fileIgnoreConfig := talismanrc.FileIgnoreConfig{filename, "833b6c24c8c2c5c7e1663226dc401b29c005492dc76a1150fc0e0f07f29d4cc3", []string{"filecontent"}, []string{}} ignores := &talismanrc.TalismanRC{FileIgnoreConfig: []talismanrc.FileIgnoreConfig{fileIgnoreConfig}} - NewPatternDetector(customPatterns).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, ignores, results) + NewPatternDetector(customPatterns).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, ignores, results, func() {}) assert.True(t, results.Successful(), "Expected file %s to be ignored by pattern", filename) } @@ -69,7 +69,7 @@ func TestShouldIgnoreAllowedPattern(t *testing.T) { fileIgnoreConfig := talismanrc.FileIgnoreConfig{filename, "", []string{}, []string{"key"}} ignores := &talismanrc.TalismanRC{FileIgnoreConfig: []talismanrc.FileIgnoreConfig{fileIgnoreConfig}, AllowedPatterns: []string{"password"}} - NewPatternDetector(customPatterns).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, ignores, results) + NewPatternDetector(customPatterns).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, ignores, results, func() {}) assert.True(t, results.Successful(), "Expected keywords %s to be ignored by Talisman", append(fileIgnoreConfig.AllowedPatterns, ignores.AllowedPatterns...)) } func TestShouldOnlyWarnSecretPatternIfBelowThreshold(t *testing.T) { @@ -79,7 +79,7 @@ func TestShouldOnlyWarnSecretPatternIfBelowThreshold(t *testing.T) { additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} talismanRCContents := "threshold: high" talismanRCWithThreshold := talismanrc.NewTalismanRC([]byte(talismanRCContents)) - NewPatternDetector(customPatterns).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanRCWithThreshold), additions, talismanRCWithThreshold, results) + NewPatternDetector(customPatterns).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanRCWithThreshold), additions, talismanRCWithThreshold, results, func() {}) assert.False(t, results.HasFailures(), "Expected file %s to not have failures", filename) assert.True(t, results.HasWarnings(), "Expected file %s to have warnings", filename) } @@ -87,7 +87,7 @@ func TestShouldOnlyWarnSecretPatternIfBelowThreshold(t *testing.T) { func DetectionOfSecretPattern(filename string, content []byte) (*helpers.DetectionResults, []gitrepo.Addition, string) { results := helpers.NewDetectionResults() additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} - NewPatternDetector(customPatterns).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results) + NewPatternDetector(customPatterns).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {}) expected := "Potential secret pattern : " + string(content) return results, additions, expected } diff --git a/go.mod b/go.mod index 8dcdb5bd..91ac912b 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,12 @@ require ( github.com/AlecAivazis/survey/v2 v2.0.4 github.com/Sirupsen/logrus v0.0.0-20151204141443-446d1c146faa github.com/bmatcuk/doublestar v1.1.1 + github.com/cheggaaa/pb/v3 v3.0.5 github.com/common-nighthawk/go-figure v0.0.0-20190529165535-67e0ed34491a github.com/drhodes/golorem v0.0.0-20120624033213-6e38d8d5e455 github.com/fatih/color v1.7.0 github.com/golang/mock v1.3.1 github.com/kr/pretty v0.1.0 // indirect - github.com/mattn/go-runewidth v0.0.4 // indirect github.com/mattn/goveralls v0.0.3 // indirect github.com/mitchellh/gox v1.0.1 // indirect github.com/olekukonko/tablewriter v0.0.1 diff --git a/go.sum b/go.sum index 27ccdce9..605b1a0e 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,13 @@ github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8 h1:xzYJEypr/85nB github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8/go.mod h1:oX5x61PbNXchhh0oikYAH+4Pcfw5LKv21+Jnpr6r6Pc= github.com/Sirupsen/logrus v0.0.0-20151204141443-446d1c146faa h1:Yt7X+jyl7iyieH6aiMRd9gCaUT7Rw+wTKlzUVjjeaQ4= github.com/Sirupsen/logrus v0.0.0-20151204141443-446d1c146faa/go.mod h1:rmk17hk6i8ZSAJkSDa7nOxamrG+SP4P0mm+DAvExv4U= +github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= +github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/bmatcuk/doublestar v1.1.1 h1:YroD6BJCZBYx06yYFEWvUuKVWQn3vLLQAVmDmvTSaiQ= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= +github.com/cheggaaa/pb/v3 v3.0.5 h1:lmZOti7CraK9RSjzExsY53+WWfub9Qv13B5m4ptEoPE= +github.com/cheggaaa/pb/v3 v3.0.5/go.mod h1:X1L61/+36nz9bjIsrDU52qHKOQukUQe2Ge+YvGuquCw= github.com/common-nighthawk/go-figure v0.0.0-20190529165535-67e0ed34491a h1:kTv7wPomOuRf17BKQKO5Y6GrKsYC52XHrjf26H6FdQU= github.com/common-nighthawk/go-figure v0.0.0-20190529165535-67e0ed34491a/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -34,8 +39,12 @@ github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= +github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/goveralls v0.0.3 h1:GnFhBAK0wJmxZBum88FqDzcDPLjAk9sL0HzhmW+9bo8= github.com/mattn/goveralls v0.0.3/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= @@ -70,6 +79,8 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1 h1:R4dVlxdmKenVdMRS/tTspEpSTRWINYrHD8ySIU9yCIU= golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= diff --git a/talisman.go b/talisman.go index f085327f..c210a2ce 100644 --- a/talisman.go +++ b/talisman.go @@ -92,7 +92,6 @@ func main() { checksum: checksum, reportdirectory: reportdirectory, scanWithHtml: scanWithHtml, - } prompter := prompt.NewPrompt()