Skip to content

Commit

Permalink
Show progress bar on scanning
Browse files Browse the repository at this point in the history
  • Loading branch information
@dineshba
dineshba committed Oct 14, 2020
1 parent 60a8368 commit 011f0ba
Show file tree
Hide file tree
Showing 16 changed files with 66 additions and 40 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Expand Up @@ -16,3 +16,4 @@ coverage.out
coverage.txt
.talismanrc
talisman_reports/
*.iml
11 changes: 10 additions & 1 deletion detector/chain.go
View file
@@ -1,6 +1,8 @@
package detector

import (
log "github.com/Sirupsen/logrus"
"github.com/cheggaaa/pb/v3"
"os"
"talisman/checksumcalculator"
"talisman/detector/detector"
Expand Down Expand Up @@ -50,7 +52,14 @@ func (dc *Chain) Test(currentAdditions []gitrepo.Addition, talismanRC *talismanr
hasher := utility.DefaultSHA256Hasher{}
calculator := checksumcalculator.NewChecksumCalculator(hasher, append(allAdditions, currentAdditions...))
cc := helpers.NewChecksumCompare(calculator, hasher, talismanRC)
log.Printf("Number of files to scan: %d\n", len(currentAdditions))
log.Printf("Number of detectors: %d\n", len(dc.detectors))
total := len(currentAdditions) * len(dc.detectors)
bar := pb.StartNew(total)
for _, v := range dc.detectors {
v.Test(cc, currentAdditions, talismanRC, result)
v.Test(cc, currentAdditions, talismanRC, result, func() {
bar.Increment()
})
}
bar.Finish()
}
4 changes: 2 additions & 2 deletions detector/chain_test.go
View file
Expand Up @@ -12,13 +12,13 @@ import (

type FailingDetection struct{}

func (v FailingDetection) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults) {
func (v FailingDetection) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) {
result.Fail("some_file", "filecontent", "FAILED BY DESIGN", []string{}, severity.Low())
}

type PassingDetection struct{}

func (p PassingDetection) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults) {
func (p PassingDetection) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) {
}

func TestEmptyValidationChainPassesAllValidations(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion detector/detector/detector.go
View file
Expand Up @@ -10,5 +10,5 @@ import (
//Detectors are expected to honor the ignores that are passed in and log them in the results
//Detectors are expected to signal any errors to the results
type Detector interface {
Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults)
Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func())
}
6 changes: 3 additions & 3 deletions detector/filecontent/base64_aggressive_detector_test.go
View file
Expand Up @@ -18,7 +18,7 @@ func TestShouldFlagPotentialAWSAccessKeysInAggressiveMode(t *testing.T) {
filename := "filename"
additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}

NewFileContentDetector(talismanRC).AggressiveMode().Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results)
NewFileContentDetector(talismanRC).AggressiveMode().Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {})
assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
}

Expand All @@ -29,7 +29,7 @@ func TestShouldFlagPotentialAWSAccessKeysAtPropertyDefinitionInAggressiveMode(t
filename := "filename"
additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}

NewFileContentDetector(talismanRC).AggressiveMode().Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results)
NewFileContentDetector(talismanRC).AggressiveMode().Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {})
assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
}

Expand All @@ -40,7 +40,7 @@ func TestShouldNotFlagPotentialSecretsWithinSafeJavaCodeEvenInAggressiveMode(t *
filename := "filename"
additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}

NewFileContentDetector(talismanRC).AggressiveMode().Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results)
NewFileContentDetector(talismanRC).AggressiveMode().Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {})
if results == nil {
additions = nil
}
Expand Down
3 changes: 2 additions & 1 deletion detector/filecontent/filecontent_detector.go
View file
Expand Up @@ -76,7 +76,7 @@ type content struct {
severity severity.Severity
}

func (fc *FileContentDetector) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults) {
func (fc *FileContentDetector) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) {
contentTypes := []struct {
contentType
fn
Expand Down Expand Up @@ -108,6 +108,7 @@ func (fc *FileContentDetector) Test(comparator helpers.ChecksumCompare, currentA
for _, addition := range currentAdditions {
go func(addition gitrepo.Addition) {
defer waitGroup.Done()
defer additionCompletionCallback()
if ignoreConfig.Deny(addition, "filecontent") || comparator.IsScanNotRequired(addition) {
ignoredFilePaths <- addition.Path
return
Expand Down
30 changes: 15 additions & 15 deletions detector/filecontent/filecontent_detector_test.go
View file
Expand Up @@ -25,7 +25,7 @@ func TestShouldNotFlagSafeText(t *testing.T) {
filename := "filename"
additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}

NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, &talismanrc.TalismanRC{}, results)
NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, &talismanrc.TalismanRC{}, results, func() {})
assert.False(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
}

Expand All @@ -35,7 +35,7 @@ func TestShouldIgnoreFileIfNeeded(t *testing.T) {
filename := "filename"
additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}

NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanrc.NewTalismanRC([]byte(talismanRCContents)), results)
NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanrc.NewTalismanRC([]byte(talismanRCContents)), results, func() {})
assert.True(t, results.Successful(), "Expected file %s to be ignored by pattern", filename)
}

Expand All @@ -49,7 +49,7 @@ func TestShouldNotFlag4CharSafeText(t *testing.T) {
filename := "filename"
additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}

NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results)
NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {})
assert.False(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
}

Expand All @@ -60,7 +60,7 @@ func TestShouldNotFlagLowEntropyBase64Text(t *testing.T) {
filename := "filename"
additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}

NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results)
NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {})
assert.False(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
}

Expand All @@ -72,7 +72,7 @@ func TestShouldFlagPotentialAWSSecretKeys(t *testing.T) {
additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
filePath := additions[0].Path

NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results)
NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {})
expectedMessage := fmt.Sprintf("Expected file to not to contain base64 encoded texts such as: %s", awsSecretAccessKey)
assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0])
Expand All @@ -87,7 +87,7 @@ func TestShouldFlagPotentialSecretWithoutTrimmingWhenLengthLessThan50Characters(
additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
filePath := additions[0].Path

NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results)
NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {})
expectedMessage := fmt.Sprintf("Expected file to not to contain base64 encoded texts such as: %s", secret)
assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0])
Expand All @@ -102,7 +102,7 @@ func TestShouldFlagPotentialJWT(t *testing.T) {
additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
filePath := additions[0].Path

NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results)
NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {})
expectedMessage := fmt.Sprintf("Expected file to not to contain base64 encoded texts such as: %s", jwt[:47]+"...")
assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0])
Expand All @@ -117,7 +117,7 @@ func TestShouldFlagPotentialSecretsWithinJavaCode(t *testing.T) {
additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
filePath := additions[0].Path

NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results)
NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {})
expectedMessage := "Expected file to not to contain base64 encoded texts such as: accessKey=\"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPL..."
assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0])
Expand All @@ -131,7 +131,7 @@ func TestShouldNotFlagPotentialSecretsWithinSafeJavaCode(t *testing.T) {
filename := "filename"
additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}

NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results)
NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {})
assert.False(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
}

Expand All @@ -142,7 +142,7 @@ func TestShouldNotFlagPotentialSecretsWithinSafeLongMethodName(t *testing.T) {
filename := "filename"
additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}

NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results)
NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {})
assert.False(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
}

Expand All @@ -154,7 +154,7 @@ func TestShouldFlagPotentialSecretsEncodedInHex(t *testing.T) {
additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
filePath := additions[0].Path

NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results)
NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {})
expectedMessage := "Expected file to not to contain hex encoded texts such as: " + hex
assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0])
assert.Len(t, results.Results, 1)
Expand All @@ -170,7 +170,7 @@ func TestShouldNotFlagPotentialCreditCardNumberIfAboveThreshold(t *testing.T) {
var talismanRCContents = "threshold: high"
talismanRCWithThreshold := talismanrc.NewTalismanRC([]byte(talismanRCContents))

NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanRCWithThreshold), additions, talismanRCWithThreshold, results)
NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanRCWithThreshold), additions, talismanRCWithThreshold, results, func() {})

assert.False(t, results.HasFailures(), "Expected file to not flag base64 encoded texts if threshold is higher")
}
Expand All @@ -185,7 +185,7 @@ func TestResultsShouldContainHexTextsIfHexAndBase64ExistInFile(t *testing.T) {
additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
filePath := additions[0].Path

NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results)
NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {})
expectedMessage := "Expected file to not to contain hex encoded texts such as: " + hex
messageReceived := strings.Join(getFailureMessages(results, filePath), " ")
assert.Regexp(t, expectedMessage, messageReceived, "Should contain hex detection message")
Expand All @@ -202,7 +202,7 @@ func TestResultsShouldContainBase64TextsIfHexAndBase64ExistInFile(t *testing.T)
additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
filePath := additions[0].Path

NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results)
NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {})
expectedMessage := "Expected file to not to contain base64 encoded texts such as: " + base64
messageReceived := strings.Join(getFailureMessages(results, filePath), " ")
assert.Regexp(t, expectedMessage, messageReceived, "Should contain base64 detection message")
Expand All @@ -217,7 +217,7 @@ func TestResultsShouldContainCreditCardNumberIfCreditCardNumberExistInFile(t *te
additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
filePath := additions[0].Path

NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results)
NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, talismanRC, results, func() {})
expectedMessage := "Expected file to not to contain credit card numbers such as: " + creditCardNumber
assert.Equal(t, expectedMessage, getFailureMessages(results, filePath)[0])
assert.Len(t, results.Results, 1)
Expand Down
4 changes: 3 additions & 1 deletion detector/filename/filename_detector.go
View file
Expand Up @@ -83,13 +83,14 @@ func NewFileNameDetector(patternsWithSeverity []*severity.PatternSeverity, thres
}

//Test tests the fileNames of the Additions to ensure that they don't look suspicious
func (fd FileNameDetector) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults) {
func (fd FileNameDetector) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) {
for _, addition := range currentAdditions {
if ignoreConfig.Deny(addition, "filename") || comparator.IsScanNotRequired(addition) {
log.WithFields(log.Fields{
"filePath": addition.Path,
}).Info("Ignoring addition as it was specified to be ignored.")
result.Ignore(addition.Path, "filename")
additionCompletionCallback()
continue
}
for _, patternWithSeverity := range fd.flagPatterns {
Expand All @@ -106,5 +107,6 @@ func (fd FileNameDetector) Test(comparator helpers.ChecksumCompare, currentAddit
}
}
}
additionCompletionCallback()
}
}

0 comments on commit 011f0ba

Please sign in to comment.