diff --git a/.talismanrc b/.talismanrc index 98d4d9cd..8bd1dfbd 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,6 +1,6 @@ fileignoreconfig: - filename: detector/pattern_detector_test.go - checksum: fbb55705e75e1fcba609b10c454e277e5c67c930bd5aedd514055570e2b868b4 + checksum: 4d70b790f28f2d23d506f808d489aa43f1efd2514549ae6a83a535e1223382e3 - filename: detector/detection_results_test.go checksum: 69fed055782cddfe0f0d23ea440cef9f9dd0b9e8a3c8a73856741bb26257b223 ignore_detectors: @@ -10,5 +10,17 @@ fileignoreconfig: ignore_detectors: [] - filename: global_install_scripts/install.bash checksum: 5d659125ecbe619ea99f5bc71c2d761b586ce3ec9ccab7683ee54f4ebde9f748 +- filename: detector/filecontent/filecontent_detector_test.go + checksum: affb25839a87476dcef4f4169ccb9b54b2d2f2437cef3aca24f4d3b69d5886c5 +- filename: detector/pattern/match_pattern_test.go + checksum: b90530d286fbc0ee864d2350fc0c532e0fb2f01149d51e81339b420439014238 +- filename: detector/pattern/pattern_detector_test.go + checksum: 4d70b790f28f2d23d506f808d489aa43f1efd2514549ae6a83a535e1223382e3 +- filename: detector/pattern/pattern_detector.go + checksum: 248bc5f67fa12d39b0fa1b63319a5b125006858a11603a837d8c53dbab2277c3 +- filename: detector/filename/filename_detector.go + checksum: 5782cb11c373723ec7b40279a3dd375c0cd1d285ac0d032599f0300d9e133eec +- filename: detector/filename/filename_detector_test.go + checksum: 0a9c9f113e203ca29d3a9bf0b4802a252e990c2132e1f168a46ab49ed532e6c9 scopeconfig: - scope: go diff --git a/README.md b/README.md index bba1df56..e7f8f102 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,10 @@ - [Talisman in action](#talisman-in-action) - [Validations](#validations) - [Ignoring files](#ignoring-files) - - [Talisman as a CLI utility](#talisman-as-a-cli-utility) - - [Git History Scanner](#git-history-scanner) - - [Checksum Calculator](#checksum-calculator) + - [Configuring severity threshold](#configuring-severity-threshold) + - [Talisman as a CLI utility](#talisman-as-a-cli-utility) + - [Git History Scanner](#git-history-scanner) + - [Checksum Calculator](#checksum-calculator) - [Talisman HTML Reporting](#talisman-html-reporting) - [Uninstallation](#uninstallation) - [From a global hook template](#uninstallation-from-a-global-hook-template) @@ -351,6 +352,22 @@ custom_patterns: * It also brings in more secure practices with every modification of a file with a potential sensitive value to be reviewed * The new format also brings in the extensibility to introduce new usable functionalities. Keep a watch out for more +## Configuring severity threshold + +Each validation is associated with a severity +1. low +2. medium +3. high + +You can specify a threshold in your .talismanrc: + +```yaml +threshold: medium +``` +This will report all Medium severity issues and higher (Potential risks that are below the threshold will be reported in the warnings) + +By default, the threshold is set to low + ## Talisman as a CLI utility If you execute `talisman` on the command line, you will be able to view all the parameter options you can pass diff --git a/detector/chain.go b/detector/chain.go index 6801547f..d163be12 100644 --- a/detector/chain.go +++ b/detector/chain.go @@ -29,7 +29,7 @@ func NewChain() *Chain { //DefaultChain returns a DetectorChain with pre-configured detectors func DefaultChain(tRC *talismanrc.TalismanRC) *Chain { result := NewChain() - result.AddDetector(filename.DefaultFileNameDetector()) + result.AddDetector(filename.DefaultFileNameDetector(tRC.Threshold)) result.AddDetector(filecontent.NewFileContentDetector(tRC)) result.AddDetector(pattern.NewPatternDetector(tRC.CustomPatterns)) return result @@ -54,4 +54,3 @@ func (dc *Chain) Test(currentAdditions []gitrepo.Addition, talismanRC *talismanr v.Test(cc, currentAdditions, talismanRC, result) } } - diff --git a/detector/chain_test.go b/detector/chain_test.go index 8c205c94..8bb655ec 100644 --- a/detector/chain_test.go +++ b/detector/chain_test.go @@ -1,17 +1,19 @@ package detector import ( - "github.com/stretchr/testify/assert" "talisman/detector/helpers" + "talisman/detector/severity" "talisman/gitrepo" "talisman/talismanrc" "testing" + + "github.com/stretchr/testify/assert" ) type FailingDetection struct{} func (v FailingDetection) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults) { - result.Fail("some_file", "filecontent", "FAILED BY DESIGN", []string{}) + result.Fail("some_file", "filecontent", "FAILED BY DESIGN", []string{}, severity.Low()) } type PassingDetection struct{} diff --git a/detector/filecontent/filecontent_detector.go b/detector/filecontent/filecontent_detector.go index d6c0e1ed..0142f474 100644 --- a/detector/filecontent/filecontent_detector.go +++ b/detector/filecontent/filecontent_detector.go @@ -6,6 +6,7 @@ import ( "strings" "sync" "talisman/detector/helpers" + "talisman/detector/severity" "talisman/gitrepo" "talisman/talismanrc" @@ -72,24 +73,29 @@ type content struct { path gitrepo.FilePath contentType contentType results []string + severity severity.Severity } func (fc *FileContentDetector) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults) { contentTypes := []struct { contentType fn + severity severity.Severity }{ { contentType: base64Content, fn: checkBase64, + severity: severity.Medium(), }, { contentType: hexContent, fn: checkHex, + severity: severity.Medium(), }, { contentType: creditCardContent, fn: checkCreditCardNumber, + severity: severity.High(), }, } re := regexp.MustCompile(`(?i)checksum[ \t]*:[ \t]*[0-9a-fA-F]+`) @@ -118,6 +124,7 @@ func (fc *FileContentDetector) Test(comparator helpers.ChecksumCompare, currentA path: addition.Path, contentType: ct.contentType, results: fc.detectFile(addition.Data, ct.fn), + severity: ct.severity, } } }(addition) @@ -141,7 +148,7 @@ func (fc *FileContentDetector) Test(comparator helpers.ChecksumCompare, currentA contentChanHasMore = false continue } - processContent(c, result) + processContent(c, ignoreConfig.Threshold, result) } } } @@ -153,16 +160,16 @@ func processIgnoredFilepath(path gitrepo.FilePath, result *helpers.DetectionResu result.Ignore(path, "filecontent") } -func processContent(c content, result *helpers.DetectionResults) { +func processContent(c content, threshold severity.SeverityValue, result *helpers.DetectionResults) { for _, res := range c.results { if res != "" { log.WithFields(log.Fields{ "filePath": c.path, }).Info(c.contentType.getInfo()) - if string(c.name) == talismanrc.DefaultRCFileName { - result.Warn(c.path, "filecontent", fmt.Sprintf(c.contentType.getMessageFormat(), formatForReporting(res)), []string{}) + if string(c.name) == talismanrc.DefaultRCFileName || !c.severity.ExceedsThreshold(threshold) { + result.Warn(c.path, "filecontent", fmt.Sprintf(c.contentType.getMessageFormat(), formatForReporting(res)), []string{}, c.severity) } else { - result.Fail(c.path, "filecontent", fmt.Sprintf(c.contentType.getMessageFormat(), formatForReporting(res)), []string{}) + result.Fail(c.path, "filecontent", fmt.Sprintf(c.contentType.getMessageFormat(), formatForReporting(res)), []string{}, c.severity) } } } diff --git a/detector/filecontent/filecontent_detector_test.go b/detector/filecontent/filecontent_detector_test.go index f5c2b354..ed7bee05 100644 --- a/detector/filecontent/filecontent_detector_test.go +++ b/detector/filecontent/filecontent_detector_test.go @@ -79,6 +79,20 @@ func TestShouldFlagPotentialAWSSecretKeys(t *testing.T) { assert.Len(t, results.Results, 1) } +func TestShouldNotFlagBase64ContentIfThresholdIsHigher(t *testing.T) { + const awsSecretAccessKey string = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + results := helpers.NewDetectionResults() + content := []byte(awsSecretAccessKey) + filename := "filename" + additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} + var talismanRCContents = "threshold: high" + talismanRCWithThreshold := talismanrc.NewTalismanRC([]byte(talismanRCContents)) + + NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanRCWithThreshold), additions, talismanRCWithThreshold, results) + assert.False(t, results.HasFailures(), "Expected file to not flag base64 encoded texts if threshold is higher") + assert.True(t, results.HasWarnings(), "Expected file to have warngings for base64 encoded texts if threshold is higher") +} + func TestShouldFlagPotentialSecretWithoutTrimmingWhenLengthLessThan50Characters(t *testing.T) { const secret string = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9asdfa" results := helpers.NewDetectionResults() @@ -160,6 +174,21 @@ func TestShouldFlagPotentialSecretsEncodedInHex(t *testing.T) { assert.Len(t, results.Results, 1) } +func TestShouldNotFlagSecretsEncodedInHexIfAboveThreshold(t *testing.T) { + const hex string = "68656C6C6F20776F726C6421" + results := helpers.NewDetectionResults() + content := []byte(hex) + filename := "filename" + additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} + + var talismanRCContents = "threshold: high" + talismanRCWithThreshold := talismanrc.NewTalismanRC([]byte(talismanRCContents)) + + NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanRCWithThreshold), additions, talismanRCWithThreshold, results) + + assert.False(t, results.HasFailures(), "Expected file to not flag base64 encoded texts if threshold is higher") +} + func TestResultsShouldContainHexTextsIfHexAndBase64ExistInFile(t *testing.T) { const hex string = "68656C6C6F20776F726C6421" const base64 string = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" diff --git a/detector/filename/filename_detector.go b/detector/filename/filename_detector.go index 63e1ea83..5bb484ba 100644 --- a/detector/filename/filename_detector.go +++ b/detector/filename/filename_detector.go @@ -5,6 +5,7 @@ import ( "regexp" "talisman/detector/detector" "talisman/detector/helpers" + "talisman/detector/severity" "talisman/gitrepo" "talisman/talismanrc" @@ -13,71 +14,72 @@ import ( ) var ( - filenamePatterns = []*regexp.Regexp{ - regexp.MustCompile(`^.+_rsa$`), - regexp.MustCompile(`^.+_dsa.*$`), - regexp.MustCompile(`^.+_ed25519$`), - regexp.MustCompile(`^.+_ecdsa$`), - regexp.MustCompile(`^\.\w+_history$`), - regexp.MustCompile(`^.+\.pem$`), - regexp.MustCompile(`^.+\.ppk$`), - regexp.MustCompile(`^.+\.key(pair)?$`), - regexp.MustCompile(`^.+\.pkcs12$`), - regexp.MustCompile(`^.+\.pfx$`), - regexp.MustCompile(`^.+\.p12$`), - regexp.MustCompile(`^.+\.asc$`), - regexp.MustCompile(`^\.?htpasswd$`), - regexp.MustCompile(`^\.?netrc$`), - regexp.MustCompile(`^.*\.tblk$`), - regexp.MustCompile(`^.*\.ovpn$`), - regexp.MustCompile(`^.*\.kdb$`), - regexp.MustCompile(`^.*\.agilekeychain$`), - regexp.MustCompile(`^.*\.keychain$`), - regexp.MustCompile(`^.*\.key(store|ring)$`), - regexp.MustCompile(`^jenkins\.plugins\.publish_over_ssh\.BapSshPublisherPlugin.xml$`), - regexp.MustCompile(`^credentials\.xml$`), - regexp.MustCompile(`^.*\.pubxml(\.user)?$`), - regexp.MustCompile(`^\.?s3cfg$`), - regexp.MustCompile(`^\.gitrobrc$`), - regexp.MustCompile(`^\.?(bash|zsh)rc$`), - regexp.MustCompile(`^\.?(bash_|zsh_)?profile$`), - regexp.MustCompile(`^\.?(bash_|zsh_)?aliases$`), - regexp.MustCompile(`^secret_token.rb$`), - regexp.MustCompile(`^omniauth.rb$`), - regexp.MustCompile(`^carrierwave.rb$`), - regexp.MustCompile(`^schema.rb$`), - regexp.MustCompile(`^database.yml$`), - regexp.MustCompile(`^settings.py$`), - regexp.MustCompile(`^.*(config)(\.inc)?\.php$`), - regexp.MustCompile(`^LocalSettings.php$`), - regexp.MustCompile(`\.?env`), - regexp.MustCompile(`\bdump|dump\b`), - regexp.MustCompile(`\bsql|sql\b`), - regexp.MustCompile(`\bdump|dump\b`), - regexp.MustCompile(`password`), - regexp.MustCompile(`backup`), - regexp.MustCompile(`private.*key`), - regexp.MustCompile(`(oauth).*(token)`), - regexp.MustCompile(`^.*\.log$`), - regexp.MustCompile(`^\.?kwallet$`), - regexp.MustCompile(`^\.?gnucash$`), + filenamePatterns = []*severity.PatternSeverity{ + {Pattern: regexp.MustCompile(`^.+_rsa$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^.+_dsa.*$`), Severity: severity.High()}, + {Pattern: regexp.MustCompile(`^.+_ed25519$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^.+_ecdsa$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^\.\w+_history$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^.+\.pem$`), Severity: severity.High()}, + {Pattern: regexp.MustCompile(`^.+\.ppk$`), Severity: severity.High()}, + {Pattern: regexp.MustCompile(`^.+\.key(pair)?$`), Severity: severity.High()}, + {Pattern: regexp.MustCompile(`^.+\.pkcs12$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^.+\.pfx$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^.+\.p12$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^.+\.asc$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^\.?htpasswd$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^\.?netrc$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^.*\.tblk$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^.*\.ovpn$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^.*\.kdb$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^.*\.agilekeychain$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^.*\.keychain$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^.*\.key(store|ring)$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^jenkins\.plugins\.publish_over_ssh\.BapSshPublisherPlugin.xml$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^credentials\.xml$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^.*\.pubxml(\.user)?$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^\.?s3cfg$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^\.gitrobrc$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^\.?(bash|zsh)rc$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^\.?(bash_|zsh_)?profile$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^\.?(bash_|zsh_)?aliases$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^secret_token.rb$`), Severity: severity.High()}, + {Pattern: regexp.MustCompile(`^omniauth.rb$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^carrierwave.rb$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^schema.rb$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^database.yml$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^settings.py$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^.*(config)(\.inc)?\.php$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^LocalSettings.php$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`\.?env`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`\bdump|dump\b`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`\bsql|sql\b`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`\bdump|dump\b`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`password`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`backup`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`private.*key`), Severity: severity.High()}, + {Pattern: regexp.MustCompile(`(oauth).*(token)`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^.*\.log$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^\.?kwallet$`), Severity: severity.Low()}, + {Pattern: regexp.MustCompile(`^\.?gnucash$`), Severity: severity.Low()}, } ) //FileNameDetector represents tests performed against the fileName of the Additions. //The Paths of the supplied Additions are tested against the configured patterns and if any of them match, it is logged as a failure during the run type FileNameDetector struct { - flagPatterns []*regexp.Regexp + flagPatterns []*severity.PatternSeverity + threshold severity.SeverityValue } //DefaultFileNameDetector returns a FileNameDetector that tests Additions against the pre-configured patterns -func DefaultFileNameDetector() detector.Detector { - return NewFileNameDetector(filenamePatterns) +func DefaultFileNameDetector(threshold severity.SeverityValue) detector.Detector { + return NewFileNameDetector(filenamePatterns, threshold) } //NewFileNameDetector returns a FileNameDetector that tests Additions against the supplied patterns -func NewFileNameDetector(patterns []*regexp.Regexp) detector.Detector { - return FileNameDetector{patterns} +func NewFileNameDetector(patternsWithSeverity []*severity.PatternSeverity, threshold severity.SeverityValue) detector.Detector { + return FileNameDetector{patternsWithSeverity, threshold} } //Test tests the fileNames of the Additions to ensure that they don't look suspicious @@ -90,13 +92,18 @@ func (fd FileNameDetector) Test(comparator helpers.ChecksumCompare, currentAddit result.Ignore(addition.Path, "filename") continue } - for _, pattern := range fd.flagPatterns { - if pattern.MatchString(string(addition.Name)) { + for _, patternWithSeverity := range fd.flagPatterns { + if patternWithSeverity.Pattern.MatchString(string(addition.Name)) { log.WithFields(log.Fields{ "filePath": addition.Path, - "pattern": pattern, + "pattern": patternWithSeverity.Pattern, + "severity": patternWithSeverity.Severity, }).Info("Failing file as it matched pattern.") - result.Fail(addition.Path, "filename", fmt.Sprintf("The file name %q failed checks against the pattern %s", addition.Path, pattern), addition.Commits) + if patternWithSeverity.Severity.ExceedsThreshold(fd.threshold) { + result.Fail(addition.Path, "filename", fmt.Sprintf("The file name %q failed checks against the pattern %s", addition.Path, patternWithSeverity.Pattern), addition.Commits, patternWithSeverity.Severity) + } else { + result.Warn(addition.Path, "filename", fmt.Sprintf("The file name %q failed checks against the pattern %s", addition.Path, patternWithSeverity.Pattern), addition.Commits, patternWithSeverity.Severity) + } } } } diff --git a/detector/filename/filename_detector_test.go b/detector/filename/filename_detector_test.go index 455763cc..66895f18 100644 --- a/detector/filename/filename_detector_test.go +++ b/detector/filename/filename_detector_test.go @@ -6,6 +6,7 @@ package filename import ( "regexp" "talisman/detector/helpers" + "talisman/detector/severity" "talisman/utility" "testing" @@ -18,134 +19,143 @@ import ( var talismanRC = &talismanrc.TalismanRC{} func TestShouldFlagPotentialSSHPrivateKeys(t *testing.T) { - shouldFail("id_rsa", "^.+_rsa$", t) - shouldFail("id_dsa", "^.+_dsa.*$", t) - shouldFail("id_dsa.pub", "^.+_dsa.*$", t) - shouldFail("id_ed25519", "^.+_ed25519$", t) - shouldFail("id_ecdsa", "^.+_ecdsa$", t) + shouldFail("id_rsa", "^.+_rsa$", severity.LowSeverity, t) + shouldFail("id_dsa", "^.+_dsa.*$", severity.LowSeverity, t) + shouldFail("id_dsa.pub", "^.+_dsa.*$", severity.LowSeverity, t) + shouldFail("id_ed25519", "^.+_ed25519$", severity.LowSeverity, t) + shouldFail("id_ecdsa", "^.+_ecdsa$", severity.LowSeverity, t) } func TestShouldFlagPotentialHistoryFiles(t *testing.T) { - shouldFail(".bash_history", "^\\.\\w+_history$", t) - shouldFail(".zsh_history", "^\\.\\w+_history$", t) - shouldFail(".z_history", "^\\.\\w+_history$", t) - shouldFail(".irb_history", "^\\.\\w+_history$", t) - shouldFail(".psql_history", "^\\.\\w+_history$", t) - shouldFail(".mysql_history", "^\\.\\w+_history$", t) + shouldFail(".bash_history", "^\\.\\w+_history$", severity.LowSeverity, t) + shouldFail(".zsh_history", "^\\.\\w+_history$", severity.LowSeverity, t) + shouldFail(".z_history", "^\\.\\w+_history$", severity.LowSeverity, t) + shouldFail(".irb_history", "^\\.\\w+_history$", severity.LowSeverity, t) + shouldFail(".psql_history", "^\\.\\w+_history$", severity.LowSeverity, t) + shouldFail(".mysql_history", "^\\.\\w+_history$", severity.LowSeverity, t) } func TestShouldFlagPotentialPrivateKeys(t *testing.T) { - shouldFail("foo.pem", "^.+\\.pem$", t) - shouldFail("foo.ppk", "^.+\\.ppk$", t) - shouldFail("foo.key", "^.+\\.key(pair)?$", t) - shouldFail("foo.keypair", "^.+\\.key(pair)?$", t) + shouldFail("foo.pem", "^.+\\.pem$", severity.LowSeverity, t) + shouldFail("foo.ppk", "^.+\\.ppk$", severity.LowSeverity, t) + shouldFail("foo.key", "^.+\\.key(pair)?$", severity.LowSeverity, t) + shouldFail("foo.keypair", "^.+\\.key(pair)?$", severity.LowSeverity, t) } func TestShouldFlagPotentialKeyBundles(t *testing.T) { - shouldFail("foo.pkcs12", "^.+\\.pkcs12$", t) - shouldFail("foo.pfx", "^.+\\.pfx$", t) - shouldFail("foo.p12", "^.+\\.p12$", t) - shouldFail("foo.asc", "^.+\\.asc$", t) + shouldFail("foo.pkcs12", "^.+\\.pkcs12$", severity.LowSeverity, t) + shouldFail("foo.pfx", "^.+\\.pfx$", severity.LowSeverity, t) + shouldFail("foo.p12", "^.+\\.p12$", severity.LowSeverity, t) + shouldFail("foo.asc", "^.+\\.asc$", severity.LowSeverity, t) } func TestShouldFlagPotentialConfigurationFiles(t *testing.T) { - shouldFail(".htpasswd", "^\\.?htpasswd$", t) - shouldFail("htpasswd", "^\\.?htpasswd$", t) - shouldFail(".netrc", "^\\.?netrc$", t) - shouldFail("netrc", "^\\.?netrc$", t) - shouldFail("foo.tblk", "^.*\\.tblk$", t) //Tunnelblick - shouldFail("foo.ovpn", "^.*\\.ovpn$", t) //OpenVPN + shouldFail(".htpasswd", "^\\.?htpasswd$", severity.LowSeverity, t) + shouldFail("htpasswd", "^\\.?htpasswd$", severity.LowSeverity, t) + shouldFail(".netrc", "^\\.?netrc$", severity.LowSeverity, t) + shouldFail("netrc", "^\\.?netrc$", severity.LowSeverity, t) + shouldFail("foo.tblk", "^.*\\.tblk$", severity.LowSeverity, t) //Tunnelblick + shouldFail("foo.ovpn", "^.*\\.ovpn$", severity.LowSeverity, t) //OpenVPN } func TestShouldFlagPotentialCrendentialDatabases(t *testing.T) { - shouldFail("foo.kdb", "^.*\\.kdb$", t) //KeePass - shouldFail("foo.agilekeychain", "^.*\\.agilekeychain$", t) //1Password - shouldFail("foo.keychain", "^.*\\.keychain$", t) //apple keychain - shouldFail("foo.keystore", "^.*\\.key(store|ring)$", t) //gnome keyring db - shouldFail("foo.keyring", "^.*\\.key(store|ring)$", t) //gnome keyring db + shouldFail("foo.kdb", "^.*\\.kdb$", severity.LowSeverity, t) //KeePass + shouldFail("foo.agilekeychain", "^.*\\.agilekeychain$", severity.LowSeverity, t) //1Password + shouldFail("foo.keychain", "^.*\\.keychain$", severity.LowSeverity, t) //apple keychain + shouldFail("foo.keystore", "^.*\\.key(store|ring)$", severity.LowSeverity, t) //gnome keyring db + shouldFail("foo.keyring", "^.*\\.key(store|ring)$", severity.LowSeverity, t) //gnome keyring db } func TestShouldFlagPotentialJenkinsAndCICompromises(t *testing.T) { - shouldFail("jenkins.plugins.publish_over_ssh.BapSshPublisherPlugin.xml", "^jenkins\\.plugins\\.publish_over_ssh\\.BapSshPublisherPlugin.xml$", t) - shouldFail("credentials.xml", "^credentials\\.xml$", t) - shouldFail("foo.pubxml.user", "^.*\\.pubxml(\\.user)?$", t) - shouldFail("foo.pubxml", "^.*\\.pubxml(\\.user)?$", t) + shouldFail("jenkins.plugins.publish_over_ssh.BapSshPublisherPlugin.xml", "^jenkins\\.plugins\\.publish_over_ssh\\.BapSshPublisherPlugin.xml$", severity.LowSeverity, t) + shouldFail("credentials.xml", "^credentials\\.xml$", severity.LowSeverity, t) + shouldFail("foo.pubxml.user", "^.*\\.pubxml(\\.user)?$", severity.LowSeverity, t) + shouldFail("foo.pubxml", "^.*\\.pubxml(\\.user)?$", severity.LowSeverity, t) } func TestShouldFlagPotentialConfigurationFilesThatMightContainSensitiveInformation(t *testing.T) { - shouldFail(".s3cfg", "^\\.?s3cfg$", t) //s3 configuration - shouldFail("foo.ovpn", "^.*\\.ovpn$", t) //OpenVPN configuration - shouldFail(".gitrobrc", "^\\.gitrobrc$", t) //Gitrob configuration - shouldFail(".bashrc", "^\\.?(bash|zsh)rc$", t) - shouldFail(".zshrc", "^\\.?(bash|zsh)rc$", t) - shouldFail(".profile", "^\\.?(bash_|zsh_)?profile$", t) - shouldFail(".bash_profile", "^\\.?(bash_|zsh_)?profile$", t) - shouldFail(".zsh_profile", "^\\.?(bash_|zsh_)?profile$", t) - shouldFail(".bash_aliases", "^\\.?(bash_|zsh_)?aliases$", t) - shouldFail(".zsh_aliases", "^\\.?(bash_|zsh_)?aliases$", t) - shouldFail(".aliases", "^\\.?(bash_|zsh_)?aliases$", t) - shouldFail("secret_token.rb", "^secret_token.rb$", t) //Rails secret token. http://www.exploit-db.com/exploits/27527 - shouldFail("omniauth.rb", "^omniauth.rb$", t) //OmniAuth configuration file, client application secrets - shouldFail("carrierwave.rb", "^carrierwave.rb$", t) //May contain Amazon S3 and Google Storage credentials - shouldFail("schema.rb", "^schema.rb$", t) //Rails application DB schema info - shouldFail("database.yml", "^database.yml$", t) //Rails db connection strings - shouldFail("settings.py", "^settings.py$", t) //Django credentials, keys etc - shouldFail("wp-config.php", "^.*(config)(\\.inc)?\\.php$", t) //Wordpress PHP config file - shouldFail("config.php", "^.*(config)(\\.inc)?\\.php$", t) //General PHP config file - shouldFail("config.inc.php", "^.*(config)(\\.inc)?\\.php$", t) //PHP MyAdmin file with credentials etc - shouldFail("LocalSettings.php", "^LocalSettings.php$", t) //MediaWiki configuration file - shouldFail(".env", "\\.?env", t) //PHP environment file that contains sensitive data + shouldFail(".s3cfg", "^\\.?s3cfg$", severity.LowSeverity, t) //s3 configuration + shouldFail("foo.ovpn", "^.*\\.ovpn$", severity.LowSeverity, t) //OpenVPN configuration + shouldFail(".gitrobrc", "^\\.gitrobrc$", severity.LowSeverity, t) //Gitrob configuration + shouldFail(".bashrc", "^\\.?(bash|zsh)rc$", severity.LowSeverity, t) + shouldFail(".zshrc", "^\\.?(bash|zsh)rc$", severity.LowSeverity, t) + shouldFail(".profile", "^\\.?(bash_|zsh_)?profile$", severity.LowSeverity, t) + shouldFail(".bash_profile", "^\\.?(bash_|zsh_)?profile$", severity.LowSeverity, t) + shouldFail(".zsh_profile", "^\\.?(bash_|zsh_)?profile$", severity.LowSeverity, t) + shouldFail(".bash_aliases", "^\\.?(bash_|zsh_)?aliases$", severity.LowSeverity, t) + shouldFail(".zsh_aliases", "^\\.?(bash_|zsh_)?aliases$", severity.LowSeverity, t) + shouldFail(".aliases", "^\\.?(bash_|zsh_)?aliases$", severity.LowSeverity, t) + shouldFail("secret_token.rb", "^secret_token.rb$", severity.LowSeverity, t) //Rails secret token. http://www.exploit-db.com/exploits/27527 + shouldFail("omniauth.rb", "^omniauth.rb$", severity.LowSeverity, t) //OmniAuth configuration file, client application secrets + shouldFail("carrierwave.rb", "^carrierwave.rb$", severity.LowSeverity, t) //May contain Amazon S3 and Google Storage credentials + shouldFail("schema.rb", "^schema.rb$", severity.LowSeverity, t) //Rails application DB schema info + shouldFail("database.yml", "^database.yml$", severity.LowSeverity, t) //Rails db connection strings + shouldFail("settings.py", "^settings.py$", severity.LowSeverity, t) //Django credentials, keys etc + shouldFail("wp-config.php", "^.*(config)(\\.inc)?\\.php$", severity.LowSeverity, t) //Wordpress PHP config file + shouldFail("config.php", "^.*(config)(\\.inc)?\\.php$", severity.LowSeverity, t) //General PHP config file + shouldFail("config.inc.php", "^.*(config)(\\.inc)?\\.php$", severity.LowSeverity, t) //PHP MyAdmin file with credentials etc + shouldFail("LocalSettings.php", "^LocalSettings.php$", severity.LowSeverity, t) //MediaWiki configuration file + shouldFail(".env", "\\.?env", severity.LowSeverity, t) //PHP environment file that contains sensitive data } func TestShouldFlagPotentialSuspiciousSoundingFileNames(t *testing.T) { - shouldFail("database.dump", "\\bdump|dump\\b", t) //Dump might contain sensitive information - shouldFail("foo.sql", "\\bsql|sql\\b", t) //Sql file, might be a dump and contain sensitive information - shouldFail("mydb.sqldump", "\\bdump|dump\\b", t) //Sql file, dump file, might be a dump and contain sensitive information + shouldFail("database.dump", "\\bdump|dump\\b", severity.LowSeverity, t) //Dump might contain sensitive information + shouldFail("foo.sql", "\\bsql|sql\\b", severity.LowSeverity, t) //Sql file, might be a dump and contain sensitive information + shouldFail("mydb.sqldump", "\\bdump|dump\\b", severity.LowSeverity, t) //Sql file, dump file, might be a dump and contain sensitive information - shouldFail("foo_password", "password", t) //Looks like a password? - shouldFail("foo.password", "password", t) //Looks like a password? - shouldFail("foo_password.txt", "password", t) //Looks like a password? + shouldFail("foo_password", "password", severity.LowSeverity, t) //Looks like a password? + shouldFail("foo.password", "password", severity.LowSeverity, t) //Looks like a password? + shouldFail("foo_password.txt", "password", severity.LowSeverity, t) //Looks like a password? - shouldFail("foo_backup", "backup", t) //Looks like a backup. Might contain sensitive information. - shouldFail("foo.backup", "backup", t) //Looks like a backup. Might contain sensitive information. - shouldFail("foo_backup.txt", "backup", t) //Looks like a backup. Might contain sensitive information. + shouldFail("foo_backup", "backup", severity.LowSeverity, t) //Looks like a backup. Might contain sensitive information. + shouldFail("foo.backup", "backup", severity.LowSeverity, t) //Looks like a backup. Might contain sensitive information. + shouldFail("foo_backup.txt", "backup", severity.LowSeverity, t) //Looks like a backup. Might contain sensitive information. - shouldFail("private_key", "private.*key", t) //Looks like a private key. - shouldFail("private.key", "private.*key", t) //Looks like a private key. - shouldFail("private_key.txt", "private.*key", t) //Looks like a private key. - shouldFail("otr.private_key", "private.*key", t) + shouldFail("private_key", "private.*key", severity.LowSeverity, t) //Looks like a private key. + shouldFail("private.key", "private.*key", severity.LowSeverity, t) //Looks like a private key. + shouldFail("private_key.txt", "private.*key", severity.LowSeverity, t) //Looks like a private key. + shouldFail("otr.private_key", "private.*key", severity.LowSeverity, t) - shouldFail("oauth_token", "(oauth).*(token)", t) //Looks like an oauth token - shouldFail("oauth.token", "(oauth).*(token)", t) //Looks like an oauth token - shouldFail("oauth_token.txt", "(oauth).*(token)", t) //Looks like an oauth token + shouldFail("oauth_token", "(oauth).*(token)", severity.LowSeverity, t) //Looks like an oauth token + shouldFail("oauth.token", "(oauth).*(token)", severity.LowSeverity, t) //Looks like an oauth token + shouldFail("oauth_token.txt", "(oauth).*(token)", severity.LowSeverity, t) //Looks like an oauth token - shouldFail("development.log", "^.*\\.log$", t) //Looks like a log file, could contain sensitive information + shouldFail("development.log", "^.*\\.log$", severity.LowSeverity, t) //Looks like a log file, could contain sensitive information } func TestFilenameDetectorReportsFailuresIfAnyFileInAdditionsMatchesAnyFlagPattern(t *testing.T) { - shouldFail(".kwallet", "^\\.?kwallet$", t) - shouldFail("kwallet", "^\\.?kwallet$", t) - shouldFail(".gnucash", "^\\.?gnucash$", t) - shouldFail("gnucash", "^\\.?gnucash$", t) + shouldFail(".kwallet", "^\\.?kwallet$", severity.LowSeverity, t) + shouldFail("kwallet", "^\\.?kwallet$", severity.LowSeverity, t) + shouldFail(".gnucash", "^\\.?gnucash$", severity.LowSeverity, t) + shouldFail("gnucash", "^\\.?gnucash$", severity.LowSeverity, t) } func TestShouldIgnoreFilesWhenAskedToDoSoByIgnores(t *testing.T) { - shouldIgnoreFilesWhichWouldOtherwiseTriggerErrors("id_rsa", "id_rsa", t) - shouldIgnoreFilesWhichWouldOtherwiseTriggerErrors("id_rsa", "*_rsa", t) - shouldIgnoreFilesWhichWouldOtherwiseTriggerErrors("id_dsa", "id_*", t) + shouldIgnoreFilesWhichWouldOtherwiseTriggerErrors("id_rsa", "id_rsa", severity.LowSeverity, t) + shouldIgnoreFilesWhichWouldOtherwiseTriggerErrors("id_rsa", "*_rsa", severity.LowSeverity, t) + shouldIgnoreFilesWhichWouldOtherwiseTriggerErrors("id_dsa", "id_*", severity.LowSeverity, t) } -func shouldFail(fileName, pattern string, t *testing.T) { - shouldFailWithSpecificPattern(fileName, pattern, t) - shouldFailWithDefaultDetector(fileName, pattern, t) +func TestShouldIgnoreIfErrorIsBelowThreshold(t *testing.T) { + results := helpers.NewDetectionResults() + severity := severity.HighSeverity + fileName := ".bash_aliases" + DefaultFileNameDetector(severity).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additionsNamed(fileName), talismanRC, results) + assert.False(t, results.HasFailures(), "Expected file %s to not fail", fileName) + assert.True(t, results.HasWarnings(), "Expected file %s to having warnings", fileName) +} + +func shouldFail(fileName, pattern string, threshold severity.SeverityValue, t *testing.T) { + shouldFailWithSpecificPattern(fileName, pattern, threshold, t) + shouldFailWithDefaultDetector(fileName, pattern, threshold, t) } -func shouldIgnoreFilesWhichWouldOtherwiseTriggerErrors(fileName, ignore string, t *testing.T) { - shouldFailWithDefaultDetector(fileName, "", t) - shouldNotFailWithDefaultDetectorAndIgnores(fileName, ignore, t) +func shouldIgnoreFilesWhichWouldOtherwiseTriggerErrors(fileName, ignore string, threshold severity.SeverityValue, t *testing.T) { + shouldFailWithDefaultDetector(fileName, "", threshold, t) + shouldNotFailWithDefaultDetectorAndIgnores(fileName, ignore, threshold, t) } -func shouldNotFailWithDefaultDetectorAndIgnores(fileName, ignore string, t *testing.T) { +func shouldNotFailWithDefaultDetectorAndIgnores(fileName, ignore string, threshold severity.SeverityValue, t *testing.T) { results := helpers.NewDetectionResults() fileIgnoreConfig := talismanrc.FileIgnoreConfig{} @@ -156,20 +166,20 @@ func shouldNotFailWithDefaultDetectorAndIgnores(fileName, ignore string, t *test talismanRC.FileIgnoreConfig = make([]talismanrc.FileIgnoreConfig, 1) talismanRC.FileIgnoreConfig[0] = fileIgnoreConfig - DefaultFileNameDetector().Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additionsNamed(fileName), talismanRC, results) + DefaultFileNameDetector(threshold).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additionsNamed(fileName), talismanRC, results) assert.True(t, results.Successful(), "Expected file %s to be ignored by pattern", fileName, ignore) } -func shouldFailWithSpecificPattern(fileName, pattern string, t *testing.T) { +func shouldFailWithSpecificPattern(fileName, pattern string, threshold severity.SeverityValue, t *testing.T) { results := helpers.NewDetectionResults() - pt := regexp.MustCompile(pattern) - NewFileNameDetector([]*regexp.Regexp{pt}).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additionsNamed(fileName), talismanRC, results) + pt := []*severity.PatternSeverity{{Pattern: regexp.MustCompile(pattern), Severity: severity.Low()}} + NewFileNameDetector(pt, threshold).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additionsNamed(fileName), talismanRC, results) assert.True(t, results.HasFailures(), "Expected file %s to fail the check against the %s pattern", fileName, pattern) } -func shouldFailWithDefaultDetector(fileName, pattern string, t *testing.T) { +func shouldFailWithDefaultDetector(fileName, pattern string, severity severity.SeverityValue, t *testing.T) { results := helpers.NewDetectionResults() - DefaultFileNameDetector().Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additionsNamed(fileName), talismanRC, results) + DefaultFileNameDetector(severity).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additionsNamed(fileName), talismanRC, results) assert.True(t, results.HasFailures(), "Expected file %s to fail the check against default detector. Missing pattern %s?", fileName, pattern) } diff --git a/detector/filesize/filesize_detector.go b/detector/filesize/filesize_detector.go index 16cb7f7a..2e264c8c 100644 --- a/detector/filesize/filesize_detector.go +++ b/detector/filesize/filesize_detector.go @@ -4,6 +4,7 @@ import ( "fmt" "talisman/detector/detector" "talisman/detector/helpers" + "talisman/detector/severity" "talisman/gitrepo" "talisman/talismanrc" @@ -19,6 +20,7 @@ func NewFileSizeDetector(size int) detector.Detector { } func (fd FileSizeDetector) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults) { + severity := severity.Medium() for _, addition := range currentAdditions { if ignoreConfig.Deny(addition, "filesize") || comparator.IsScanNotRequired(addition) { log.WithFields(log.Fields{ @@ -34,7 +36,11 @@ func (fd FileSizeDetector) Test(comparator helpers.ChecksumCompare, currentAddit "fileSize": size, "maxSize": fd.size, }).Info("Failing file as it is larger than max allowed file size.") - result.Fail(addition.Path, "filesize", fmt.Sprintf("The file name %q with file size %d is larger than max allowed file size(%d)", addition.Path, size, fd.size), addition.Commits) + if severity.ExceedsThreshold(ignoreConfig.Threshold) { + result.Fail(addition.Path, "filesize", fmt.Sprintf("The file name %q with file size %d is larger than max allowed file size(%d)", addition.Path, size, fd.size), addition.Commits, severity) + } else { + result.Warn(addition.Path, "filesize", fmt.Sprintf("The file name %q with file size %d is larger than max allowed file size(%d)", addition.Path, size, fd.size), addition.Commits, severity) + } } } } diff --git a/detector/filesize/filesize_detector_test.go b/detector/filesize/filesize_detector_test.go index 09e53906..8775e0e8 100644 --- a/detector/filesize/filesize_detector_test.go +++ b/detector/filesize/filesize_detector_test.go @@ -21,6 +21,17 @@ func TestShouldFlagLargeFiles(t *testing.T) { assert.True(t, results.HasFailures(), "Expected file to fail the check against file size detector.") } +func TestShouldNotFlagLargeFilesIfThresholdIsBelowSeverity(t *testing.T) { + results := helpers.NewDetectionResults() + content := []byte("more than one byte") + var talismanRCContents = "threshold: high" + talismanRCWithThreshold := talismanrc.NewTalismanRC([]byte(talismanRCContents)) + additions := []gitrepo.Addition{gitrepo.NewAddition("filename", content)} + NewFileSizeDetector(2).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanRCWithThreshold), additions, talismanRCWithThreshold, results) + assert.False(t, results.HasFailures(), "Expected file to not to fail the check against file size detector.") + assert.True(t, results.HasWarnings(), "Expected file to have warnings against file size detector.") +} + func TestShouldNotFlagSmallFiles(t *testing.T) { results := helpers.NewDetectionResults() content := []byte("m") diff --git a/detector/helpers/detection_results.go b/detector/helpers/detection_results.go index 32b16c62..8ba4c4bc 100644 --- a/detector/helpers/detection_results.go +++ b/detector/helpers/detection_results.go @@ -5,6 +5,7 @@ import ( "log" "os" "strings" + "talisman/detector/severity" "talisman/gitrepo" "talisman/prompt" "talisman/talismanrc" @@ -17,9 +18,10 @@ import ( ) type Details struct { - Category string `json:"type"` - Message string `json:"message"` - Commits []string `json:"commits"` + Category string `json:"type"` + Message string `json:"message"` + Commits []string `json:"commits"` + Severity severity.Severity `json:"severity,omitempty"` } type ResultsDetails struct { @@ -65,7 +67,7 @@ func (r *ResultsDetails) getWarningDataByCategoryAndMessage(failureMessage strin func (r *ResultsDetails) getFailureDataByCategoryAndMessage(failureMessage string, category string) *Details { detail := getDetaisByCategoryAndMessage(r.FailureList, category, failureMessage) if detail == nil { - detail = &Details{category, failureMessage, make([]string, 0)} + detail = &Details{category, failureMessage, make([]string, 0), severity.Low()} r.FailureList = append(r.FailureList, *detail) } return detail @@ -79,7 +81,7 @@ func (r *ResultsDetails) addIgnoreDataByCategory(category string) { } } if !isCategoryAlreadyPresent { - detail := Details{category, "", make([]string, 0)} + detail := Details{category, "", make([]string, 0), severity.Low()} r.IgnoreList = append(r.IgnoreList, detail) } } @@ -114,7 +116,7 @@ func NewDetectionResults() *DetectionResults { //Fail is used to mark the supplied FilePath as failing a detection for a supplied reason. //Detectors are encouraged to provide context sensitive messages so that fixing the errors is made simple for the end user //Fail may be called multiple times for each FilePath and the calls accumulate the provided reasons -func (r *DetectionResults) Fail(filePath gitrepo.FilePath, category string, message string, commits []string) { +func (r *DetectionResults) Fail(filePath gitrepo.FilePath, category string, message string, commits []string, severity severity.Severity) { isFilePresentInResults := false for resultIndex := 0; resultIndex < len(r.Results); resultIndex++ { if r.Results[resultIndex].Filename == filePath { @@ -127,12 +129,12 @@ func (r *DetectionResults) Fail(filePath gitrepo.FilePath, category string, mess } } if !isEntryPresentForGivenCategoryAndMessage { - r.Results[resultIndex].FailureList = append(r.Results[resultIndex].FailureList, Details{category, message, commits}) + r.Results[resultIndex].FailureList = append(r.Results[resultIndex].FailureList, Details{category, message, commits, severity}) } } } if !isFilePresentInResults { - failureDetails := Details{category, message, commits} + failureDetails := Details{category, message, commits, severity} resultDetails := ResultsDetails{filePath, make([]Details, 0), make([]Details, 0), make([]Details, 0)} resultDetails.FailureList = append(resultDetails.FailureList, failureDetails) r.Results = append(r.Results, resultDetails) @@ -140,7 +142,7 @@ func (r *DetectionResults) Fail(filePath gitrepo.FilePath, category string, mess r.updateResultsSummary(category) } -func (r *DetectionResults) Warn(filePath gitrepo.FilePath, category string, message string, commits []string) { +func (r *DetectionResults) Warn(filePath gitrepo.FilePath, category string, message string, commits []string, severity severity.Severity) { isFilePresentInResults := false for resultIndex := 0; resultIndex < len(r.Results); resultIndex++ { if r.Results[resultIndex].Filename == filePath { @@ -153,12 +155,12 @@ func (r *DetectionResults) Warn(filePath gitrepo.FilePath, category string, mess } } if !isEntryPresentForGivenCategoryAndMessage { - r.Results[resultIndex].WarningList = append(r.Results[resultIndex].WarningList, Details{category, message, commits}) + r.Results[resultIndex].WarningList = append(r.Results[resultIndex].WarningList, Details{category, message, commits, severity}) } } } if !isFilePresentInResults { - warningDetails := Details{category, message, commits} + warningDetails := Details{category, message, commits, severity} resultDetails := ResultsDetails{filePath, make([]Details, 0), make([]Details, 0), make([]Details, 0)} resultDetails.WarningList = append(resultDetails.WarningList, warningDetails) r.Results = append(r.Results, resultDetails) @@ -182,13 +184,13 @@ func (r *DetectionResults) Ignore(filePath gitrepo.FilePath, category string) { } } if !isEntryPresentForGivenCategory { - detail := Details{category, "", make([]string, 0)} + detail := Details{category, "", make([]string, 0), severity.Low()} r.Results[resultIndex].IgnoreList = append(r.Results[resultIndex].IgnoreList, detail) } } } if !isFilePresentInResults { - ignoreDetails := Details{category, "", make([]string, 0)} + ignoreDetails := Details{category, "", make([]string, 0), severity.Low()} resultDetails := ResultsDetails{filePath, make([]Details, 0), make([]Details, 0), make([]Details, 0)} resultDetails.IgnoreList = append(resultDetails.IgnoreList, ignoreDetails) r.Results = append(r.Results, resultDetails) @@ -196,8 +198,8 @@ func (r *DetectionResults) Ignore(filePath gitrepo.FilePath, category string) { r.Summary.Types.Ignores++ } -func createNewResultForFile(category string, message string, commits []string, filePath gitrepo.FilePath) ResultsDetails { - failureDetails := Details{category, message, commits} +func createNewResultForFile(category string, message string, commits []string, filePath gitrepo.FilePath, severity severity.Severity) ResultsDetails { + failureDetails := Details{category, message, commits, severity} resultDetails := ResultsDetails{filePath, make([]Details, 0), make([]Details, 0), make([]Details, 0)} resultDetails.FailureList = append(resultDetails.FailureList, failureDetails) return resultDetails @@ -252,7 +254,7 @@ func (r *DetectionResults) ReportWarnings() string { var data [][]string table := tablewriter.NewWriter(os.Stdout) - table.SetHeader([]string{"File", "Warnings"}) + table.SetHeader([]string{"File", "Warnings", "Severity"}) table.SetRowLine(true) for _, resultDetails := range r.Results { @@ -281,7 +283,7 @@ func (r *DetectionResults) Report(fs afero.Fs, ignoreFile string, promptContext var data [][]string table := tablewriter.NewWriter(os.Stdout) - table.SetHeader([]string{"File", "Errors"}) + table.SetHeader([]string{"File", "Errors", "Severity"}) table.SetRowLine(true) for _, resultDetails := range r.Results { @@ -369,7 +371,7 @@ func (r *DetectionResults) ReportFileFailures(filePath gitrepo.FilePath) [][]str if len(detail.Message) > 150 { detail.Message = detail.Message[:150] + "\n" + detail.Message[150:] } - data = append(data, []string{string(filePath), detail.Message}) + data = append(data, []string{string(filePath), detail.Message, detail.Severity.String()}) } } return data @@ -383,7 +385,7 @@ func (r *DetectionResults) ReportFileWarnings(filePath gitrepo.FilePath) [][]str if len(detail.Message) > 150 { detail.Message = detail.Message[:150] + "\n" + detail.Message[150:] } - data = append(data, []string{string(filePath), detail.Message}) + data = append(data, []string{string(filePath), detail.Message, detail.Severity.String()}) } } return data diff --git a/detector/helpers/detection_results_test.go b/detector/helpers/detection_results_test.go index 99c5617f..ecf5c8b9 100644 --- a/detector/helpers/detection_results_test.go +++ b/detector/helpers/detection_results_test.go @@ -2,6 +2,7 @@ package helpers import ( "strings" + "talisman/detector/severity" mock "talisman/internal/mock/prompt" "talisman/prompt" "talisman/talismanrc" @@ -21,25 +22,25 @@ func TestNewDetectionResultsAreSuccessful(t *testing.T) { func TestCallingFailOnDetectionResultsFails(t *testing.T) { results := NewDetectionResults() - results.Fail("some_filename", "filename", "Bomb", []string{}) + results.Fail("some_filename", "filename", "Bomb", []string{}, severity.Low()) assert.False(t, results.Successful(), "Calling fail on a result should not make it succeed") assert.True(t, results.HasFailures(), "Calling fail on a result should make it fail") } func TestCanRecordMultipleErrorsAgainstASingleFile(t *testing.T) { results := NewDetectionResults() - results.Fail("some_filename", "filename", "Bomb", []string{}) - results.Fail("some_filename", "filename", "Complete & utter failure", []string{}) - results.Fail("another_filename", "filename", "Complete & utter failure", []string{}) + results.Fail("some_filename", "filename", "Bomb", []string{}, severity.Low()) + results.Fail("some_filename", "filename", "Complete & utter failure", []string{}, severity.Low()) + results.Fail("another_filename", "filename", "Complete & utter failure", []string{}, severity.Low()) assert.Len(t, results.GetFailures("some_filename"), 2, "Expected two errors against some_filename.") assert.Len(t, results.GetFailures("another_filename"), 1, "Expected one error against another_filename") } func TestResultsReportsFailures(t *testing.T) { results := NewDetectionResults() - results.Fail("some_filename", "", "Bomb", []string{}) - results.Fail("some_filename", "", "Complete & utter failure", []string{}) - results.Fail("another_filename", "", "Complete & utter failure", []string{}) + results.Fail("some_filename", "", "Bomb", []string{}, severity.Low()) + results.Fail("some_filename", "", "Complete & utter failure", []string{}, severity.Low()) + results.Fail("another_filename", "", "Complete & utter failure", []string{}, severity.Low()) actualErrorReport := results.ReportFileFailures("some_filename") firstErrorMessage := strings.Join(actualErrorReport[0], " ") @@ -107,7 +108,7 @@ func TestTalismanRCSuggestionWhenThereAreFailures(t *testing.T) { t.Run("when user declines, entry should not be added to talismanrc", func(t *testing.T) { promptContext := prompt.NewPromptContext(true, prompter) prompter.EXPECT().Confirm("Do you want to add some_file.pem with above checksum in talismanrc ?").Return(false) - results.Fail("some_file.pem", "filecontent", "Bomb", []string{}) + results.Fail("some_file.pem", "filecontent", "Bomb", []string{}, severity.Low()) results.Report(fs, ignoreFile, promptContext) bytesFromFile, err := afero.ReadFile(fs, ignoreFile) @@ -120,7 +121,7 @@ func TestTalismanRCSuggestionWhenThereAreFailures(t *testing.T) { t.Run("when interactive flag is set to false, it should not ask user", func(t *testing.T) { promptContext := prompt.NewPromptContext(false, prompter) prompter.EXPECT().Confirm(gomock.Any()).Return(false).Times(0) - results.Fail("some_file.pem", "filecontent", "Bomb", []string{}) + results.Fail("some_file.pem", "filecontent", "Bomb", []string{}, severity.Low()) results.Report(fs, ignoreFile, promptContext) bytesFromFile, err := afero.ReadFile(fs, ignoreFile) @@ -134,7 +135,7 @@ func TestTalismanRCSuggestionWhenThereAreFailures(t *testing.T) { promptContext := prompt.NewPromptContext(true, prompter) prompter.EXPECT().Confirm("Do you want to add some_file.pem with above checksum in talismanrc ?").Return(true) - results.Fail("some_file.pem", "filecontent", "Bomb", []string{}) + results.Fail("some_file.pem", "filecontent", "Bomb", []string{}, severity.Low()) expectedFileContent := `fileignoreconfig: - filename: some_file.pem @@ -152,7 +153,7 @@ func TestTalismanRCSuggestionWhenThereAreFailures(t *testing.T) { promptContext := prompt.NewPromptContext(true, prompter) prompter.EXPECT().Confirm("Do you want to add existing.pem with above checksum in talismanrc ?").Return(true) results := NewDetectionResults() - results.Fail("existing.pem", "filecontent", "This will bomb!", []string{}) + results.Fail("existing.pem", "filecontent", "This will bomb!", []string{}, severity.Low()) expectedFileContent := `fileignoreconfig: - filename: existing.pem @@ -172,8 +173,8 @@ func TestTalismanRCSuggestionWhenThereAreFailures(t *testing.T) { prompter.EXPECT().Confirm("Do you want to add some_file.pem with above checksum in talismanrc ?").Return(true) prompter.EXPECT().Confirm("Do you want to add another.pem with above checksum in talismanrc ?").Return(true) - results.Fail("some_file.pem", "filecontent", "Bomb", []string{}) - results.Fail("another.pem", "filecontent", "password", []string{}) + results.Fail("some_file.pem", "filecontent", "Bomb", []string{}, severity.Low()) + results.Fail("another.pem", "filecontent", "password", []string{}, severity.Low()) expectedFileContent := `fileignoreconfig: - filename: another.pem diff --git a/detector/pattern/match_pattern.go b/detector/pattern/match_pattern.go index 38290aa8..46b7a554 100644 --- a/detector/pattern/match_pattern.go +++ b/detector/pattern/match_pattern.go @@ -2,28 +2,35 @@ package pattern import ( "fmt" - "github.com/Sirupsen/logrus" "regexp" + "talisman/detector/severity" "talisman/talismanrc" + + "github.com/Sirupsen/logrus" ) type PatternMatcher struct { - regexes []*regexp.Regexp + regexes []*severity.PatternSeverity } -func (pm *PatternMatcher) check(content string) []string { - var detected []string - for _, regex := range pm.regexes { +type DetectionsWithSeverity struct { + detections []string + severity severity.Severity +} + +func (pm *PatternMatcher) check(content string, thresholdValue severity.SeverityValue) []DetectionsWithSeverity { + var detectionsWithSeverity []DetectionsWithSeverity + for _, pattern := range pm.regexes { + var detected []string + regex := pattern.Pattern logrus.Debugf("checking for pattern %v", regex) matches := regex.FindAllString(content, -1) if matches != nil { detected = append(detected, matches...) + detectionsWithSeverity = append(detectionsWithSeverity, DetectionsWithSeverity{detections: detected, severity: pattern.Severity}) } } - if detected != nil { - return detected - } - return []string{""} + return detectionsWithSeverity } func (pm *PatternMatcher) add(ps talismanrc.PatternString) { @@ -32,10 +39,10 @@ func (pm *PatternMatcher) add(ps talismanrc.PatternString) { logrus.Warnf("ignoring invalid pattern '%s'", ps) return } - logrus.Infof("added custom pattern '%s'", ps) - pm.regexes = append(pm.regexes, re) + logrus.Infof("added custom pattern '%s' with high severity", ps) + pm.regexes = append(pm.regexes, &severity.PatternSeverity{Pattern: re, Severity: severity.High()}) } -func NewPatternMatcher(patterns []*regexp.Regexp) *PatternMatcher { +func NewPatternMatcher(patterns []*severity.PatternSeverity) *PatternMatcher { return &PatternMatcher{patterns} } diff --git a/detector/pattern/match_pattern_test.go b/detector/pattern/match_pattern_test.go index 06a3226e..43eb0698 100644 --- a/detector/pattern/match_pattern_test.go +++ b/detector/pattern/match_pattern_test.go @@ -1,10 +1,12 @@ package pattern import ( - "github.com/stretchr/testify/assert" "regexp" + "talisman/detector/severity" "talisman/talismanrc" "testing" + + "github.com/stretchr/testify/assert" ) var ( @@ -15,23 +17,26 @@ var ( ) func TestShouldReturnEmptyStringWhenDoesNotMatchAnyRegex(t *testing.T) { - assert.Equal(t, "", NewPatternMatcher([]*regexp.Regexp{testRegexpPassword}).check("safeString")[0]) + detections := NewPatternMatcher([]*severity.PatternSeverity{{Pattern: testRegexpPassword, Severity: severity.Low()}}).check("safeString", severity.LowSeverity) + assert.Equal(t, []DetectionsWithSeverity(nil), detections) } func TestShouldReturnStringWhenMatchedPasswordPattern(t *testing.T) { - assert.Equal(t, []string{"password\" : 123456789"}, NewPatternMatcher([]*regexp.Regexp{testRegexpPassword}).check("password\" : 123456789")) - assert.Equal(t, []string{"pw\" : 123456789"}, NewPatternMatcher([]*regexp.Regexp{testRegexpPw}).check("pw\" : 123456789")) + detections1 := NewPatternMatcher([]*severity.PatternSeverity{{Pattern: testRegexpPassword, Severity: severity.Low()}}).check("password\" : 123456789", severity.LowSeverity) + detections2 := NewPatternMatcher([]*severity.PatternSeverity{{Pattern: testRegexpPw, Severity: severity.Medium()}}).check("pw\" : 123456789", severity.LowSeverity) + assert.Equal(t, []DetectionsWithSeverity{{detections: []string{"password\" : 123456789"}, severity: severity.Low()}}, detections1) + assert.Equal(t, []DetectionsWithSeverity{{detections: []string{"pw\" : 123456789"}, severity: severity.Medium()}}, detections2) } -func TestShouldAddGoodPatternToMatcher(t *testing.T) { - pm := NewPatternMatcher([]*regexp.Regexp{}) +func TestShouldAddGoodPatternWithHighSeverityToMatcher(t *testing.T) { + pm := NewPatternMatcher([]*severity.PatternSeverity{}) pm.add(talismanrc.PatternString(testRegexpPwPattern)) - assert.Equal(t, []string{"pw\" : 123456789"}, NewPatternMatcher([]*regexp.Regexp{testRegexpPw}).check("pw\" : 123456789")) + detections := pm.check("pw\" : 123456789", severity.LowSeverity) + assert.Equal(t, []DetectionsWithSeverity{{detections: []string{"pw\" : 123456789"}, severity: severity.High()}}, detections) } - func TestShouldNotAddBadPatternToMatcher(t *testing.T) { - pm := NewPatternMatcher([]*regexp.Regexp{}) + pm := NewPatternMatcher([]*severity.PatternSeverity{}) pm.add(`*a(crappy|regex`) assert.Equal(t, 0, len(pm.regexes)) } diff --git a/detector/pattern/pattern_detector.go b/detector/pattern/pattern_detector.go index 2e3756c5..dead2aed 100644 --- a/detector/pattern/pattern_detector.go +++ b/detector/pattern/pattern_detector.go @@ -5,6 +5,7 @@ import ( "regexp" "sync" "talisman/detector/helpers" + "talisman/detector/severity" "talisman/gitrepo" "talisman/talismanrc" @@ -16,14 +17,14 @@ type PatternDetector struct { } var ( - detectorPatterns = []*regexp.Regexp{ - regexp.MustCompile(`(?i)((.*)(password|passphrase|secret|key|pwd|pword|pass)(.*) *[:=>][^,;\n]{8,})`), - regexp.MustCompile(`(?i)(['"_]?pw['"]? *[:=][^,;\n]{8,})`), - regexp.MustCompile(`(?i)(\S*)`), - regexp.MustCompile(`(?i)(\S*)`), - regexp.MustCompile(`(?i)(AWS[ \w]+key[ \w]+[:=])`), - regexp.MustCompile(`(?i)(AWS[ \w]+secret[ \w]+[:=])`), - regexp.MustCompile(`(?s)(BEGIN RSA PRIVATE KEY.*END RSA PRIVATE KEY)`), + detectorPatterns = []*severity.PatternSeverity{ + {Pattern: regexp.MustCompile(`(?i)((.*)(password|passphrase|secret|key|pwd|pword|pass)(.*) *[:=>][^,;\n]{8,})`), Severity: severity.Medium()}, + {Pattern: regexp.MustCompile(`(?i)(['"_]?pw['"]? *[:=][^,;\n]{8,})`), Severity: severity.Medium()}, + {Pattern: regexp.MustCompile(`(?i)(\S*)`), Severity: severity.High()}, + {Pattern: regexp.MustCompile(`(?i)(\S*)`), Severity: severity.High()}, + {Pattern: regexp.MustCompile(`(?i)(AWS[ \w]+key[ \w]+[:=])`), Severity: severity.High()}, + {Pattern: regexp.MustCompile(`(?i)(AWS[ \w]+secret[ \w]+[:=])`), Severity: severity.High()}, + {Pattern: regexp.MustCompile(`(?s)(BEGIN RSA PRIVATE KEY.*END RSA PRIVATE KEY)`), Severity: severity.High()}, } ) @@ -31,7 +32,7 @@ type match struct { name gitrepo.FileName path gitrepo.FilePath commits []string - detections []string + detections []DetectionsWithSeverity } //Test tests the contents of the Additions to ensure that they don't look suspicious @@ -47,7 +48,7 @@ func (detector PatternDetector) Test(comparator helpers.ChecksumCompare, current ignoredFilePaths <- addition.Path return } - detections := detector.secretsPattern.check(processAllowedPatterns(addition, ignoreConfig)) + detections := detector.secretsPattern.check(processAllowedPatterns(addition, ignoreConfig), ignoreConfig.Threshold) matches <- match{name: addition.Name, path: addition.Path, detections: detections, commits: addition.Commits} }(addition) } @@ -63,7 +64,7 @@ func (detector PatternDetector) Test(comparator helpers.ChecksumCompare, current matchChanHasMore = false continue } - detector.processMatch(match, result) + detector.processMatch(match, result, ignoreConfig.Threshold) case ignore, hasMore := <-ignoredFilePaths: if !hasMore { ignoredChanHasMore = false @@ -103,21 +104,23 @@ func (detector PatternDetector) processIgnore(ignoredFilePath gitrepo.FilePath, result.Ignore(ignoredFilePath, "filecontent") } -func (detector PatternDetector) processMatch(match match, result *helpers.DetectionResults) { - for _, detection := range match.detections { - if detection != "" { - if string(match.name) == talismanrc.DefaultRCFileName { - log.WithFields(log.Fields{ - "filePath": match.path, - "pattern": detection, - }).Warn("Warning file as it matched pattern.") - result.Warn(match.path, "filecontent", fmt.Sprintf("Potential secret pattern : %s", detection), match.commits) - } else { - log.WithFields(log.Fields{ - "filePath": match.path, - "pattern": detection, - }).Info("Failing file as it matched pattern.") - result.Fail(match.path, "filecontent", fmt.Sprintf("Potential secret pattern : %s", detection), match.commits) +func (detector PatternDetector) processMatch(match match, result *helpers.DetectionResults, threshold severity.SeverityValue) { + for _, detectionWithSeverity := range match.detections { + for _, detection := range detectionWithSeverity.detections { + if detection != "" { + if string(match.name) == talismanrc.DefaultRCFileName || !detectionWithSeverity.severity.ExceedsThreshold(threshold) { + log.WithFields(log.Fields{ + "filePath": match.path, + "pattern": detection, + }).Warn("Warning file as it matched pattern.") + result.Warn(match.path, "filecontent", fmt.Sprintf("Potential secret pattern : %s", detection), match.commits, detectionWithSeverity.severity) + } else { + log.WithFields(log.Fields{ + "filePath": match.path, + "pattern": detection, + }).Info("Failing file as it matched pattern.") + result.Fail(match.path, "filecontent", fmt.Sprintf("Potential secret pattern : %s", detection), match.commits, detectionWithSeverity.severity) + } } } } diff --git a/detector/pattern/pattern_detector_test.go b/detector/pattern/pattern_detector_test.go index a9a4e151..998782b6 100644 --- a/detector/pattern/pattern_detector_test.go +++ b/detector/pattern/pattern_detector_test.go @@ -19,7 +19,7 @@ var ( func TestShouldDetectPasswordPatterns(t *testing.T) { filename := "secret.txt" - values := [7]string {"password","secret", "key", "pwd","pass","pword","passphrase"} + values := [7]string{"password", "secret", "key", "pwd", "pass", "pword", "passphrase"} for i := 0; i < len(values); i++ { shouldPassDetectionOfSecretPattern(filename, []byte(strings.ToTitle(values[i])+":UnsafeString"), t) shouldPassDetectionOfSecretPattern(filename, []byte(values[i]+"=UnsafeString"), t) @@ -70,6 +70,17 @@ func TestShouldIgnoreAllowedPattern(t *testing.T) { NewPatternDetector(customPatterns).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanrc.NewTalismanRC(nil)), additions, ignores, results) assert.True(t, results.Successful(), "Expected keywords %s to be ignored by Talisman", append(fileIgnoreConfig.AllowedPatterns, ignores.AllowedPatterns...)) } +func TestShouldOnlyWarnSecretPatternIfBelowThreshold(t *testing.T) { + results := helpers.NewDetectionResults() + content := []byte(`password=UnsafeString`) + filename := "secret.txt" + additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} + talismanRCContents := "threshold: high" + talismanRCWithThreshold := talismanrc.NewTalismanRC([]byte(talismanRCContents)) + NewPatternDetector(customPatterns).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanRCWithThreshold), additions, talismanRCWithThreshold, results) + assert.False(t, results.HasFailures(), "Expected file %s to not have failures", filename) + assert.True(t, results.HasWarnings(), "Expected file %s to have warnings", filename) +} func DetectionOfSecretPattern(filename string, content []byte) (*helpers.DetectionResults, []gitrepo.Addition, string) { results := helpers.NewDetectionResults() diff --git a/detector/severity/pattern_severity.go b/detector/severity/pattern_severity.go new file mode 100644 index 00000000..df097d6a --- /dev/null +++ b/detector/severity/pattern_severity.go @@ -0,0 +1,46 @@ +package severity + +import ( + "regexp" +) + +type SeverityValue int + +const ( + LowSeverity = SeverityValue(iota + 1) + MediumSeverity + HighSeverity +) + +type PatternSeverity struct { + Pattern *regexp.Regexp + Severity Severity +} + +type Severity struct { + Value SeverityValue +} + +func (s Severity) String() string { + return SeverityValueToString(s.Value) +} +func (s Severity) ExceedsThreshold(threshold SeverityValue) bool { + return s.Value >= threshold +} +func Low() Severity { + return Severity{ + Value: LowSeverity, + } +} + +func Medium() Severity { + return Severity{ + Value: MediumSeverity, + } +} + +func High() Severity { + return Severity{ + Value: HighSeverity, + } +} diff --git a/detector/severity/severity_map.go b/detector/severity/severity_map.go new file mode 100644 index 00000000..db7d1a5d --- /dev/null +++ b/detector/severity/severity_map.go @@ -0,0 +1,23 @@ +package severity + +import "strings" + +var severityMap = map[SeverityValue]string{ + LowSeverity: "low", + MediumSeverity: "medium", + HighSeverity: "high", +} + +func SeverityValueToString(severity SeverityValue) string { + return severityMap[severity] +} + +func SeverityStringToValue(severity string) SeverityValue { + severityInLowerCase := strings.ToLower(severity) + for k, v := range severityMap { + if v == severityInLowerCase { + return k + } + } + return 0 +} diff --git a/detector/severity/severity_map_test.go b/detector/severity/severity_map_test.go new file mode 100644 index 00000000..5192e564 --- /dev/null +++ b/detector/severity/severity_map_test.go @@ -0,0 +1,26 @@ +package severity + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestShouldReturnSeverityStringForDefinedSeverity(t *testing.T) { + assert.Equal(t, SeverityValueToString(LowSeverity), "low") + assert.Equal(t, SeverityValueToString(MediumSeverity), "medium") + assert.Equal(t, SeverityValueToString(HighSeverity), "high") +} +func TestShouldReturnEmptyForInvalidSeverity(t *testing.T) { + assert.Equal(t, SeverityValueToString(10), "") +} + +func TestShouldReturnSeverityValueForDefinedStrings(t *testing.T) { + assert.Equal(t, SeverityStringToValue("Low"), LowSeverity) + assert.Equal(t, SeverityStringToValue("MEDIUM"), MediumSeverity) + assert.Equal(t, SeverityStringToValue("high"), HighSeverity) +} + +func TestShouldReturnSeverityZeroForUnknownStrings(t *testing.T) { + assert.Equal(t, SeverityStringToValue("FakeSeverity"), SeverityValue(0)) +} diff --git a/go.mod b/go.mod index 5f73c080..8dcdb5bd 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/spf13/afero v1.2.2 github.com/spf13/pflag v1.0.3 github.com/stretchr/testify v1.4.0 + golang.org/x/text v0.3.3 // indirect gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/yaml.v2 v2.2.2 ) diff --git a/go.sum b/go.sum index 6a3966ab..27ccdce9 100644 --- a/go.sum +++ b/go.sum @@ -72,6 +72,9 @@ golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1 h1:R4dVlxdmKenVdMRS/tTspEpST golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190425150028-36563e24a262 h1:qsl9y/CJx34tuA7QCPNp86JNJe4spst6Ff8MjvPUdPg= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/talismanrc/talismanrc.go b/talismanrc/talismanrc.go index cd7f2e46..e66aca83 100644 --- a/talismanrc/talismanrc.go +++ b/talismanrc/talismanrc.go @@ -1,15 +1,17 @@ package talismanrc import ( - logr "github.com/Sirupsen/logrus" - "github.com/spf13/afero" - "gopkg.in/yaml.v2" "log" "os" "reflect" "regexp" "sort" + logr "github.com/Sirupsen/logrus" + "github.com/spf13/afero" + "gopkg.in/yaml.v2" + + "talisman/detector/severity" "talisman/gitrepo" ) @@ -24,7 +26,6 @@ var ( currentRCFileName = DefaultRCFileName ) - type FileIgnoreConfig struct { FileName string `yaml:"filename"` Checksum string `yaml:"checksum,omitempty"` @@ -43,11 +44,21 @@ type ExperimentalConfig struct { type PatternString string type TalismanRC struct { + FileIgnoreConfig []FileIgnoreConfig `yaml:"fileignoreconfig,omitempty"` + ScopeConfig []ScopeConfig `yaml:"scopeconfig,omitempty"` + CustomPatterns []PatternString `yaml:"custom_patterns,omitempty"` + AllowedPatterns []string `yaml:"allowed_patterns,omitempty"` + Experimental ExperimentalConfig `yaml:"experimental,omitempty"` + Threshold severity.SeverityValue `default:"1" yaml:"threshold,omitempty"` +} + +type TalismanRCFile struct { FileIgnoreConfig []FileIgnoreConfig `yaml:"fileignoreconfig,omitempty"` ScopeConfig []ScopeConfig `yaml:"scopeconfig,omitempty"` CustomPatterns []PatternString `yaml:"custom_patterns,omitempty"` AllowedPatterns []string `yaml:"allowed_patterns,omitempty"` Experimental ExperimentalConfig `yaml:"experimental,omitempty"` + Threshold string `default:"low" yaml:"threshold,omitempty"` } func SetFs(_fs afero.Fs) { @@ -81,14 +92,21 @@ func readRepoFile() func(string) ([]byte, error) { } func NewTalismanRC(fileContents []byte) *TalismanRC { - talismanRC := TalismanRC{} - err := yaml.Unmarshal(fileContents, &talismanRC) + talismanRCFile := TalismanRCFile{} + err := yaml.Unmarshal(fileContents, &talismanRCFile) if err != nil { log.Println("Unable to parse .talismanrc") log.Printf("error: %v", err) - return &talismanRC + return &TalismanRC{} + } + return &TalismanRC{ + FileIgnoreConfig: talismanRCFile.FileIgnoreConfig, + ScopeConfig: talismanRCFile.ScopeConfig, + CustomPatterns: talismanRCFile.CustomPatterns, + AllowedPatterns: talismanRCFile.AllowedPatterns, + Experimental: talismanRCFile.Experimental, + Threshold: severity.SeverityStringToValue(talismanRCFile.Threshold), } - return &talismanRC } func (i FileIgnoreConfig) isEffective(detectorName string) bool { diff --git a/talismanrc/talismanrc_test.go b/talismanrc/talismanrc_test.go index b0c1d297..23f1c172 100644 --- a/talismanrc/talismanrc_test.go +++ b/talismanrc/talismanrc_test.go @@ -3,6 +3,7 @@ package talismanrc import ( "testing" + "talisman/detector/severity" "talisman/gitrepo" "github.com/stretchr/testify/assert" @@ -20,6 +21,11 @@ func TestShouldIgnoreUnformattedFiles(t *testing.T) { } } +func TestShouldConvertThresholdToValue(t *testing.T) { + talismanRCContents := []byte("threshold: high") + assert.Equal(t, NewTalismanRC(talismanRCContents).Threshold, severity.HighSeverity) +} + func TestDirectoryPatterns(t *testing.T) { assertAccepts("foo/", "", "bar", t) assertAccepts("foo/", "", "foo", t) @@ -108,4 +114,4 @@ func CreatetalismanRCWithScopeIgnore(scopesToIgnore []string) *TalismanRC { talismanRC := TalismanRC{ScopeConfig: scopeConfigs} return &talismanRC -} \ No newline at end of file +}