Skip to content

Commit

Permalink
Merge a4b78bc into 27bc869
Browse files Browse the repository at this point in the history
  • Loading branch information
tinamthomas committed Jul 25, 2020
2 parents 27bc869 + a4b78bc commit c13ef92
Show file tree
Hide file tree
Showing 20 changed files with 448 additions and 253 deletions.
14 changes: 13 additions & 1 deletion .talismanrc
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
fileignoreconfig:
- filename: detector/pattern_detector_test.go
checksum: fbb55705e75e1fcba609b10c454e277e5c67c930bd5aedd514055570e2b868b4
checksum: 4d70b790f28f2d23d506f808d489aa43f1efd2514549ae6a83a535e1223382e3
- filename: detector/detection_results_test.go
checksum: 69fed055782cddfe0f0d23ea440cef9f9dd0b9e8a3c8a73856741bb26257b223
ignore_detectors:
Expand All @@ -10,5 +10,17 @@ fileignoreconfig:
ignore_detectors: []
- filename: global_install_scripts/install.bash
checksum: 5d659125ecbe619ea99f5bc71c2d761b586ce3ec9ccab7683ee54f4ebde9f748
- filename: detector/filecontent/filecontent_detector_test.go
checksum: affb25839a87476dcef4f4169ccb9b54b2d2f2437cef3aca24f4d3b69d5886c5
- filename: detector/pattern/match_pattern_test.go
checksum: b90530d286fbc0ee864d2350fc0c532e0fb2f01149d51e81339b420439014238
- filename: detector/pattern/pattern_detector_test.go
checksum: 4d70b790f28f2d23d506f808d489aa43f1efd2514549ae6a83a535e1223382e3
- filename: detector/pattern/pattern_detector.go
checksum: 248bc5f67fa12d39b0fa1b63319a5b125006858a11603a837d8c53dbab2277c3
- filename: detector/filename/filename_detector.go
checksum: 5782cb11c373723ec7b40279a3dd375c0cd1d285ac0d032599f0300d9e133eec
- filename: detector/filename/filename_detector_test.go
checksum: 0a9c9f113e203ca29d3a9bf0b4802a252e990c2132e1f168a46ab49ed532e6c9
scopeconfig:
- scope: go
24 changes: 21 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,10 @@
- [Talisman in action](#talisman-in-action)
- [Validations](#validations)
- [Ignoring files](#ignoring-files)
- [Talisman as a CLI utility](#talisman-as-a-cli-utility)
- [Git History Scanner](#git-history-scanner)
- [Checksum Calculator](#checksum-calculator)
- [Configuring severity threshold](#configuring-severity-threshold)
- [Talisman as a CLI utility](#talisman-as-a-cli-utility)
- [Git History Scanner](#git-history-scanner)
- [Checksum Calculator](#checksum-calculator)
- [Talisman HTML Reporting](#talisman-html-reporting)
- [Uninstallation](#uninstallation)
- [From a global hook template](#uninstallation-from-a-global-hook-template)
Expand Down Expand Up @@ -351,6 +352,23 @@ custom_patterns:
* It also brings in more secure practices with every modification of a file with a potential sensitive value to be reviewed
* The new format also brings in the extensibility to introduce new usable functionalities. Keep a watch out for more </i>

## Configuring severity threshold

Each validation is associated with a severity value

1. Low Severity
2. Medium Severity
3. High Severity

You can specify a threshold in your .talismanrc:

```yaml
threshold: 2
```
This will report all Medium severity issues and higher (Potential risks that are below the threshold will be reported in the warnings)

By default, the threshold is set to 1

## Talisman as a CLI utility

If you execute `talisman` on the command line, you will be able to view all the parameter options you can pass
Expand Down
3 changes: 1 addition & 2 deletions detector/chain.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ func NewChain() *Chain {
//DefaultChain returns a DetectorChain with pre-configured detectors
func DefaultChain(tRC *talismanrc.TalismanRC) *Chain {
result := NewChain()
result.AddDetector(filename.DefaultFileNameDetector())
result.AddDetector(filename.DefaultFileNameDetector(tRC.Threshold))
result.AddDetector(filecontent.NewFileContentDetector(tRC))
result.AddDetector(pattern.NewPatternDetector(tRC.CustomPatterns))
return result
Expand All @@ -54,4 +54,3 @@ func (dc *Chain) Test(currentAdditions []gitrepo.Addition, talismanRC *talismanr
v.Test(cc, currentAdditions, talismanRC, result)
}
}

6 changes: 4 additions & 2 deletions detector/chain_test.go
Original file line number Diff line number Diff line change
@@ -1,17 +1,19 @@
package detector

import (
"github.com/stretchr/testify/assert"
"talisman/detector/helpers"
"talisman/detector/severity"
"talisman/gitrepo"
"talisman/talismanrc"
"testing"

"github.com/stretchr/testify/assert"
)

type FailingDetection struct{}

func (v FailingDetection) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults) {
result.Fail("some_file", "filecontent", "FAILED BY DESIGN", []string{})
result.Fail("some_file", "filecontent", "FAILED BY DESIGN", []string{}, severity.Low())
}

type PassingDetection struct{}
Expand Down
17 changes: 12 additions & 5 deletions detector/filecontent/filecontent_detector.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"strings"
"sync"
"talisman/detector/helpers"
"talisman/detector/severity"
"talisman/gitrepo"
"talisman/talismanrc"

Expand Down Expand Up @@ -72,24 +73,29 @@ type content struct {
path gitrepo.FilePath
contentType contentType
results []string
severity severity.Severity
}

func (fc *FileContentDetector) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults) {
contentTypes := []struct {
contentType
fn
severity severity.Severity
}{
{
contentType: base64Content,
fn: checkBase64,
severity: severity.Medium(),
},
{
contentType: hexContent,
fn: checkHex,
severity: severity.Medium(),
},
{
contentType: creditCardContent,
fn: checkCreditCardNumber,
severity: severity.High(),
},
}
re := regexp.MustCompile(`(?i)checksum[ \t]*:[ \t]*[0-9a-fA-F]+`)
Expand Down Expand Up @@ -118,6 +124,7 @@ func (fc *FileContentDetector) Test(comparator helpers.ChecksumCompare, currentA
path: addition.Path,
contentType: ct.contentType,
results: fc.detectFile(addition.Data, ct.fn),
severity: ct.severity,
}
}
}(addition)
Expand All @@ -141,7 +148,7 @@ func (fc *FileContentDetector) Test(comparator helpers.ChecksumCompare, currentA
contentChanHasMore = false
continue
}
processContent(c, result)
processContent(c, ignoreConfig.Threshold, result)
}
}
}
Expand All @@ -153,16 +160,16 @@ func processIgnoredFilepath(path gitrepo.FilePath, result *helpers.DetectionResu
result.Ignore(path, "filecontent")
}

func processContent(c content, result *helpers.DetectionResults) {
func processContent(c content, threshold severity.SeverityValue, result *helpers.DetectionResults) {
for _, res := range c.results {
if res != "" {
log.WithFields(log.Fields{
"filePath": c.path,
}).Info(c.contentType.getInfo())
if string(c.name) == talismanrc.DefaultRCFileName {
result.Warn(c.path, "filecontent", fmt.Sprintf(c.contentType.getMessageFormat(), formatForReporting(res)), []string{})
if string(c.name) == talismanrc.DefaultRCFileName || !c.severity.ExceedsThreshold(threshold) {
result.Warn(c.path, "filecontent", fmt.Sprintf(c.contentType.getMessageFormat(), formatForReporting(res)), []string{}, c.severity)
} else {
result.Fail(c.path, "filecontent", fmt.Sprintf(c.contentType.getMessageFormat(), formatForReporting(res)), []string{})
result.Fail(c.path, "filecontent", fmt.Sprintf(c.contentType.getMessageFormat(), formatForReporting(res)), []string{}, c.severity)
}
}
}
Expand Down
29 changes: 29 additions & 0 deletions detector/filecontent/filecontent_detector_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,20 @@ func TestShouldFlagPotentialAWSSecretKeys(t *testing.T) {
assert.Len(t, results.Results, 1)
}

func TestShouldNotFlagBase64ContentIfThresholdIsHigher(t *testing.T) {
const awsSecretAccessKey string = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
results := helpers.NewDetectionResults()
content := []byte(awsSecretAccessKey)
filename := "filename"
additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
var talismanRCContents = "threshold: 3"
talismanRCWithThreshold := talismanrc.NewTalismanRC([]byte(talismanRCContents))

NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanRCWithThreshold), additions, talismanRCWithThreshold, results)
assert.False(t, results.HasFailures(), "Expected file to not flag base64 encoded texts if threshold is higher")
assert.True(t, results.HasWarnings(), "Expected file to have warngings for base64 encoded texts if threshold is higher")
}

func TestShouldFlagPotentialSecretWithoutTrimmingWhenLengthLessThan50Characters(t *testing.T) {
const secret string = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9asdfa"
results := helpers.NewDetectionResults()
Expand Down Expand Up @@ -160,6 +174,21 @@ func TestShouldFlagPotentialSecretsEncodedInHex(t *testing.T) {
assert.Len(t, results.Results, 1)
}

func TestShouldNotFlagSecretsEncodedInHexIfAboveThreshold(t *testing.T) {
const hex string = "68656C6C6F20776F726C6421"
results := helpers.NewDetectionResults()
content := []byte(hex)
filename := "filename"
additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}

var talismanRCContents = "threshold: 3"
talismanRCWithThreshold := talismanrc.NewTalismanRC([]byte(talismanRCContents))

NewFileContentDetector(talismanRC).Test(helpers.NewChecksumCompare(nil, utility.DefaultSHA256Hasher{}, talismanRCWithThreshold), additions, talismanRCWithThreshold, results)

assert.False(t, results.HasFailures(), "Expected file to not flag base64 encoded texts if threshold is higher")
}

func TestResultsShouldContainHexTextsIfHexAndBase64ExistInFile(t *testing.T) {
const hex string = "68656C6C6F20776F726C6421"
const base64 string = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
Expand Down
121 changes: 64 additions & 57 deletions detector/filename/filename_detector.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"regexp"
"talisman/detector/detector"
"talisman/detector/helpers"
"talisman/detector/severity"

"talisman/gitrepo"
"talisman/talismanrc"
Expand All @@ -13,71 +14,72 @@ import (
)

var (
filenamePatterns = []*regexp.Regexp{
regexp.MustCompile(`^.+_rsa$`),
regexp.MustCompile(`^.+_dsa.*$`),
regexp.MustCompile(`^.+_ed25519$`),
regexp.MustCompile(`^.+_ecdsa$`),
regexp.MustCompile(`^\.\w+_history$`),
regexp.MustCompile(`^.+\.pem$`),
regexp.MustCompile(`^.+\.ppk$`),
regexp.MustCompile(`^.+\.key(pair)?$`),
regexp.MustCompile(`^.+\.pkcs12$`),
regexp.MustCompile(`^.+\.pfx$`),
regexp.MustCompile(`^.+\.p12$`),
regexp.MustCompile(`^.+\.asc$`),
regexp.MustCompile(`^\.?htpasswd$`),
regexp.MustCompile(`^\.?netrc$`),
regexp.MustCompile(`^.*\.tblk$`),
regexp.MustCompile(`^.*\.ovpn$`),
regexp.MustCompile(`^.*\.kdb$`),
regexp.MustCompile(`^.*\.agilekeychain$`),
regexp.MustCompile(`^.*\.keychain$`),
regexp.MustCompile(`^.*\.key(store|ring)$`),
regexp.MustCompile(`^jenkins\.plugins\.publish_over_ssh\.BapSshPublisherPlugin.xml$`),
regexp.MustCompile(`^credentials\.xml$`),
regexp.MustCompile(`^.*\.pubxml(\.user)?$`),
regexp.MustCompile(`^\.?s3cfg$`),
regexp.MustCompile(`^\.gitrobrc$`),
regexp.MustCompile(`^\.?(bash|zsh)rc$`),
regexp.MustCompile(`^\.?(bash_|zsh_)?profile$`),
regexp.MustCompile(`^\.?(bash_|zsh_)?aliases$`),
regexp.MustCompile(`^secret_token.rb$`),
regexp.MustCompile(`^omniauth.rb$`),
regexp.MustCompile(`^carrierwave.rb$`),
regexp.MustCompile(`^schema.rb$`),
regexp.MustCompile(`^database.yml$`),
regexp.MustCompile(`^settings.py$`),
regexp.MustCompile(`^.*(config)(\.inc)?\.php$`),
regexp.MustCompile(`^LocalSettings.php$`),
regexp.MustCompile(`\.?env`),
regexp.MustCompile(`\bdump|dump\b`),
regexp.MustCompile(`\bsql|sql\b`),
regexp.MustCompile(`\bdump|dump\b`),
regexp.MustCompile(`password`),
regexp.MustCompile(`backup`),
regexp.MustCompile(`private.*key`),
regexp.MustCompile(`(oauth).*(token)`),
regexp.MustCompile(`^.*\.log$`),
regexp.MustCompile(`^\.?kwallet$`),
regexp.MustCompile(`^\.?gnucash$`),
filenamePatterns = []*severity.PatternSeverity{
{Pattern: regexp.MustCompile(`^.+_rsa$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^.+_dsa.*$`), Severity: severity.High()},
{Pattern: regexp.MustCompile(`^.+_ed25519$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^.+_ecdsa$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^\.\w+_history$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^.+\.pem$`), Severity: severity.High()},
{Pattern: regexp.MustCompile(`^.+\.ppk$`), Severity: severity.High()},
{Pattern: regexp.MustCompile(`^.+\.key(pair)?$`), Severity: severity.High()},
{Pattern: regexp.MustCompile(`^.+\.pkcs12$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^.+\.pfx$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^.+\.p12$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^.+\.asc$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^\.?htpasswd$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^\.?netrc$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^.*\.tblk$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^.*\.ovpn$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^.*\.kdb$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^.*\.agilekeychain$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^.*\.keychain$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^.*\.key(store|ring)$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^jenkins\.plugins\.publish_over_ssh\.BapSshPublisherPlugin.xml$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^credentials\.xml$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^.*\.pubxml(\.user)?$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^\.?s3cfg$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^\.gitrobrc$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^\.?(bash|zsh)rc$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^\.?(bash_|zsh_)?profile$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^\.?(bash_|zsh_)?aliases$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^secret_token.rb$`), Severity: severity.High()},
{Pattern: regexp.MustCompile(`^omniauth.rb$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^carrierwave.rb$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^schema.rb$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^database.yml$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^settings.py$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^.*(config)(\.inc)?\.php$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^LocalSettings.php$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`\.?env`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`\bdump|dump\b`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`\bsql|sql\b`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`\bdump|dump\b`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`password`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`backup`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`private.*key`), Severity: severity.High()},
{Pattern: regexp.MustCompile(`(oauth).*(token)`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^.*\.log$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^\.?kwallet$`), Severity: severity.Low()},
{Pattern: regexp.MustCompile(`^\.?gnucash$`), Severity: severity.Low()},
}
)

//FileNameDetector represents tests performed against the fileName of the Additions.
//The Paths of the supplied Additions are tested against the configured patterns and if any of them match, it is logged as a failure during the run
type FileNameDetector struct {
flagPatterns []*regexp.Regexp
flagPatterns []*severity.PatternSeverity
threshold severity.SeverityValue
}

//DefaultFileNameDetector returns a FileNameDetector that tests Additions against the pre-configured patterns
func DefaultFileNameDetector() detector.Detector {
return NewFileNameDetector(filenamePatterns)
func DefaultFileNameDetector(threshold severity.SeverityValue) detector.Detector {
return NewFileNameDetector(filenamePatterns, threshold)
}

//NewFileNameDetector returns a FileNameDetector that tests Additions against the supplied patterns
func NewFileNameDetector(patterns []*regexp.Regexp) detector.Detector {
return FileNameDetector{patterns}
func NewFileNameDetector(patternsWithSeverity []*severity.PatternSeverity, threshold severity.SeverityValue) detector.Detector {
return FileNameDetector{patternsWithSeverity, threshold}
}

//Test tests the fileNames of the Additions to ensure that they don't look suspicious
Expand All @@ -90,13 +92,18 @@ func (fd FileNameDetector) Test(comparator helpers.ChecksumCompare, currentAddit
result.Ignore(addition.Path, "filename")
continue
}
for _, pattern := range fd.flagPatterns {
if pattern.MatchString(string(addition.Name)) {
for _, patternWithSeverity := range fd.flagPatterns {
if patternWithSeverity.Pattern.MatchString(string(addition.Name)) {
log.WithFields(log.Fields{
"filePath": addition.Path,
"pattern": pattern,
"pattern": patternWithSeverity.Pattern,
"severity": patternWithSeverity.Severity,
}).Info("Failing file as it matched pattern.")
result.Fail(addition.Path, "filename", fmt.Sprintf("The file name %q failed checks against the pattern %s", addition.Path, pattern), addition.Commits)
if patternWithSeverity.Severity.ExceedsThreshold(fd.threshold) {
result.Fail(addition.Path, "filename", fmt.Sprintf("The file name %q failed checks against the pattern %s", addition.Path, patternWithSeverity.Pattern), addition.Commits, patternWithSeverity.Severity)
} else {
result.Warn(addition.Path, "filename", fmt.Sprintf("The file name %q failed checks against the pattern %s", addition.Path, patternWithSeverity.Pattern), addition.Commits, patternWithSeverity.Severity)
}
}
}
}
Expand Down
Loading

0 comments on commit c13ef92

Please sign in to comment.