#183 Custom patterns (#187)

* Rename TalismanRCIgnore to TalismanRC

* Code cleanup:
Rename NewtalismanRC to NewTalismanRC
Remove code and tests related to talismanignore

* Code cleanup: Make talismanRC receiver name consistent

* Introduce PatternString type
Omit empty fields when writing yaml

* #183 | Add ability to specify custom scan pattern via talismanrc

* Update Readme:
Update help text for CLI options (fix typo in -i message description)
Add custom_pattern section with example

.talismanrc

@@ -3,5 +3,8 @@ fileignoreconfig:
   checksum: 69fed055782cddfe0f0d23ea440cef9f9dd0b9e8a3c8a73856741bb26257b223
   ignore_detectors:
   - filecontent
+- filename: detector/match_pattern_test.go
+  checksum: d68aa0e06355e3b848941727d1fcb32cf47e3d615f9921f0db39855325010446
+  ignore_detectors: []
 scopeconfig:
 - scope: go

README.md

@@ -263,7 +263,7 @@ For example, if your `init-env.sh` filename triggers a warning, you can only dis
 this warning while still being alerted if other things go wrong (e.g. file content):
 
 
-```bash
+```yaml
 fileignoreconfig:
 - filename: init-env.sh
   checksum: cf97abd34cebe895417eb4d97fbd7374aa138dcb65b1fe7f6b6cc1238aaf4d48
@@ -295,7 +295,7 @@ If any of the files are modified, talisman will scan the files again, unless you
 
 You can choose to ignore files by specifying the language scope for your project in your talismanrc.
 
-```
+```yaml
 scopeconfig:
   - scope: go
   - scope: node
@@ -307,6 +307,16 @@ You can specify multiple scopes.
 
 Currently .talismanrc only supports scopeconfig support for go and node. Other scopes will be added shortly.
 
+### Custom search patterns
+
+You can specify custom regex patterns to look for in the current repository
+
+```yaml
+custom_patterns:
+- pattern1
+- pattern2
+```
+
 <br/><i>
 **Note**: The use of .talismanignore has been deprecated. File .talismanrc replaces it because:
 
@@ -319,9 +329,10 @@ Currently .talismanrc only supports scopeconfig support for go and node. Other s
 If you execute `talisman` on the command line, you will be able to view all the parameter options you can pass
 
 ```
-  -c, --checksum string          checksum calculator calculates checksum and suggests .talsimarc format
+  -c, --checksum string          checksum calculator calculates checksum and suggests .talismanrc format
   -d, --debug                    enable debug mode (warning: very verbose)
   -g, --githook string           either pre-push or pre-commit (default "pre-push")
+  -i, --interactive              interactively update talismanrc (only makes sense with -g/--githook)
   -p, --pattern string           pattern (glob-like) of files to scan (ignores githooks)
   -r, --reportdirectory string   directory where the scan reports will be stored
   -s, --scan                     scanner scans the git commit history for potential secrets

checksumcalculator/checksumcalculator.go

@@ -38,8 +38,8 @@ func (cc *ChecksumCalculator) SuggestTalismanRC() string {
 	}
 	if len(fileIgnoreConfigs) != 0 {
 		result = result + fmt.Sprintf("\n\x1b[33m.talismanrc format for given file names / patterns\x1b[0m\n")
-		talismanRCIgnoreConfig := talismanrc.TalismanRCIgnore{FileIgnoreConfig: fileIgnoreConfigs}
-		m, _ := yaml.Marshal(&talismanRCIgnoreConfig)
+		talismanRCConfig := talismanrc.TalismanRC{FileIgnoreConfig: fileIgnoreConfigs}
+		m, _ := yaml.Marshal(&talismanRCConfig)
 		result = result + string(m)
 	}
 	return result

detector/base64_aggressive_detector_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-var talismanRCIgnore = &talismanrc.TalismanRCIgnore{}
+var talismanRC = &talismanrc.TalismanRC{}
 
 func TestShouldFlagPotentialAWSAccessKeysInAggressiveMode(t *testing.T) {
 	const awsAccessKeyIDExample string = "AKIAIOSFODNN7EXAMPLE\n"
@@ -18,7 +18,7 @@ func TestShouldFlagPotentialAWSAccessKeysInAggressiveMode(t *testing.T) {
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().AggressiveMode().Test(additions, talismanRCIgnore, results)
+	NewFileContentDetector().AggressiveMode().Test(additions, talismanRC, results)
 	assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 }
 
@@ -29,7 +29,7 @@ func TestShouldFlagPotentialAWSAccessKeysAtPropertyDefinitionInAggressiveMode(t
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().AggressiveMode().Test(additions, talismanRCIgnore, results)
+	NewFileContentDetector().AggressiveMode().Test(additions, talismanRC, results)
 	assert.True(t, results.HasFailures(), "Expected file to not to contain base64 encoded texts")
 }
 
@@ -40,7 +40,7 @@ func TestShouldNotFlagPotentialSecretsWithinSafeJavaCodeEvenInAggressiveMode(t *
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
-	NewFileContentDetector().AggressiveMode().Test(additions, talismanRCIgnore, results)
+	NewFileContentDetector().AggressiveMode().Test(additions, talismanRC, results)
 	if results == nil {
 		additions = nil
 	}

detector/checksum_compare.go

@@ -8,12 +8,12 @@ import (
 
 type ChecksumCompare struct {
 	additions    []gitrepo.Addition
-	ignoreConfig *talismanrc.TalismanRCIgnore
+	ignoreConfig *talismanrc.TalismanRC
 }
 
 //NewChecksumCompare returns new instance of the ChecksumCompare
-func NewChecksumCompare(gitAdditions []gitrepo.Addition, talismanRCIgnoreConfig *talismanrc.TalismanRCIgnore) *ChecksumCompare {
-	cc := ChecksumCompare{additions: gitAdditions, ignoreConfig: talismanRCIgnoreConfig}
+func NewChecksumCompare(gitAdditions []gitrepo.Addition, talismanRCConfig *talismanrc.TalismanRC) *ChecksumCompare {
+	cc := ChecksumCompare{additions: gitAdditions, ignoreConfig: talismanRCConfig}
 	return &cc
 }
 
@@ -31,8 +31,8 @@ func (cc *ChecksumCompare) IsScanNotRequired(addition gitrepo.Addition) bool {
 
 }
 
-//FilterIgnoresBasedOnChecksums filters the file ignores from the talismanrc.TalismanRCIgnore which doesn't have any checksum value or having mismatched checksum value from the .talsimanrc
-func (cc *ChecksumCompare) FilterIgnoresBasedOnChecksums() talismanrc.TalismanRCIgnore {
+//FilterIgnoresBasedOnChecksums filters the file ignores from the talismanrc.TalismanRC which doesn't have any checksum value or having mismatched checksum value from the .talsimanrc
+func (cc *ChecksumCompare) FilterIgnoresBasedOnChecksums() talismanrc.TalismanRC {
 	finalIgnores := []talismanrc.FileIgnoreConfig{}
 	for _, ignore := range cc.ignoreConfig.FileIgnoreConfig {
 		currentCollectiveChecksum := cc.calculateCollectiveChecksumForPattern(ignore.FileName, cc.additions)
@@ -41,7 +41,7 @@ func (cc *ChecksumCompare) FilterIgnoresBasedOnChecksums() talismanrc.TalismanRC
 			finalIgnores = append(finalIgnores, ignore)
 		}
 	}
-	rc := talismanrc.TalismanRCIgnore{}
+	rc := talismanrc.TalismanRC{}
 	rc.FileIgnoreConfig = finalIgnores
 	return rc
 }

detector/checksum_compare_test.go

@@ -37,7 +37,7 @@ fileignoreconfig:
 `
 
 func TestShouldConsiderBothFilesForDetection(t *testing.T) {
-	rc := talismanrc.NewTalismanRCIgnore([]byte(talismanRCWithInCorrectChecksum))
+	rc := talismanrc.NewTalismanRC([]byte(talismanRCWithInCorrectChecksum))
 	addition1 := gitrepo.NewAddition("some_file.pem", make([]byte, 0))
 	addition2 := gitrepo.NewAddition("test/some_file.pem", make([]byte, 0))
 	cc := NewChecksumCompare([]gitrepo.Addition{addition1, addition2}, rc)
@@ -50,7 +50,7 @@ func TestShouldConsiderBothFilesForDetection(t *testing.T) {
 func TestShouldNotConsiderBothFilesForDetection(t *testing.T) {
 	addition1 := gitrepo.NewAddition("some_file.pem", make([]byte, 0))
 	addition2 := gitrepo.NewAddition("test/some_file.pem", make([]byte, 0))
-	rc := talismanrc.NewTalismanRCIgnore([]byte(talismanRCWithCorrectChecksum))
+	rc := talismanrc.NewTalismanRC([]byte(talismanRCWithCorrectChecksum))
 	cc := NewChecksumCompare([]gitrepo.Addition{addition1, addition2}, rc)
 
 	filteredRC := cc.FilterIgnoresBasedOnChecksums()
@@ -61,7 +61,7 @@ func TestShouldNotConsiderBothFilesForDetection(t *testing.T) {
 func TestShouldConsiderOneFileForDetection(t *testing.T) {
 	addition1 := gitrepo.NewAddition("some_file.pem", make([]byte, 0))
 	addition2 := gitrepo.NewAddition("test/some1_file.pem", make([]byte, 0))
-	rc := talismanrc.NewTalismanRCIgnore([]byte(talismanRCWithOneCorrectChecksum))
+	rc := talismanrc.NewTalismanRC([]byte(talismanRCWithOneCorrectChecksum))
 	cc := NewChecksumCompare([]gitrepo.Addition{addition1, addition2}, rc)
 
 	filteredRC := cc.FilterIgnoresBasedOnChecksums()
@@ -72,7 +72,7 @@ func TestShouldConsiderOneFileForDetection(t *testing.T) {
 func TestShouldConsiderBothFilesForDetectionIfTalismanRCIsEmpty(t *testing.T) {
 	addition1 := gitrepo.NewAddition("some_file.pem", make([]byte, 0))
 	addition2 := gitrepo.NewAddition("test/some_file.pem", make([]byte, 0))
-	rc := talismanrc.NewTalismanRCIgnore([]byte{})
+	rc := talismanrc.NewTalismanRC([]byte{})
 	cc := NewChecksumCompare([]gitrepo.Addition{addition1, addition2}, rc)
 
 	filteredRC := cc.FilterIgnoresBasedOnChecksums()

detector/detection_results.go

@@ -333,8 +333,8 @@ func getUserConfirmation(configs []talismanrc.FileIgnoreConfig, promptContext pr
 }
 
 func printTalismanIgnoreSuggestion(entriesToAdd []talismanrc.FileIgnoreConfig) {
-	talismanRcIgnoreConfig := talismanrc.TalismanRCIgnore{FileIgnoreConfig: entriesToAdd}
-	ignoreEntries, _ := yaml.Marshal(&talismanRcIgnoreConfig)
+	talismanRCConfig := talismanrc.TalismanRC{FileIgnoreConfig: entriesToAdd}
+	ignoreEntries, _ := yaml.Marshal(&talismanRCConfig)
 	suggestString := fmt.Sprintf("\n\x1b[33mIf you are absolutely sure that you want to ignore the " +
 		"above files from talisman detectors, consider pasting the following format in .talismanrc file" +
 		" in the project root\x1b[0m\n")

detector/detection_results_test.go

@@ -86,8 +86,6 @@ func TestTalismanRCSuggestionWhenThereAreFailures(t *testing.T) {
 	existingContent := `fileignoreconfig:
 - filename: existing.pem
   checksum: 123444ddssa75333b25b6275f97680604add51b84eb8f4a3b9dcbbc652e6f27ac
-  ignore_detectors: []
-scopeconfig: []
 `
 	err = afero.WriteFile(fs, ignoreFile, []byte(existingContent), 0666)
 	assert.NoError(t, err)
@@ -141,8 +139,6 @@ scopeconfig: []
 		expectedFileContent := `fileignoreconfig:
 - filename: some_file.pem
   checksum: 87139cc4d975333b25b6275f97680604add51b84eb8f4a3b9dcbbc652e6f27ac
-  ignore_detectors: []
-scopeconfig: []
 `
 		results.Report(fs, ignoreFile, promptContext)
 		bytesFromFile, err := afero.ReadFile(fs, ignoreFile)
@@ -161,8 +157,6 @@ scopeconfig: []
 		expectedFileContent := `fileignoreconfig:
 - filename: existing.pem
   checksum: 5bc0b0692a316bb2919263addaef0ffba3a21b9e1cca62a1028390e97e861e4e
-  ignore_detectors: []
-scopeconfig: []
 
 `
 		results.Report(fs, ignoreFile, promptContext)
@@ -183,11 +177,8 @@ scopeconfig: []
 		expectedFileContent := `fileignoreconfig:
 - filename: another.pem
   checksum: 117e23557c02cbd472854ebce4933d6daec1fd207971286f6ffc9f1774c1a83b
-  ignore_detectors: []
 - filename: some_file.pem
   checksum: 87139cc4d975333b25b6275f97680604add51b84eb8f4a3b9dcbbc652e6f27ac
-  ignore_detectors: []
-scopeconfig: []
 `
 		results.Report(fs, ignoreFile, promptContext)
 		bytesFromFile, err := afero.ReadFile(fs, ignoreFile)

detector/detector.go

@@ -10,7 +10,7 @@ import (
 //Detectors are expected to honor the ignores that are passed in and log them in the results
 //Detectors are expected to signal any errors to the results
 type Detector interface {
-	Test(additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRCIgnore, result *DetectionResults)
+	Test(additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *DetectionResults)
 }
 
 //Chain represents a chain of Detectors.
@@ -27,11 +27,11 @@ func NewChain() *Chain {
 }
 
 //DefaultChain returns a DetectorChain with pre-configured detectors
-func DefaultChain() *Chain {
+func DefaultChain(tRC *talismanrc.TalismanRC) *Chain {
 	result := NewChain()
 	result.AddDetector(DefaultFileNameDetector())
 	result.AddDetector(NewFileContentDetector())
-	result.AddDetector(NewPatternDetector())
+	result.AddDetector(NewPatternDetector(tRC.CustomPatterns))
 	return result
 }
 
@@ -43,7 +43,7 @@ func (dc *Chain) AddDetector(d Detector) *Chain {
 
 //Test validates the additions against each detector in the chain.
 //The results are passed in from detector to detector and thus collect all errors from all detectors
-func (dc *Chain) Test(additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRCIgnore, result *DetectionResults) {
+func (dc *Chain) Test(additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *DetectionResults) {
 	wd, _ := os.Getwd()
 	repo := gitrepo.RepoLocatedAt(wd)
 	gitTrackedFilesAsAdditions := repo.TrackedFilesAsAdditions()

detector/detector_test.go

@@ -12,7 +12,7 @@ import (
 func TestEmptyValidationChainPassesAllValidations(t *testing.T) {
 	v := NewChain()
 	results := NewDetectionResults()
-	v.Test(nil, &talismanrc.TalismanRCIgnore{}, results)
+	v.Test(nil, &talismanrc.TalismanRC{}, results)
 	assert.False(t, results.HasFailures(), "Empty validation chain is expected to always pass")
 }
 
@@ -21,18 +21,18 @@ func TestValidationChainWithFailingValidationAlwaysFails(t *testing.T) {
 	v.AddDetector(PassingDetection{})
 	v.AddDetector(FailingDetection{})
 	results := NewDetectionResults()
-	v.Test(nil, &talismanrc.TalismanRCIgnore{}, results)
+	v.Test(nil, &talismanrc.TalismanRC{}, results)
 
 	assert.False(t, results.Successful(), "Expected validation chain with a failure to fail.")
 }
 
 type FailingDetection struct{}
 
-func (v FailingDetection) Test(additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRCIgnore, result *DetectionResults) {
+func (v FailingDetection) Test(additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *DetectionResults) {
 	result.Fail("some_file", "filecontent", "FAILED BY DESIGN", []string{})
 }
 
 type PassingDetection struct{}
 
-func (p PassingDetection) Test(additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRCIgnore, result *DetectionResults) {
+func (p PassingDetection) Test(additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *DetectionResults) {
 }

detector/filecontent_detector.go

@@ -72,7 +72,7 @@ type content struct {
 	results     []string
 }
 
-func (fc *FileContentDetector) Test(additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRCIgnore, result *DetectionResults) {
+func (fc *FileContentDetector) Test(additions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *DetectionResults) {
 	contentTypes := []struct {
 		contentType
 		fn