Issue #254 | Implementignore for scans

- Separate runtime configuration from persisted configuration in talismanRC
- Fix JSON Marshalling of severity values
- Remove Mode member from TalismanRC
- Upgrade yaml and testify versions

.gitignore

@@ -14,4 +14,9 @@ coverage.txt
 .talismanrc
 talisman_reports/
 *.iml
-dist/
+dist/
+*.out
+*.pprof
+*.mprof
+.envrc
+talisman_html_report

checksumcalculator/checksumcalculator.go

@@ -6,7 +6,7 @@ import (
 	"talisman/talismanrc"
 	"talisman/utility"
 
-	yaml "gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v2"
 )
 
 type ChecksumCalculator interface {
@@ -45,6 +45,7 @@ func (cc *DefaultChecksumCalculator) SuggestTalismanRC(fileNamePatterns []string
 	return result
 }
 
+//CalculateCollectiveChecksumForPattern calculates and returns the checksum for files matching the input pattern
 func (cc *DefaultChecksumCalculator) CalculateCollectiveChecksumForPattern(fileNamePattern string) string {
 	var patternPaths []string
 	currentCollectiveChecksum := ""

cmd/runner.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-	"github.com/spf13/afero"
 	"talisman/detector"
 	"talisman/detector/helpers"
 	"talisman/detector/severity"
@@ -13,10 +12,10 @@ import (
 
 const (
 	//CompletedSuccessfully is an exit status that says that the current runners run completed without errors
-	CompletedSuccessfully int = 0
+	CompletedSuccessfully = 0
 
 	//CompletedWithErrors is an exit status that says that the current runners run completed with failures
-	CompletedWithErrors int = 1
+	CompletedWithErrors = 1
 )
 
 //runner represents a single run of the validations for a given commit range
@@ -29,7 +28,7 @@ type runner struct {
 func NewRunner(additions []gitrepo.Addition) *runner {
 	return &runner{
 		additions: additions,
-		results:   helpers.NewDetectionResults(talismanrc.Hook),
+		results:   helpers.NewDetectionResults(talismanrc.HookMode),
 	}
 }
 
@@ -54,8 +53,7 @@ func (r *runner) printReport(promptContext prompt.PromptContext) {
 		fmt.Println(r.results.ReportWarnings())
 	}
 	if r.results.HasIgnores() || r.results.HasFailures() {
-		fs := afero.NewOsFs()
-		r.results.Report(fs, talismanrc.DefaultRCFileName, promptContext)
+		r.results.Report(promptContext)
 	}
 }
 

cmd/scanner_cmd.go

@@ -21,10 +21,7 @@ type ScannerCmd struct {
 //Run scans git commit history for potential secrets and returns 0 or 1 as exit code
 func (s *ScannerCmd) Run(tRC *talismanrc.TalismanRC) int {
 	fmt.Printf("\n\n")
-	utility.CreateArt("Running Scan..")
-
-	setCustomSeverities(tRC)
-	tRC.SetMode(talismanrc.Scan)
+	utility.CreateArt("Running ScanMode..")
 	detector.DefaultChain(tRC).Test(s.additions, tRC, s.results)
 	reportsPath, err := report.GenerateReport(s.results, s.reportDirectory)
 	if err != nil {
@@ -43,12 +40,12 @@ func (s *ScannerCmd) exitStatus() int {
 	return CompletedSuccessfully
 }
 
-//Returns a new scanner command
-func NewScannerCmd(ignoreHistory bool, reportDirectory string, mode talismanrc.Mode) *ScannerCmd {
+//NewScannerCmd Returns a new scanner command
+func NewScannerCmd(ignoreHistory bool, reportDirectory string) *ScannerCmd {
 	additions := scanner.GetAdditions(ignoreHistory)
 	return &ScannerCmd{
 		additions:       additions,
-		results:         helpers.NewDetectionResults(mode),
+		results:         helpers.NewDetectionResults(talismanrc.ScanMode),
 		reportDirectory: reportDirectory,
 	}
 }

cmd/talisman.go

@@ -1,10 +1,13 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
+	"runtime/pprof"
 	"strings"
+	"time"
 
 	"talisman/prompt"
 	"talisman/talismanrc"
@@ -31,6 +34,7 @@ const (
 
 var options struct {
 	debug           bool
+	loglevel        string
 	githook         string
 	pattern         string
 	scan            bool
@@ -39,22 +43,25 @@ var options struct {
 	reportDirectory string
 	scanWithHtml    bool
 	input           io.Reader
+	shouldProfile	bool
 }
 
 //var options Options
 
 func init() {
 	log.SetOutput(os.Stderr)
 	flag.BoolVarP(&options.debug, "debug", "d", false, "enable debug mode (warning: very verbose)")
+	flag.StringVarP(&options.loglevel, "loglevel", "l", "error", "enable debug mode (warning: very verbose)")
 	flag.BoolVarP(&showVersion, "version", "v", false, "show current version of talisman")
 	flag.StringVarP(&options.pattern, "pattern", "p", "", "pattern (glob-like) of files to scan (ignores githooks)")
 	flag.StringVarP(&options.githook, "githook", "g", PrePush, "either pre-push or pre-commit")
 	flag.BoolVarP(&options.scan, "scan", "s", false, "scanner scans the git commit history for potential secrets")
 	flag.BoolVar(&options.ignoreHistory, "ignoreHistory", false, "scanner scans all files on current head, will not scan through git commit history")
 	flag.StringVarP(&options.checksum, "checksum", "c", "", "checksum calculator calculates checksum and suggests .talismanrc format")
-	flag.StringVarP(&options.reportDirectory, "reportDirectory", "r", "", "directory where the scan reports will be stored")
+	flag.StringVarP(&options.reportDirectory, "reportDirectory", "r", "talisman_report", "directory where the scan reports will be stored")
 	flag.BoolVarP(&options.scanWithHtml, "scanWithHtml", "w", false, "generate html report (**Make sure you have installed talisman_html_report to use this, as mentioned in Readme**)")
 	flag.BoolVarP(&interactive, "interactive", "i", false, "interactively update talismanrc (only makes sense with -g/--githook)")
+	flag.BoolVarP(&options.shouldProfile, "profile", "", false, "interactively update talismanrc (only makes sense with -g/--githook)")
 
 }
 
@@ -85,40 +92,80 @@ func main() {
 }
 
 func run(promptContext prompt.PromptContext) (returnCode int) {
+	if options.shouldProfile {
+		log.Info("Profiling initiated")
+		defer func() { log.Info("Profiling completed") } ()
+		f, err := os.Create("talisman.pprof")
+		if err != nil {
+			log.Fatal(err)
+		}
+		_ = pprof.StartCPUProfile(f)
+		defer pprof.StopCPUProfile()
+		progEnded := false
+		defer func() { progEnded = true }()
+		go func() {
+			t := time.NewTimer(500 * time.Millisecond)
+			f1, _ := os.Create("talisman.mprof")
+			for !progEnded {
+				select {
+				case <-t.C:
+					if f1 != nil {
+						_ = pprof.WriteHeapProfile(f1)
+					} else {
+						log.Error("Could not write memory profiling info")
+					}
+
+				}
+			}
+			if f1 != nil {
+				_ = f1.Close()
+			}
+		}()
+	}
+	start := time.Now()
+	defer func() { fmt.Printf("Talisman done in %v", time.Now().Sub(start)) }()
 	if err := validateGitExecutable(afero.NewOsFs(), runtime.GOOS); err != nil {
 		log.Errorf("error validating git executable:"+" %v", err)
 		return 1
 	}
 
+	switch(options.loglevel) {
+	case "info": log.SetLevel(log.InfoLevel)
+	case "debug": log.SetLevel(log.DebugLevel)
+	case "error": log.SetLevel(log.ErrorLevel)
+	case "warn": log.SetLevel(log.WarnLevel)
+	default: log.SetLevel(log.ErrorLevel)
+	}
 	if options.debug {
 		log.SetLevel(log.DebugLevel)
-	} else {
-		log.SetLevel(log.ErrorLevel)
 	}
 
 	if options.githook == "" {
 		options.githook = PrePush
 	}
+	bytes, _ := json.Marshal(options)
+	fields := make(map[string]interface{})
+	_ = json.Unmarshal(bytes, fields)
+	log.WithFields(fields).Debug("Execution environment")
 
-	talismanRC := talismanrc.Get()
 	if options.checksum != "" {
 		log.Infof("Running %s patterns against checksum calculator", options.checksum)
 		return NewChecksumCmd(strings.Fields(options.checksum)).Run()
 	} else if options.scan {
 		log.Infof("Running scanner")
-		return NewScannerCmd(options.ignoreHistory, options.reportDirectory, talismanrc.Scan).Run(talismanRC)
+		return NewScannerCmd(options.ignoreHistory, options.reportDirectory).Run(talismanrc.For(talismanrc.ScanMode))
 	} else if options.scanWithHtml {
 		log.Infof("Running scanner with html report")
-		return NewScannerCmd(options.ignoreHistory, "talisman_html_report", talismanrc.Scan).Run(talismanRC)
+		return NewScannerCmd(options.ignoreHistory, "talisman_html_report").Run(talismanrc.For(talismanrc.ScanMode))
 	} else if options.pattern != "" {
 		log.Infof("Running scan for %s", options.pattern)
-		return NewPatternCmd(options.pattern).Run(talismanRC, promptContext)
+		return NewPatternCmd(options.pattern).Run(talismanrc.For(talismanrc.HookMode), promptContext)
 	} else if options.githook == PreCommit {
 		log.Infof("Running %s hook", options.githook)
-		return NewPreCommitHook().Run(talismanRC, promptContext)
+		return NewPreCommitHook().Run(talismanrc.For(talismanrc.HookMode), promptContext)
 	} else {
 		log.Infof("Running %s hook", options.githook)
-		return NewPrePushHook(options.input).Run(talismanRC, promptContext)
+		return NewPrePushHook(options.input).Run(talismanrc.For(talismanrc.HookMode), promptContext)
 	}
 }
 

detector/chain.go

@@ -31,9 +31,9 @@ func NewChain() *Chain {
 //DefaultChain returns a DetectorChain with pre-configured detectors
 func DefaultChain(tRC *talismanrc.TalismanRC) *Chain {
 	result := NewChain()
-	result.AddDetector(filename.DefaultFileNameDetector(tRC.GetThreshold()))
+	result.AddDetector(filename.DefaultFileNameDetector(tRC.Threshold))
 	result.AddDetector(filecontent.NewFileContentDetector(tRC))
-	result.AddDetector(pattern.NewPatternDetector(tRC.GetCustomPatterns()))
+	result.AddDetector(pattern.NewPatternDetector(tRC.CustomPatterns))
 	return result
 }
 

detector/chain_test.go

@@ -18,28 +18,28 @@ func init() {
 
 type FailingDetection struct{}
 
-func (v FailingDetection) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) {
+func (v FailingDetection) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.talismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) {
 	result.Fail("some_file", "filecontent", "FAILED BY DESIGN", []string{}, severity.Low)
 }
 
 type PassingDetection struct{}
 
-func (p PassingDetection) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) {
+func (p PassingDetection) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.talismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) {
 }
 
 func TestEmptyValidationChainPassesAllValidations(t *testing.T) {
 	v := NewChain()
-	results := helpers.NewDetectionResults(talismanrc.Hook)
-	v.Test(nil, &talismanrc.TalismanRC{}, results)
+	results := helpers.NewDetectionResults(talismanrc.HookMode)
+	v.Test(nil, &talismanrc.talismanRC{}, results)
 	assert.False(t, results.HasFailures(), "Empty validation chain is expected to always pass")
 }
 
 func TestValidationChainWithFailingValidationAlwaysFails(t *testing.T) {
 	v := NewChain()
 	v.AddDetector(PassingDetection{})
 	v.AddDetector(FailingDetection{})
-	results := helpers.NewDetectionResults(talismanrc.Hook)
-	v.Test(nil, &talismanrc.TalismanRC{}, results)
+	results := helpers.NewDetectionResults(talismanrc.HookMode)
+	v.Test(nil, &talismanrc.talismanRC{}, results)
 
 	assert.False(t, results.Successful(), "Expected validation chain with a failure to fail.")
 }

detector/filecontent/base64_aggressive_detector_test.go

@@ -13,7 +13,7 @@ import (
 
 func TestShouldFlagPotentialAWSAccessKeysInAggressiveMode(t *testing.T) {
 	const awsAccessKeyIDExample string = "AKIAIOSFODNN7EXAMPLE\n"
-	results := helpers.NewDetectionResults(talismanrc.Hook)
+	results := helpers.NewDetectionResults(talismanrc.HookMode)
 	content := []byte(awsAccessKeyIDExample)
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
@@ -24,7 +24,7 @@ func TestShouldFlagPotentialAWSAccessKeysInAggressiveMode(t *testing.T) {
 
 func TestShouldFlagPotentialAWSAccessKeysAtPropertyDefinitionInAggressiveMode(t *testing.T) {
 	const awsAccessKeyIDExample string = "accessKey=AKIAIOSFODNN7EXAMPLE"
-	results := helpers.NewDetectionResults(talismanrc.Hook)
+	results := helpers.NewDetectionResults(talismanrc.HookMode)
 	content := []byte(awsAccessKeyIDExample)
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
@@ -35,7 +35,7 @@ func TestShouldFlagPotentialAWSAccessKeysAtPropertyDefinitionInAggressiveMode(t
 
 func TestShouldNotFlagPotentialSecretsWithinSafeJavaCodeEvenInAggressiveMode(t *testing.T) {
 	const awsAccessKeyIDExample string = "public class HelloWorld {\r\n\r\n    public static void main(String[] args) {\r\n        // Prints \"Hello, World\" to the terminal window.\r\n        System.out.println(\"Hello, World\");\r\n    }\r\n\r\n}"
-	results := helpers.NewDetectionResults(talismanrc.Hook)
+	results := helpers.NewDetectionResults(talismanrc.HookMode)
 	content := []byte(awsAccessKeyIDExample)
 	filename := "filename"
 	additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}

detector/filecontent/base64_detector.go

@@ -2,6 +2,7 @@ package filecontent
 
 import (
 	log "github.com/Sirupsen/logrus"
+	"math"
 	"talisman/talismanrc"
 )
 
@@ -23,7 +24,7 @@ func NewBase64Detector(tRC *talismanrc.TalismanRC) *Base64Detector {
 	bd.AggressiveDetector = nil
 
 	bd.base64EntropyThreshold = BASE64_ENTROPY_THRESHOLD
-	if tRC.GetExperimental().Base64EntropyThreshold > 0.0 {
+	if tRC.Experimental.Base64EntropyThreshold > 0.0 {
 		bd.base64EntropyThreshold = tRC.Experimental.Base64EntropyThreshold
 		log.Debugf("Setting b64 entropy threshold to %f", bd.base64EntropyThreshold)
 	}
@@ -43,7 +44,8 @@ func (bd *Base64Detector) CheckBase64Encoding(word string) string {
 	entropyCandidates := bd.entropy.GetEntropyCandidatesWithinWord(word, MIN_BASE64_SECRET_LENGTH, bd.base64Map)
 	for _, candidate := range entropyCandidates {
 		entropy := bd.entropy.GetShannonEntropy(candidate, BASE64_CHARS)
-		log.Debugf("Detected entropy for word %s = %f", candidate, entropy)
+		sliceLimit := int(math.Min(50, float64(len(candidate))))
+		log.Debugf("Detected entropy for word %s = %f", candidate[0:sliceLimit], entropy)
 		if entropy > bd.base64EntropyThreshold && !bd.wordCheck.containsWordsOnly(candidate) {
 			return word
 		}

detector/filecontent/filecontent_detector.go

@@ -46,11 +46,11 @@ const (
 func (ct contentType) getInfo() string {
 	switch ct {
 	case base64Content:
-		return "Failing file as it contains a base64 encoded text."
+		return "Base64Detector: Failing file as it contains a base64 encoded text."
 	case hexContent:
-		return "Failing file as it contains a hex encoded text."
+		return "HexDetector: Failing file as it contains a hex encoded text."
 	case creditCardContent:
-		return "Failing file as it contains a potential credit card number."
+		return "CreditCardDetector: Failing file as it contains a potential credit card number."
 	}
 	return ""
 }
@@ -76,7 +76,7 @@ type content struct {
 	severity    severity.Severity
 }
 
-func (fc *FileContentDetector) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) {
+func (fc *FileContentDetector) Test(comparator helpers.ChecksumCompare, currentAdditions []gitrepo.Addition, talismanRC *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) {
 	contentTypes := []struct {
 		contentType
 		fn
@@ -109,7 +109,7 @@ func (fc *FileContentDetector) Test(comparator helpers.ChecksumCompare, currentA
 		go func(addition gitrepo.Addition) {
 			defer waitGroup.Done()
 			defer additionCompletionCallback()
-			if ignoreConfig.Deny(addition, "filecontent") || comparator.IsScanNotRequired(addition) {
+			if talismanRC.Deny(addition, "filecontent") || comparator.IsScanNotRequired(addition) {
 				ignoredFilePaths <- addition.Path
 				return
 			}
@@ -149,7 +149,7 @@ func (fc *FileContentDetector) Test(comparator helpers.ChecksumCompare, currentA
 				contentChanHasMore = false
 				continue
 			}
-			processContent(c, ignoreConfig.GetThreshold(), result)
+			processContent(c, talismanRC.Threshold, result)
 		}
 	}
 }