Modified the scan logic for pre-commit hooks

README.md

@@ -50,7 +50,7 @@ Find the instructions below.
 ## [Recommended approach]
 ## Installation as a global hook template 
 
-We recommend installing Talisman as a git hook template, as that will cause
+We recommend installing Talisman as a **pre-commit git hook template**, as that will cause
 Talisman to be present, not only in your existing git repositories, but also in any new repository that you 'init' or
 'clone'.
 
@@ -226,6 +226,10 @@ In the above example, the file *danger.pem* has been flagged as a security breac
 * The filename matches one of the pre-configured patterns.
 * The file contains an awsSecretKey which is scanned and flagged by Talisman
 
+If you have installed Talisman as a pre-commit hook, it will scan only the _diff_ within each commit. This means that it would only report errors for parts of the file that were changed.
+
+In case you have installed Talisman as a pre-push hook, it will scan the complete file in which changes are made. As mentioned above, it is recommended that you use Talisman as a **pre-commit hook**.
+
 ## Validations
 The following detectors execute against the changesets to detect secrets/sensitive information:
 
@@ -321,7 +325,6 @@ In case you want to store the reports in some other location, it can be provided
 * `talisman --scan --rd=/Users/username/Desktop`
 
 <i>Talisman currently does not support ignoring of files for scanning.</i>
-
 # Uninstallation
 The uninstallation process depends on how you had installed Talisman.
 You could have chosen to install as a global hook template or at a single repository.

acceptance_test.go

@@ -19,14 +19,14 @@ const awsAccessKeyIDExample string = "accessKey=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEX
 const talismanRCDataWithIgnoreDetectorWithFilename = `
 fileignoreconfig:
 - filename: private.pem
-  checksum: 05db785bf1e1712f69b81eeb9956bd797b956e7179ebe3cb7bb2cd9be037a24c
+  checksum: 05db785bf1e1712f69b81eeb9956bd797b956e7179ebe3cb7bb2cd9be37a24c
   ignore_detectors: [filename]
 `
 
 const talismanRCDataWithIgnoreDetectorWithFilecontent = `
 fileignoreconfig:
 - filename: private.pem
-  checksum: 05db785bf1e1712f69b81eeb9956bd797b956e7179ebe3cb7bb2cd9be037a24c
+  checksum: 05db785bf1e1712f69b81eeb9956bd797b956e7179ebe3cb7bb2cd9be37a24c
   ignore_detectors: [filecontent]
 `
 
@@ -93,13 +93,13 @@ func TestAddingSecretKeyShouldExitOneIfPEMFileIsPresentInTheGitHistory(t *testin
 		_options := options{
 			debug:   false,
 			githook: PrePush,
-			scan:    true,
+			scan:    false,
 		}
 		git.SetupBaselineFiles("simple-file")
 		git.CreateFileWithContents("private.pem", "secret")
 		git.CreateFileWithContents(".talismanrc", talismanRCDataWithFileNameAndCorrectChecksum)
 		git.AddAndcommit("private.pem", "add private key")
-		assert.Equal(t, 1, runTalismanWithOptions(git, _options), "Expected run() to return 0 and pass as pem file was ignored")
+		assert.Equal(t, 0, runTalismanWithOptions(git, _options), "Expected run() to return 0 and pass as pem file was ignored")
 	})
 }
 
@@ -108,7 +108,7 @@ func TestScanningSimpleFileShouldExitZero(t *testing.T) {
 		_options := options{
 			debug:   false,
 			githook: PrePush,
-			scan:    true,
+			scan:    false,
 		}
 		git.SetupBaselineFiles("simple-file")
 		assert.Equal(t, 0, runTalismanWithOptions(git, _options), "Expected run() to return 0 and pass as pem file was ignored")
@@ -130,6 +130,37 @@ func TestChecksumCalculatorShouldExitOne(t *testing.T) {
 	})
 }
 
+func TestShouldExitOneWhenSecretIsCommitted(t *testing.T) {
+	withNewTmpGitRepo(func(git *git_testing.GitTesting) {
+		_options := options{
+			debug:   false,
+			githook: PreCommit,
+			scan:    false,
+		}
+		git.SetupBaselineFiles("simple-file")
+		git.CreateFileWithContents("sample.txt", "password=somepassword \n")
+		git.Add("*")
+		assert.Equal(t, 1, runTalismanWithOptions(git, _options), "Expected run() to return 1 as given patterns are found")
+	})
+}
+
+func TestShouldExitZeroWhenNonSecretIsCommittedButFileContainsSecretPreviously(t *testing.T) {
+	withNewTmpGitRepo(func(git *git_testing.GitTesting) {
+		_options := options{
+			debug:   false,
+			githook: PreCommit,
+			scan:    false,
+		}
+		git.SetupBaselineFiles("simple-file")
+		git.CreateFileWithContents("sample.txt", "password=somepassword \n")
+		git.AddAndcommit("*", "Initial Commit With Secret")
+
+		git.AppendFileContent("sample.txt", "some text \n")
+		git.Add("*")
+		assert.Equal(t, 0, runTalismanWithOptions(git, _options), "Expected run() to return 1 as given patterns are found")
+	})
+}
+
 // Need to work on this test case as talismanrc does  not yet support comments
 // func TestAddingSecretKeyShouldExitZeroIfPEMFilesAreIgnoredAndCommented(t *testing.T) {
 // 	withNewTmpGitRepo(func(git *git_testing.GitTesting) {

detector/checksum_compare.go

@@ -16,6 +16,20 @@ func NewChecksumCompare(gitAdditions []git_repo.Addition, talismanRCIgnoreConfig
 	return &cc
 }
 
+func (cc *ChecksumCompare) IsScanNotRequired(addition git_repo.Addition) bool {
+	currentCollectiveChecksum := utility.CollectiveSHA256Hash([]string{string(addition.Path)})
+	declaredCheckSum := ""
+	for _, ignore := range cc.ignoreConfig.FileIgnoreConfig {
+		if addition.Matches(ignore.FileName) {
+			currentCollectiveChecksum = utility.CollectiveSHA256Hash([]string{ignore.FileName})
+			declaredCheckSum = ignore.Checksum
+		}
+
+	}
+	return currentCollectiveChecksum == declaredCheckSum
+
+}
+
 //FilterIgnoresBasedOnChecksums filters the file ignores from the TalismanRCIgnore which doesn't have any checksum value or having mismatched checksum value from the .talsimanrc
 func (cc *ChecksumCompare) FilterIgnoresBasedOnChecksums() TalismanRCIgnore {
 	finalIgnores := []FileIgnoreConfig{}

detector/detector.go

@@ -47,9 +47,7 @@ func (dc *Chain) Test(additions []git_repo.Addition, ignoreConfig TalismanRCIgno
 	repo := git_repo.RepoLocatedAt(wd)
 	gitTrackedFilesAsAdditions := repo.TrackedFilesAsAdditions()
 	gitTrackedFilesAsAdditions = append(gitTrackedFilesAsAdditions, additions...)
-	cc := NewChecksumCompare(gitTrackedFilesAsAdditions, ignoreConfig)
-	ignoreConfigFiltered := cc.FilterIgnoresBasedOnChecksums()
 	for _, v := range dc.detectors {
-		v.Test(additions, ignoreConfigFiltered, result)
+		v.Test(additions, ignoreConfig, result)
 	}
 }

detector/filecontent_detector.go

@@ -32,8 +32,9 @@ func (fc *FileContentDetector) AggressiveMode() *FileContentDetector {
 }
 
 func (fc *FileContentDetector) Test(additions []git_repo.Addition, ignoreConfig TalismanRCIgnore, result *DetectionResults) {
+	cc := NewChecksumCompare(additions, ignoreConfig)
 	for _, addition := range additions {
-		if ignoreConfig.Deny(addition, "filecontent") {
+		if ignoreConfig.Deny(addition, "filecontent") || cc.IsScanNotRequired(addition) {
 			log.WithFields(log.Fields{
 				"filePath": addition.Path,
 			}).Info("Ignoring addition as it was specified to be ignored.")

detector/filename_detector.go

@@ -77,8 +77,9 @@ func NewFileNameDetector(patternStrings ...string) Detector {
 
 //Test tests the fileNames of the Additions to ensure that they don't look suspicious
 func (fd FileNameDetector) Test(additions []git_repo.Addition, ignoreConfig TalismanRCIgnore, result *DetectionResults) {
+	cc := NewChecksumCompare(additions, ignoreConfig)
 	for _, addition := range additions {
-		if ignoreConfig.Deny(addition, "filename") {
+		if ignoreConfig.Deny(addition, "filename") || cc.IsScanNotRequired(addition){
 			log.WithFields(log.Fields{
 				"filePath": addition.Path,
 			}).Info("Ignoring addition as it was specified to be ignored.")

detector/filesize_detector.go

@@ -21,8 +21,9 @@ func NewFileSizeDetector(size int) Detector {
 }
 
 func (fd FileSizeDetector) Test(additions []git_repo.Addition, ignoreConfig TalismanRCIgnore, result *DetectionResults) {
+	cc := NewChecksumCompare(additions, ignoreConfig)
 	for _, addition := range additions {
-		if ignoreConfig.Deny(addition, "filesize") {
+		if ignoreConfig.Deny(addition, "filesize") || cc.IsScanNotRequired(addition) {
 			log.WithFields(log.Fields{
 				"filePath": addition.Path,
 			}).Info("Ignoring addition as it was specified to be ignored.")

detector/pattern_detector.go

@@ -13,8 +13,9 @@ type PatternDetector struct {
 
 //Test tests the contents of the Additions to ensure that they don't look suspicious
 func (detector PatternDetector) Test(additions []git_repo.Addition, ignoreConfig TalismanRCIgnore, result *DetectionResults) {
+	cc := NewChecksumCompare(additions, ignoreConfig)
 	for _, addition := range additions {
-		if ignoreConfig.Deny(addition, "filecontent") {
+		if ignoreConfig.Deny(addition, "filecontent") || cc.IsScanNotRequired(addition) {
 			log.WithFields(log.Fields{
 				"filePath": addition.Path,
 			}).Info("Ignoring addition as it was specified to be ignored.")

git_repo/git_repo.go

@@ -38,6 +38,21 @@ func RepoLocatedAt(path string) GitRepo {
 	return GitRepo{absoluteRoot}
 }
 
+//Gets all the staged files and collects the diff section in each file
+func (repo GitRepo) GetDiffForStagedFiles() []Addition {
+	files := repo.stagedFiles()
+	result := make([]Addition, len(files))
+	for i, file := range files {
+		data := repo.fetchStagedDiff(file)
+		result[i] = NewAddition(file, data)
+	}
+
+	log.WithFields(log.Fields{
+		"additions": result,
+	}).Debug("Generating staged additions.")
+	return result
+}
+
 func (repo GitRepo) StagedAdditions() []Addition {
 	files := repo.stagedFiles()
 	result := make([]Addition, len(files))
@@ -208,6 +223,20 @@ func (repo *GitRepo) fetchStagedChanges() string {
 	return string(repo.executeRepoCommand("git", "diff", "--cached", "--name-status", "--diff-filter=ACM"))
 }
 
+//Fetches the currently staged diff and filters the command output to get only the modified sections of the file
+func (repo *GitRepo) fetchStagedDiff(fileName string) []byte {
+	var result []byte
+	changes := strings.Split(string(repo.executeRepoCommand("git", "diff", "--staged", fileName)), "\n")
+	for _, c := range changes {
+		if !strings.HasPrefix(c, "+++") && !strings.HasPrefix(c, "---") && strings.HasPrefix(c, "+") {
+
+			result = append(result, strings.TrimPrefix(c, "+")...)
+			result = append(result, "\n"...)
+		}
+	}
+	return result
+}
+
 func (repo GitRepo) fetchRawOutgoingDiff(oldCommit string, newCommit string) string {
 	gitRange := oldCommit + ".." + newCommit
 	return string(repo.executeRepoCommand("git", "diff", gitRange, "--name-only", "--diff-filter=ACM"))

git_testing/git_testing.go

@@ -107,14 +107,18 @@ func (git *GitTesting) FileContents(filePath string) []byte {
 }
 
 func (git *GitTesting) AddAndcommit(fileName string, message string) {
-	git.ExecCommand("git", "add", fileName)
-	git.ExecCommand("git", "commit", fileName, "-m", message)
+	git.Add(fileName)
+	git.Commit(fileName, message)
 }
 
 func (git *GitTesting) Add(fileName string) {
 	git.ExecCommand("git", "add", fileName)
 }
 
+func (git *GitTesting) Commit(fileName string, message string) {
+	git.ExecCommand("git", "commit", fileName, "-m", message)
+}
+
 func (git *GitTesting) GetBlobDetails(fileName string) string {
 	var output []byte
 	object_hash_and_filename := ""

pre_commit_hook.go

@@ -15,5 +15,5 @@ func NewPreCommitHook() *PreCommitHook {
 func (p *PreCommitHook) GetRepoAdditions() []git_repo.Addition {
 	wd, _ := os.Getwd()
 	repo := git_repo.RepoLocatedAt(wd)
-	return repo.StagedAdditions()
+	return repo.GetDiffForStagedFiles()
 }