Decrease Out Of Memory chances by scanning 250 commits at a time

runner.go

@@ -4,6 +4,9 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"os/exec"
+	"strconv"
+	"strings"
 	"talisman/checksumcalculator"
 	"talisman/detector"
 	"talisman/detector/helpers"
@@ -46,14 +49,30 @@ func (r *Runner) RunWithoutErrors(promptContext prompt.PromptContext) int {
 	return r.exitStatus()
 }
 
+func getCommitCount() uint64 {
+	out, err := exec.Command("git", "rev-list", "--all", "--count").CombinedOutput()
+	if err != nil {
+		log.Fatal(err)
+	}
+	result := strings.Split(string(out), "\n")[0]
+	count, _ := strconv.ParseUint(result, 10, 64)
+	return count
+}
+
 //Scan scans git commit history for potential secrets and returns 0 or 1 as exit code
 func (r *Runner) Scan(reportDirectory string) int {
 
 	fmt.Printf("\n\n")
 	utility.CreateArt("Running Scan..")
-	additions := scanner.GetAdditions()
-	ignores := &talismanrc.TalismanRC{}
-	detector.DefaultChain(ignores).Test(additions, ignores, r.results)
+	commitsToScanAtATime := uint64(250)
+	totalNumberOfCommits := getCommitCount()
+	fmt.Println("Number of commits to scan", totalNumberOfCommits)
+	for commitNumber := uint64(0); commitNumber < totalNumberOfCommits; commitNumber += commitsToScanAtATime {
+		additions := scanner.GetAdditionsInCommitRange(commitNumber, commitsToScanAtATime)
+		fmt.Printf("Scanning <=%d Commits after %dth commit\n", commitsToScanAtATime, commitNumber)
+		ignores := &talismanrc.TalismanRC{}
+		detector.DefaultChain(ignores).Test(additions, ignores, r.results)
+	}
 	reportsPath, err := report.GenerateReport(r.results, reportDirectory)
 	if err != nil {
 		log.Printf("error while generating report: %v", err)

scanner/scanner.go

@@ -3,6 +3,7 @@ package scanner
 import (
 	"log"
 	"os/exec"
+	"strconv"
 	"strings"
 	"talisman/gitrepo"
 )
@@ -12,9 +13,9 @@ type BlobsInCommits struct {
 	commits map[string][]string
 }
 
-// GetAdditions will get all the additions for entire git history
-func GetAdditions() []gitrepo.Addition {
-	blobsInCommits := getBlobsInCommit()
+// GetAdditionsInCommitRange will get all the additions from "afterCommitNumber"th commit to "afterCommitNumber+numberOfCommits"th commit
+func GetAdditionsInCommitRange(afterCommitNumber uint64, numberOfCommits uint64) []gitrepo.Addition {
+	blobsInCommits := getBlobsInCommitRange(afterCommitNumber, numberOfCommits)
 	var additions []gitrepo.Addition
 	for blob := range blobsInCommits.commits {
 		objectDetails := strings.Split(blob, "\t")
@@ -27,8 +28,8 @@ func GetAdditions() []gitrepo.Addition {
 	return additions
 }
 
-func getBlobsInCommit() BlobsInCommits {
-	commits := getAllCommits()
+func getBlobsInCommitRange(afterCommitNumber uint64, numberOfCommits uint64) BlobsInCommits {
+	commits := getAllCommitsInRange(afterCommitNumber, numberOfCommits)
 	blobsInCommits := newBlobsInCommit()
 	result := make(chan []string, len(commits))
 	for _, commit := range commits {
@@ -62,8 +63,10 @@ func getBlobsFromChannel(blobsInCommits BlobsInCommits, result chan []string) {
 	}
 }
 
-func getAllCommits() []string {
-	out, err := exec.Command("git", "log", "--all", "--pretty=%H").CombinedOutput()
+func getAllCommitsInRange(afterCommitNumber uint64, numberOfCommits uint64) []string {
+	n := strconv.FormatUint(numberOfCommits, 10)
+	skip := strconv.FormatUint(afterCommitNumber, 10)
+	out, err := exec.Command("git", "log", "--all", "-"+n, "--skip="+skip, "--pretty=%H").CombinedOutput()
 	if err != nil {
 		log.Fatal(err)
 	}