feat: ignore files that not scanned usually (#225)

Copied from _gitleaks_ the "Allowlist" paths: paths that we didn't
expect to find a secret there.

```
gitleaks.toml
(.*?)(jpg|gif|doc|docx|zip|xls|pdf|bin|svg|socket|vsidx|v2|suo|wsuo|.dll|pdb|exe)$
(go.mod|go.sum)$
gradle.lockfile
node_modules
package-lock.json
yarn.lock
pnpm-lock.yaml
Database.refactorlog
vendor
```

.github/workflows/pr-validation.yml

@@ -32,12 +32,12 @@ jobs:
       - name: Go Linter
         run: docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:v1.52.0 golangci-lint run -v -E gofmt --timeout=5m --out-format github-actions
 
-      - name: Run 2ms Scan
-        run: go run . git . --config .2ms.yml
-
       - name: Go Test
         run: go test -v ./...
 
+      - name: Run 2ms Scan
+        run: go run . git . --config .2ms.yml
+
   build:
     runs-on: ubuntu-latest
     steps:

engine/config.go

@@ -0,0 +1,25 @@
+package engine
+
+import (
+	"regexp"
+
+	"github.com/zricethezav/gitleaks/v8/config"
+)
+
+// Taken from gitleaks config https://github.com/gitleaks/gitleaks/blob/6c52f878cc48a513849900a9aa6f9d68e1c2dbdd/config/gitleaks.toml#L15-L26
+var cfg = config.Config{
+	Allowlist: config.Allowlist{
+		Paths: []*regexp.Regexp{
+			regexp.MustCompile(`gitleaks.toml`),
+			regexp.MustCompile(`(.*?)(jpg|gif|doc|docx|zip|xls|pdf|bin|svg|socket|vsidx|v2|suo|wsuo|.dll|pdb|exe)$`),
+			regexp.MustCompile(`(go.mod|go.sum)$`),
+			regexp.MustCompile(`gradle.lockfile`),
+			regexp.MustCompile(`node_modules`),
+			regexp.MustCompile(`package-lock.json`),
+			regexp.MustCompile(`yarn.lock`),
+			regexp.MustCompile(`pnpm-lock.yaml`),
+			regexp.MustCompile(`Database.refactorlog`),
+			regexp.MustCompile(`vendor`),
+		},
+	},
+}

engine/engine.go

@@ -48,10 +48,9 @@ func Init(engineConfig EngineConfig) (*Engine, error) {
 		rule.Rule.Keywords = []string{}
 		rulesToBeApplied[rule.Rule.RuleID] = rule.Rule
 	}
+	cfg.Rules = rulesToBeApplied
 
-	detector := detect.NewDetector(config.Config{
-		Rules: rulesToBeApplied,
-	})
+	detector := detect.NewDetector(cfg)
 	detector.MaxTargetMegaBytes = engineConfig.MaxTargetMegabytes
 
 	return &Engine{
@@ -65,7 +64,8 @@ func (s *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.S
 	defer wg.Done()
 
 	fragment := detect.Fragment{
-		Raw: *item.GetContent(),
+		Raw:      *item.GetContent(),
+		FilePath: item.GetSource(),
 	}
 	for _, value := range s.detector.Detect(fragment) {
 		itemId := getFindingId(item, value)

engine/engine_test.go

@@ -61,6 +61,32 @@ func Test_Init(t *testing.T) {
 	}
 }
 
+func TestDetector(t *testing.T) {
+	t.Run("ignore go.sum file", func(t *testing.T) {
+		token := "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"
+		i := item{
+			content: &token,
+			source:  "path/to/go.sum",
+		}
+
+		detector, err := Init(EngineConfig{})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		secretsChan := make(chan *secrets.Secret, 1)
+		wg := &sync.WaitGroup{}
+		wg.Add(1)
+		detector.Detect(i, secretsChan, wg, nil)
+		close(secretsChan)
+
+		s := <-secretsChan
+		if s != nil {
+			t.Error("expected nil, got secret")
+		}
+	})
+}
+
 func TestSecrets(t *testing.T) {
 	secretsCases := []struct {
 		Content    string
@@ -143,6 +169,8 @@ func TestSecrets(t *testing.T) {
 
 type item struct {
 	content *string
+	id      string
+	source  string
 }
 
 var _ plugins.ISourceItem = (*item)(nil)
@@ -151,8 +179,14 @@ func (i item) GetContent() *string {
 	return i.content
 }
 func (i item) GetID() string {
+	if i.id != "" {
+		return i.id
+	}
 	return "test"
 }
 func (i item) GetSource() string {
+	if i.source != "" {
+		return i.source
+	}
 	return "test"
 }

tests/e2e.go

@@ -43,11 +43,11 @@ func createCLI(outputDir string) (cli, error) {
 		nil
 }
 
-func generateProject(outputDir string) error {
+func generateFileWithSecret(outputDir string, filename string) error {
 	token := "g" + "hp" + "_ixOl" + "iEFNK4O" + "brYB506" + "8oXFd" + "9JUF" + "iRy0RU" + "KNl"
 	content := "bla bla bla\nGitHubToken: " + token + "\nbla bla bla"
 
-	if err := os.WriteFile(path.Join(outputDir, "secret.txt"), []byte(content), 0644); err != nil {
+	if err := os.WriteFile(path.Join(outputDir, filename), []byte(content), 0644); err != nil {
 		return err
 	}
 

tests/e2e_test.go

@@ -15,7 +15,7 @@ func TestIntegration(t *testing.T) {
 	t.Run("filesystem: one secret found", func(t *testing.T) {
 		projectDir := t.TempDir()
 
-		if err := generateProject(projectDir); err != nil {
+		if err := generateFileWithSecret(projectDir, "secret.txt"); err != nil {
 			t.Fatalf("failed to generate project: %s", err)
 		}
 
@@ -55,4 +55,25 @@ func TestIntegration(t *testing.T) {
 			}
 		}
 	})
+
+	t.Run("filesystem: ignore go.sum file", func(t *testing.T) {
+		projectDir := t.TempDir()
+
+		if err := generateFileWithSecret(projectDir, "go.sum"); err != nil {
+			t.Fatalf("failed to generate project: %s", err)
+		}
+
+		if err := executable.run("filesystem", "--path", projectDir); err != nil {
+			t.Errorf("expected no error, got %s", err)
+		}
+
+		report, err := executable.getReport()
+		if err != nil {
+			t.Fatalf("failed to get report: %s", err)
+		}
+
+		if len(report.Results) != 0 {
+			t.Errorf("expected no results, got %d", len(report.Results))
+		}
+	})
 }