Expose detector-specific false positive logic

pkg/custom_detectors/custom_detectors.go

@@ -29,6 +29,7 @@ type CustomRegexWebhook struct {
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*CustomRegexWebhook)(nil)
+var _ detectors.CustomFalsePositiveChecker = (*CustomRegexWebhook)(nil)
 
 // NewWebhookCustomRegex initializes and validates a CustomRegexWebhook. An
 // unexported type is intentionally returned here to ensure the values have
@@ -109,6 +110,10 @@ func (c *CustomRegexWebhook) FromData(ctx context.Context, verify bool, data []b
 	return results, nil
 }
 
+func (c *CustomRegexWebhook) IsFalsePositive(_ detectors.Result) bool {
+	return false
+}
+
 func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string][]string, verify bool, results chan<- detectors.Result) error {
 	if common.IsDone(ctx) {
 		// TODO: Log we're possibly leaving out results.

pkg/detectors/azurebatch/azurebatch.go

@@ -23,6 +23,7 @@ type Scanner struct {
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)
 
 var (
 	defaultClient = common.SaneHttpClient()
@@ -105,6 +106,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return results, nil
 }
 
+func (s Scanner) IsFalsePositive(_ detectors.Result) bool {
+	return false
+}
+
 func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_AzureBatch
 }

pkg/detectors/azurecontainerregistry/azurecontainerregistry.go

@@ -19,6 +19,7 @@ type Scanner struct {
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)
 
 var (
 	defaultClient = common.SaneHttpClient()
@@ -94,6 +95,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return results, nil
 }
 
+func (s Scanner) IsFalsePositive(_ detectors.Result) bool {
+	return false
+}
+
 func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_AzureContainerRegistry
 }

pkg/detectors/falsepositives.go

@@ -8,15 +8,17 @@ import (
 	"unicode/utf8"
 
 	ahocorasick "github.com/BobuSumisu/aho-corasick"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 )
 
 var DefaultFalsePositives = []FalsePositive{"example", "xxxxxx", "aaaaaa", "abcde", "00000", "sample", "www"}
 
 type FalsePositive string
 
+type CustomFalsePositiveChecker interface {
+	IsFalsePositive(result Result) bool
+}
+
 //go:embed "badlist.txt"
 var badList []byte
 
@@ -43,6 +45,17 @@ func init() {
 	filter = builder.Build()
 }
 
+func GetFalsePositiveCheck(detector Detector) func(Result) bool {
+	checker, ok := detector.(CustomFalsePositiveChecker)
+	if ok {
+		return checker.IsFalsePositive
+	}
+
+	return func(res Result) bool {
+		return IsKnownFalsePositive(string(res.Raw), DefaultFalsePositives, true)
+	}
+}
+
 // IsKnownFalsePositive will not return a valid secret finding if any of the disqualifying conditions are met
 // Currently that includes: No number, english word in key, or matches common example pattens.
 // Only the secret key material should be passed into this function
@@ -132,34 +145,17 @@ func FilterResultsWithEntropy(ctx context.Context, results []Result, entropy flo
 }
 
 // FilterKnownFalsePositives filters out known false positives from the results.
-func FilterKnownFalsePositives(ctx context.Context, results []Result, falsePositives []FalsePositive, wordCheck bool, shouldLog bool) []Result {
+func FilterKnownFalsePositives(ctx context.Context, detector Detector, results []Result, shouldLog bool) []Result {
 	var filteredResults []Result
+
+	isFalsePositive := GetFalsePositiveCheck(detector)
+
 	for _, result := range results {
-		if !result.Verified {
-			switch result.DetectorType {
-			case detectorspb.DetectorType_CustomRegex:
-				filteredResults = append(filteredResults, result)
-			case detectorspb.DetectorType_GCP,
-				detectorspb.DetectorType_URI,
-				detectorspb.DetectorType_AzureBatch,
-				detectorspb.DetectorType_AzureContainerRegistry,
-				detectorspb.DetectorType_Shopify,
-				detectorspb.DetectorType_Postgres,
-				detectorspb.DetectorType_MongoDB,
-				detectorspb.DetectorType_JDBC:
+		if !result.Verified && result.Raw != nil {
+			if !isFalsePositive(result) {
 				filteredResults = append(filteredResults, result)
-			default:
-				if result.Raw != nil {
-					if !IsKnownFalsePositive(string(result.Raw), falsePositives, wordCheck) {
-						filteredResults = append(filteredResults, result)
-					} else {
-						if shouldLog {
-							ctx.Logger().Info("Filtered out known false positive", "result", result)
-						}
-					}
-				} else {
-					filteredResults = append(filteredResults, result)
-				}
+			} else if shouldLog {
+				ctx.Logger().Info("Filtered out known false positive", "result", result)
 			}
 		} else {
 			filteredResults = append(filteredResults, result)

pkg/detectors/falsepositives_test.go

@@ -4,10 +4,63 @@
 package detectors
 
 import (
+	"context"
 	_ "embed"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
+	logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
+type fakeDetector struct{}
+type customFalsePositiveChecker struct{ fakeDetector }
+
+func (d fakeDetector) FromData(ctx context.Context, verify bool, data []byte) ([]Result, error) {
+	return nil, nil
+}
+
+func (d fakeDetector) Keywords() []string {
+	return nil
+}
+
+func (d fakeDetector) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType(0)
+}
+
+func (d customFalsePositiveChecker) IsFalsePositive(result Result) bool {
+	return IsKnownFalsePositive(string(result.Raw), []FalsePositive{"a specific magic string"}, false)
+}
+
+func TestFilterKnownFalsePositives_DefaultLogic(t *testing.T) {
+	results := []Result{
+		{Raw: []byte("00000")},           // "default" false positive list
+		{Raw: []byte("number")},          // from wordlist
+		{Raw: []byte("hga8adshla3434g")}, // real secret
+	}
+	expected := []Result{
+		{Raw: []byte("hga8adshla3434g")},
+	}
+	filtered := FilterKnownFalsePositives(logContext.Background(), fakeDetector{}, results, false)
+	assert.ElementsMatch(t, expected, filtered)
+}
+
+func TestFilterKnownFalsePositives_CustomLogic(t *testing.T) {
+	results := []Result{
+		{Raw: []byte("a specific magic string")}, // specific target
+		{Raw: []byte("00000")},                   // "default" false positive list
+		{Raw: []byte("number")},                  // from wordlist
+		{Raw: []byte("hga8adshla3434g")},         // real secret
+	}
+	expected := []Result{
+		{Raw: []byte("00000")},
+		{Raw: []byte("number")},
+		{Raw: []byte("hga8adshla3434g")},
+	}
+	filtered := FilterKnownFalsePositives(logContext.Background(), customFalsePositiveChecker{}, results, false)
+	assert.ElementsMatch(t, expected, filtered)
+}
+
 func TestIsFalsePositive(t *testing.T) {
 	type args struct {
 		match          string

pkg/detectors/ftp/ftp.go

@@ -30,6 +30,7 @@ type Scanner struct {
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)
 
 var (
 	keyPat = regexp.MustCompile(`\bftp://[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`)
@@ -96,16 +97,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 		}
 
-		if detectors.IsKnownFalsePositive(string(s1.Raw), []detectors.FalsePositive{"@ftp.freebsd.org"}, false) {
-			continue
-		}
-
 		results = append(results, s1)
 	}
 
 	return results, nil
 }
 
+func (s Scanner) IsFalsePositive(result detectors.Result) bool {
+	return detectors.IsKnownFalsePositive(string(result.Raw), []detectors.FalsePositive{"@ftp.freebsd.org"}, false)
+}
+
 func isErrDeterminate(e error) bool {
 	ftpErr := &textproto.Error{}
 	return errors.As(e, &ftpErr) && ftpErr.Code == ftpNotLoggedIn

pkg/detectors/gcp/gcp.go

@@ -18,6 +18,7 @@ type Scanner struct{}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)
 
 var (
 	keyPat = regexp.MustCompile(`\{[^{]+auth_provider_x509_cert_url[^}]+\}`)
@@ -120,6 +121,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return
 }
 
+func (s Scanner) IsFalsePositive(_ detectors.Result) bool {
+	return false
+}
+
 func (s Scanner) Type() detectorspb.DetectorType {
 	return detectorspb.DetectorType_GCP
 }

pkg/detectors/gcpapplicationdefaultcredentials/gcpapplicationdefaultcredentials.go

@@ -82,11 +82,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1.SetVerificationError(verificationErr, match)
 		}
 
-		// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-		if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
-			continue
-		}
-
 		results = append(results, s1)
 	}
 

pkg/detectors/generic/generic.go

@@ -75,10 +75,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		// Least expensive-> most expensive filters.
 		// Substrings, then patterns.
 
-		if detectors.IsKnownFalsePositive(token, detectors.DefaultFalsePositives, true) {
-			continue
-		}
-
 		// toss any that match regexes
 		if hasReMatch(s.excludeMatchers, token) {
 			continue

pkg/detectors/github/v1/github_old.go

@@ -69,6 +69,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 		token := match[1]
 
+		// Note that this false positive check happens **before** verification! I don't know why it's written this way
+		// but that's why this logic wasn't moved into a CustomFalsePositiveChecker implementation.
 		specificFPs := []detectors.FalsePositive{"github commit"}
 		if detectors.IsKnownFalsePositive(token, specificFPs, false) {
 			continue

pkg/detectors/github/v2/github.go

@@ -83,9 +83,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 		}
 
-		if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
-			continue
-		}
 		results = append(results, s1)
 	}
 

pkg/detectors/jdbc/jdbc.go

@@ -45,6 +45,7 @@ func WithIgnorePattern(ignoreStrings []string) func(*Scanner) {
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)
 
 var (
 	keyPat = regexp.MustCompile(`(?i)jdbc:[\w]{3,10}:[^\s"']{0,512}`)
@@ -98,14 +99,16 @@ matchLoop:
 			// TODO: specialized redaction
 		}
 
-
-
 		results = append(results, s)
 	}
 
 	return
 }
 
+func (s Scanner) IsFalsePositive(_ detectors.Result) bool {
+	return false
+}
+
 func tryRedactAnonymousJDBC(conn string) string {
 	if s, ok := tryRedactURLParams(conn); ok {
 		return s

pkg/detectors/jupiterone/jupiterone.go

@@ -83,11 +83,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 		}
 
-		// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-		if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-			continue
-		}
-
 		results = append(results, s1)
 	}
 

pkg/detectors/mongodb/mongodb.go

@@ -24,6 +24,7 @@ type Scanner struct {
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)
 
 var (
 	defaultTimeout = 2 * time.Second
@@ -72,6 +73,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return results, nil
 }
 
+func (s Scanner) IsFalsePositive(_ detectors.Result) bool {
+	return false
+}
+
 func isErrDeterminate(err error) bool {
 	switch e := err.(type) {
 	case topology.ConnectionError:

pkg/detectors/onfleet/onfleet.go

@@ -60,11 +60,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1.SetVerificationError(verificationErr, match)
 		}
 
-		// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-		if !s1.Verified && detectors.IsKnownFalsePositive(match, detectors.DefaultFalsePositives, true) {
-			continue
-		}
-
 		results = append(results, s1)
 	}
 

pkg/detectors/pagarme/pagarme.go

@@ -60,11 +60,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1.SetVerificationError(verificationErr, match)
 		}
 
-		// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-		if !s1.Verified && detectors.IsKnownFalsePositive(match, detectors.DefaultFalsePositives, true) {
-			continue
-		}
-
 		results = append(results, s1)
 	}
 

pkg/detectors/postgres/postgres.go

@@ -55,6 +55,9 @@ type Scanner struct {
 	detectLoopback bool // Automated tests run against localhost, but we want to ignore those results in the wild
 }
 
+var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)
+
 func (s Scanner) Keywords() []string {
 	return []string{"postgres"}
 }
@@ -144,6 +147,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]dete
 	return results, nil
 }
 
+func (s Scanner) IsFalsePositive(_ detectors.Result) bool {
+	return false
+}
+
 func findUriMatches(data []byte) []map[string]string {
 	var matches []map[string]string
 	for _, uri := range uriPattern.FindAll(data, -1) {