feat(cli): configure verify err logging

main.go

@@ -37,18 +37,21 @@ import (
 )
 
 var (
-	cli                      = kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")
-	cmd                      string
-	debug                    = cli.Flag("debug", "Run in debug mode.").Bool()
-	trace                    = cli.Flag("trace", "Run in trace mode.").Bool()
-	profile                  = cli.Flag("profile", "Enables profiling and sets a pprof and fgprof server on :18066.").Bool()
-	localDev                 = cli.Flag("local-dev", "Hidden feature to disable overseer for local dev.").Hidden().Bool()
-	jsonOut                  = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
-	jsonLegacy               = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
-	gitHubActionsFormat      = cli.Flag("github-actions", "Output in GitHub Actions format.").Bool()
-	concurrency              = cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
-	noVerification           = cli.Flag("no-verification", "Don't verify the results.").Bool()
-	onlyVerified             = cli.Flag("only-verified", "Only output verified results.").Bool()
+	cli                        = kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")
+	cmd                        string
+	debug                      = cli.Flag("debug", "Run in debug mode.").Bool()
+	trace                      = cli.Flag("trace", "Run in trace mode.").Bool()
+	profile                    = cli.Flag("profile", "Enables profiling and sets a pprof and fgprof server on :18066.").Bool()
+	localDev                   = cli.Flag("local-dev", "Hidden feature to disable overseer for local dev.").Hidden().Bool()
+	jsonOut                    = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
+	jsonLegacy                 = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
+	gitHubActionsFormat        = cli.Flag("github-actions", "Output in GitHub Actions format.").Bool()
+	concurrency                = cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
+	noVerification             = cli.Flag("no-verification", "Don't verify the results.").Bool()
+	onlyVerified               = cli.Flag("only-verified", "Only output verified results.").Bool()
+	logVerificationErrorsIsSet bool
+	logVerificationErrors      = cli.Flag("log-verification-errors", "Output verification errors").IsSetByUser(&logVerificationErrorsIsSet).Bool()
+
 	allowVerificationOverlap = cli.Flag("allow-verification-overlap", "Allow verification of similar credentials across detectors").Bool()
 	filterUnverified         = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool()
 	filterEntropy            = cli.Flag("filter-entropy", "Filter unverified results with Shannon entropy. Start with 3.0.").Float64()
@@ -398,6 +401,13 @@ func run(state overseer.State) {
 		fmt.Fprintf(os.Stderr, "🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
 	}
 
+	// If |logVerificationErrors| was not set by the user,
+	// default to false if --only-verified is specified.
+	if !logVerificationErrorsIsSet {
+		value := !*onlyVerified
+		logVerificationErrors = &value
+	}
+
 	e, err := engine.Start(ctx,
 		engine.WithConcurrency(uint8(*concurrency)),
 		engine.WithDecoders(decoders.DefaultDecoders()...),
@@ -409,6 +419,7 @@ func run(state overseer.State) {
 		engine.WithFilterDetectors(endpointCustomizer),
 		engine.WithFilterUnverified(*filterUnverified),
 		engine.WithOnlyVerified(*onlyVerified),
+		engine.WithLogVerificationErrors(*logVerificationErrors),
 		engine.WithPrintAvgDetectorTime(*printAvgDetectorTime),
 		engine.WithPrinter(printer),
 		engine.WithFilterEntropy(*filterEntropy),

pkg/engine/engine.go

@@ -53,7 +53,7 @@ type runtimeMetrics struct {
 // Printer is used to format found results and output them to the user. Ex JSON, plain text, etc.
 // Please note printer implementations SHOULD BE thread safe.
 type Printer interface {
-	Print(ctx context.Context, r *detectors.ResultWithMetadata) error
+	Print(ctx context.Context, r *detectors.ResultWithMetadata, logErrors *bool) error
 }
 
 type Engine struct {
@@ -66,10 +66,11 @@ type Engine struct {
 	// only the first one will be kept.
 	filterUnverified bool
 	// entropyFilter is used to filter out unverified results using Shannon entropy.
-	filterEntropy        *float64
-	onlyVerified         bool
-	verificationOverlap  bool
-	printAvgDetectorTime bool
+	filterEntropy         *float64
+	onlyVerified          bool
+	logVerificationErrors bool
+	verificationOverlap   bool
+	printAvgDetectorTime  bool
 
 	// ahoCorasickHandler manages the Aho-Corasick trie and related keyword lookups.
 	ahoCorasickCore *ahocorasick.AhoCorasickCore
@@ -164,6 +165,15 @@ func WithOnlyVerified(onlyVerified bool) Option {
 	}
 }
 
+// WithLogVerificationErrors sets the |logVerificationErrors| flag on the engine.
+// If set to true, the engine will print results with verification errors,
+// even if the result is unverified and |onlyVerified| is true.
+func WithLogVerificationErrors(logVerificationErrors bool) Option {
+	return func(e *Engine) {
+		e.logVerificationErrors = logVerificationErrors
+	}
+}
+
 // WithPrintAvgDetectorTime sets the printAvgDetectorTime flag on the engine. If set to
 // true, the engine will print the average time taken by each detector.
 // This option allows us to measure the time taken for each detector ONLY if
@@ -810,8 +820,11 @@ func (e *Engine) processResult(ctx context.Context, data detectableChunk, res de
 
 func (e *Engine) notifyResults(ctx context.Context) {
 	for r := range e.ResultsChan() {
-		if e.onlyVerified && !r.Verified {
-			continue
+		if !r.Verified && e.onlyVerified {
+			// Skip unverified errors, unless they have a verification error and |logVerificationErrors| is true.
+			if !(e.logVerificationErrors && r.VerificationError() != nil) {
+				continue
+			}
 		}
 		atomic.AddUint32(&e.numFoundResults, 1)
 
@@ -834,7 +847,7 @@ func (e *Engine) notifyResults(ctx context.Context) {
 			atomic.AddUint64(&e.metrics.UnverifiedSecretsFound, 1)
 		}
 
-		if err := e.printer.Print(ctx, &r); err != nil {
+		if err := e.printer.Print(ctx, &r, &e.logVerificationErrors); err != nil {
 			ctx.Logger().Error(err, "error printing result")
 		}
 	}

pkg/output/github_actions.go

@@ -16,7 +16,7 @@ var dedupeCache = make(map[string]struct{})
 // GitHubActionsPrinter is a printer that prints results in GitHub Actions format.
 type GitHubActionsPrinter struct{ mu sync.Mutex }
 
-func (p *GitHubActionsPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
+func (p *GitHubActionsPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata, _ *bool) error {
 	out := gitHubActionsOutputFormat{
 		DetectorType: r.Result.DetectorType.String(),
 		DecoderType:  r.Result.DecoderType.String(),

pkg/output/json.go

@@ -16,9 +16,9 @@ import (
 // JSONPrinter is a printer that prints results in JSON format.
 type JSONPrinter struct{ mu sync.Mutex }
 
-func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
+func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata, logErrors *bool) error {
 	verificationErr := func(err error) string {
-		if err != nil {
+		if err != nil && (logErrors != nil && *logErrors) {
 			return err.Error()
 		}
 		return ""

pkg/output/legacy_json.go

@@ -24,7 +24,7 @@ import (
 // LegacyJSONPrinter is a printer that prints results in legacy JSON format for backwards compatibility.
 type LegacyJSONPrinter struct{ mu sync.Mutex }
 
-func (p *LegacyJSONPrinter) Print(ctx context.Context, r *detectors.ResultWithMetadata) error {
+func (p *LegacyJSONPrinter) Print(ctx context.Context, r *detectors.ResultWithMetadata, _ *bool) error {
 	var repo string
 	switch r.SourceType {
 	case sourcespb.SourceType_SOURCE_TYPE_GIT:

pkg/output/plain.go

@@ -28,7 +28,7 @@ var (
 // PlainPrinter is a printer that prints results in plain text format.
 type PlainPrinter struct{ mu sync.Mutex }
 
-func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error {
+func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata, logErrors *bool) error {
 	out := outputFormat{
 		DetectorType:      r.Result.DetectorType.String(),
 		DecoderType:       r.Result.DecoderType.String(),
@@ -49,7 +49,7 @@ func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
 
 	if out.Verified {
 		boldGreenPrinter.Print("✅ Found verified result 🐷🔑\n")
-	} else if out.VerificationError != nil {
+	} else if out.VerificationError != nil && (logErrors != nil && *logErrors) {
 		printer = yellowPrinter
 		boldYellowPrinter.Print("⚠️  Found result - unable to verify due to error 🐷🔑❗️\n")
 		printer.Printf("Verification Error: %s\n", out.VerificationError)