feat: add string entropy greater than filter (#1334)

internal/commands/process/settings/settings.go

@@ -236,6 +236,7 @@ type PatternFilter struct {
 	GreaterThan        *int     `mapstructure:"greater_than" json:"greater_than" yaml:"greater_than"`
 	GreaterThanOrEqual *int     `mapstructure:"greater_than_or_equal" json:"greater_than_or_equal" yaml:"greater_than_or_equal"`
 	StringRegex        *Regexp  `mapstructure:"string_regex" json:"string_regex" yaml:"string_regex"`
+	EntropyGreaterThan *float64 `mapstructure:"entropy_greater_than" json:"entropy_greater_than" yaml:"entropy_greater_than"`
 	FilenameRegex      *Regexp  `mapstructure:"filename_regex" json:"filename_regex" yaml:"filename_regex"`
 }
 

internal/scanner/detectors/customrule/filter.go

@@ -160,6 +160,13 @@ func translateFilter(
 		}, nil
 	}
 
+	if sourceFilter.EntropyGreaterThan != nil {
+		return &filters.EntropyGreaterThan{
+			Variable: variable,
+			Value:    *sourceFilter.EntropyGreaterThan,
+		}, nil
+	}
+
 	if sourceFilter.LessThan != nil {
 		return &filters.IntegerLessThan{
 			Variable: variable,
@@ -228,6 +235,7 @@ func scoreFilter(filter settings.PatternFilter) int {
 	}
 
 	if filter.StringRegex != nil ||
+		filter.EntropyGreaterThan != nil ||
 		filter.Detection != "" && filter.Scope == settings.CURSOR_STRICT_SCOPE {
 		return 2
 	}

internal/scanner/detectors/customrule/filters/filters.go

@@ -14,6 +14,7 @@ import (
 	detectortypes "github.com/bearer/bearer/internal/scanner/detectors/types"
 	"github.com/bearer/bearer/internal/scanner/ruleset"
 	"github.com/bearer/bearer/internal/scanner/variableshape"
+	"github.com/bearer/bearer/internal/util/entropy"
 )
 
 type Result struct {
@@ -407,6 +408,45 @@ func (filter *StringRegex) Evaluate(
 	return boolResult(patternVariables, result), nil
 }
 
+type EntropyGreaterThan struct {
+	Variable *variableshape.Variable
+	Value    float64
+}
+
+func (filter *EntropyGreaterThan) Evaluate(
+	detectorContext detectortypes.Context,
+	patternVariables variableshape.Values,
+) (*Result, error) {
+	node := patternVariables.Node(filter.Variable)
+	value, isString, err := lookupString(detectorContext, node)
+	if err != nil {
+		return nil, err
+	}
+
+	if !isString {
+		if log.Trace().Enabled() {
+			log.Trace().Msgf("filters.EntropyGreaterThan: nil for min %f at %s", filter.Value, node.Debug())
+		}
+
+		return nil, nil
+	}
+
+	entropy := entropy.Shannon(value)
+	result := entropy > filter.Value
+	if log.Trace().Enabled() {
+		log.Trace().Msgf(
+			"filters.EntropyGreaterThan: %t for entropy %f with min %f at %s, content=%s",
+			result,
+			entropy,
+			filter.Value,
+			node.Debug(),
+			value,
+		)
+	}
+
+	return boolResult(patternVariables, result), nil
+}
+
 type IntegerLessThan struct {
 	Variable *variableshape.Variable
 	Value    int

internal/scanner/detectors/customrule/filters/filters_test.go

@@ -479,6 +479,52 @@ var _ = Describe("StringRegex", func() {
 	})
 })
 
+var _ = Describe("EntropyGreaterThan", func() {
+	var filter *filters.EntropyGreaterThan
+	var variable *variableshape.Variable
+	var detectorContext detectortypes.Context
+	var patternVariables variableshape.Values
+
+	BeforeEach(func(ctx SpecContext) {
+		variable, patternVariables = setupContentTest(ctx, "hello", "other")
+		// entropy("Au+u1hvsvJeEXxky") == 3.75
+		detectorContext = setupStringTest(patternVariables.Node(variable), pointers.String("Au+u1hvsvJeEXxky"))
+	})
+
+	When("the variable node's content is a string with entropy greater than the filter value", func() {
+		BeforeEach(func(ctx SpecContext) {
+			filter = &filters.EntropyGreaterThan{Variable: variable, Value: 3.7}
+		})
+
+		It("returns a result with a match using the pattern variables", func(ctx SpecContext) {
+			Expect(filter.Evaluate(detectorContext, patternVariables)).To(Equal(
+				filters.NewResult(filters.NewMatch(patternVariables, nil)),
+			))
+		})
+	})
+
+	When("the variable node's content is a string with entropy less than or equal to the filter value", func() {
+		BeforeEach(func(ctx SpecContext) {
+			filter = &filters.EntropyGreaterThan{Variable: variable, Value: 3.8}
+		})
+
+		It("returns a result with NO matches", func(ctx SpecContext) {
+			Expect(filter.Evaluate(detectorContext, patternVariables)).To(Equal(filters.NewResult()))
+		})
+	})
+
+	When("the variable node is not a string value", func() {
+		BeforeEach(func(ctx SpecContext) {
+			filter = &filters.EntropyGreaterThan{Variable: variable, Value: 3.7}
+			detectorContext = setupStringTest(patternVariables.Node(variable), nil)
+		})
+
+		It("returns an unknown result", func(ctx SpecContext) {
+			Expect(filter.Evaluate(detectorContext, patternVariables)).To(BeNil())
+		})
+	})
+})
+
 var _ = Describe("IntegerLessThan", func() {
 	var filter *filters.IntegerLessThan
 	var variable *variableshape.Variable

internal/util/entropy/.snapshots/TestShannon

@@ -0,0 +1,6 @@
+([]string) (len=4) {
+  (string) (len=4) "1.58",
+  (string) (len=4) "2.75",
+  (string) (len=4) "2.85",
+  (string) (len=4) "3.75"
+}

internal/util/entropy/entropy.go

@@ -0,0 +1,24 @@
+package entropy
+
+import "math"
+
+func Shannon(value string) float64 {
+	if value == "" {
+		return 0
+	}
+
+	counts := make(map[rune]int)
+	for _, character := range value {
+		counts[character]++
+	}
+
+	length := float64(len(value))
+	var negativeEntropy float64
+
+	for _, count := range counts {
+		characterProbability := float64(count) / length
+		negativeEntropy += characterProbability * math.Log2(characterProbability)
+	}
+
+	return -negativeEntropy
+}

internal/util/entropy/entropy_test.go

@@ -0,0 +1,26 @@
+package entropy_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/bradleyjkemp/cupaloy"
+
+	"github.com/bearer/bearer/internal/util/entropy"
+)
+
+func TestShannon(t *testing.T) {
+	examples := []string{
+		"one",
+		"password",
+		"secret_key",
+		"Au+u1hvsvJeEXxky",
+	}
+
+	results := make([]string, len(examples))
+	for i, example := range examples {
+		results[i] = fmt.Sprintf("%.2f", entropy.Shannon(example))
+	}
+
+	cupaloy.SnapshotT(t, results)
+}