Ensure results are collected correctly when verification is off, and …

…dedupe twilio (#1420)
trufflesecurity · Jun 23, 2023 · eeefde1 · eeefde1
1 parent f3152b6
commit eeefde1
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 36 deletions.
diff --git a/pkg/detectors/razorpay/razorpay.go b/pkg/detectors/razorpay/razorpay.go
@@ -38,19 +38,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	keyMatches := keyPat.FindAllString(dataStr, -1)
 
 	for _, key := range keyMatches {
+		secMatches := secretPat.FindAllString(dataStr, -1)
 
-		if verify {
-			secMatches := secretPat.FindAllString(dataStr, -1)
+		for _, secret := range secMatches {
 
-			for _, secret := range secMatches {
-
-				s1 := detectors.Result{
-					DetectorType: detectorspb.DetectorType_RazorPay,
-					Raw:          []byte(key),
-					RawV2:        []byte(key + secret),
-					Redacted:     key,
-				}
+			s1 := detectors.Result{
+				DetectorType: detectorspb.DetectorType_RazorPay,
+				Raw:          []byte(key),
+				RawV2:        []byte(key + secret),
+				Redacted:     key,
+			}
 
+			if verify {
 				req, err := http.NewRequest("GET", "https://api.razorpay.com/v1/items?count=1", nil)
 				if err != nil {
 					continue
@@ -66,25 +65,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						if json.Valid(bodyBytes) {
 							s1.Verified = true
-						} else {
-							s1.Verified = false
-						}
-					} else {
-						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-						if detectors.IsKnownFalsePositive(key, detectors.DefaultFalsePositives, true) {
-							continue
 						}
 					}
 				}
+			}
 
-				results = append(results, s1)
+			if !s1.Verified && detectors.IsKnownFalsePositive(key, detectors.DefaultFalsePositives, true) {
+				continue
 			}
+
+			results = append(results, s1)
 		}
 
 	}
 
-	results = detectors.CleanResults(results)
-	return
+	return detectors.CleanResults(results), nil
 }
 
 func (s Scanner) Type() detectorspb.DetectorType {

diff --git a/pkg/detectors/twilio/twilio.go b/pkg/detectors/twilio/twilio.go
@@ -21,6 +21,7 @@ var (
 	identifierPat = regexp.MustCompile(`(?i)sid.{0,20}AC[0-9a-f]{32}`) // Should we have this? Seems restrictive.
 	sidPat        = regexp.MustCompile(`\bAC[0-9a-f]{32}\b`)
 	keyPat        = regexp.MustCompile(`\b[0-9a-f]{32}\b`)
+	client        = common.SaneHttpClient()
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -43,15 +44,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	sidMatches := sidPat.FindAllString(dataStr, -1)
 
 	for _, sid := range sidMatches {
-		s := detectors.Result{
-			DetectorType: detectorspb.DetectorType_Twilio,
-			Raw:          []byte(sid),
-			Redacted:     sid,
-		}
+		for _, key := range keyMatches {
+			s1 := detectors.Result{
+				DetectorType: detectorspb.DetectorType_Twilio,
+				Raw:          []byte(sid),
+				RawV2:        []byte(sid + key),
+				Redacted:     sid,
+			}
 
-		if verify {
-			client := common.SaneHttpClient()
-			for _, key := range keyMatches {
+			if verify {
 
 				form := url.Values{}
 				form.Add("FriendlyName", "MyServiceName")
@@ -71,22 +72,22 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					res.Body.Close() // The request body is unused.
 
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
-						s.Verified = true
+						s1.Verified = true
 					}
 				}
 			}
-		}
 
-		if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
-			continue
-		}
+			if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
+				continue
+			}
 
-		if len(keyMatches) > 0 {
-			results = append(results, s)
+			if len(keyMatches) > 0 {
+				results = append(results, s1)
+			}
 		}
 	}
 
-	return
+	return detectors.CleanResults(results), nil
 }
 
 func (s Scanner) Type() detectorspb.DetectorType {

diff --git a/pkg/detectors/twilio/twilio_test.go b/pkg/detectors/twilio/twilio_test.go
@@ -98,6 +98,7 @@ func TestTwilio_FromChunk(t *testing.T) {
 					t.Fatal("no raw secret present")
 				}
 				got[i].Raw = nil
+				got[i].RawV2 = nil
 			}
 			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("Twilio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)