test: exhaustive coverage for the #91-#96 behavior

.keywatch-baseline.json

@@ -917,6 +917,139 @@
       "finding_type": "AWS Access Key",
       "matched_content_hash": "3f733150de7916d4778298d7f90493889c38b76876b80c058e439851ce60cb2b",
       "plugin_name": "AWSKeyDetector"
+    },
+    {
+      "file_path": "./src/detector.rs",
+      "line_number": 147,
+      "finding_type": "Aadhaar Card Number",
+      "matched_content_hash": "0476e6cf99db1c000b7e0a433ff83591ffe0472f3156c9ebac7b11d0862f3429",
+      "plugin_name": "AadhaarCardDetector"
+    },
+    {
+      "file_path": "./src/detector.rs",
+      "line_number": 148,
+      "finding_type": "Aadhaar Card Number",
+      "matched_content_hash": "a1de1619345526170197aa72dd7bcb7e805e4c694270d74c752c317afd4a625e",
+      "plugin_name": "AadhaarCardDetector"
+    },
+    {
+      "file_path": "./src/detector.rs",
+      "line_number": 520,
+      "finding_type": "Credit Card Number",
+      "matched_content_hash": "4541206d542811878a9374508fe296fa321a8b56c2902736a02f388836f6e108",
+      "plugin_name": "CreditCardDetector"
+    },
+    {
+      "file_path": "./src/detector.rs",
+      "line_number": 521,
+      "finding_type": "Credit Card Number",
+      "matched_content_hash": "13ae894eedbfba2dbd06400ba5b215ffd661885646ab86e050fb1a0d192c1c5b",
+      "plugin_name": "CreditCardDetector"
+    },
+    {
+      "file_path": "./src/detector.rs",
+      "line_number": 522,
+      "finding_type": "Credit Card Number",
+      "matched_content_hash": "0d30829f4cbd240de78f8dc72d0a5ed0a77887656aa572ee8fb1392cf9ae34a1",
+      "plugin_name": "CreditCardDetector"
+    },
+    {
+      "file_path": "./src/detector.rs",
+      "line_number": 537,
+      "finding_type": "Random String",
+      "matched_content_hash": "e51298df0e431de2bfdf6180e3a7b9f3f092c3e9a912facd350e8c7179936e75",
+      "plugin_name": "RandomString"
+    },
+    {
+      "file_path": "./tests/detector_tests.rs",
+      "line_number": 389,
+      "finding_type": "Generic Key/Secret",
+      "matched_content_hash": "6daabe7b8b2ca9ca4362fee06c2d25e32e59004c5834d23a755f774b02a054c1",
+      "plugin_name": "GenericKeyValueDetector"
+    },
+    {
+      "file_path": "./tests/detector_tests.rs",
+      "line_number": 394,
+      "finding_type": "AWS Access Key",
+      "matched_content_hash": "0cbae582394e61dc81d9946ed87ec44f4c83cf1167181f62cd1454eb5e2e5469",
+      "plugin_name": "AWSKeyDetector"
+    },
+    {
+      "file_path": "./tests/detector_tests.rs",
+      "line_number": 421,
+      "finding_type": "Password",
+      "matched_content_hash": "f6bd37622d846ac435a7b7dcbde2347d59494dfd3c3a787604e8b91491a39c95",
+      "plugin_name": "PasswordDetector"
+    },
+    {
+      "file_path": "./tests/detector_tests.rs",
+      "line_number": 421,
+      "finding_type": "Generic Key/Secret",
+      "matched_content_hash": "f6bd37622d846ac435a7b7dcbde2347d59494dfd3c3a787604e8b91491a39c95",
+      "plugin_name": "GenericKeyValueDetector"
+    },
+    {
+      "file_path": "./tests/detector_tests.rs",
+      "line_number": 422,
+      "finding_type": "Generic Key/Secret",
+      "matched_content_hash": "c98bc65cd589366ec37fedb07e874646469d675fc56078e5b8a3e18d2a70e51e",
+      "plugin_name": "GenericKeyValueDetector"
+    },
+    {
+      "file_path": "./tests/detector_tests.rs",
+      "line_number": 423,
+      "finding_type": "Password",
+      "matched_content_hash": "af29a321067a1bbd4a7d3f56ba583751c3f0496bd204f642e1e5764766fe9a8c",
+      "plugin_name": "PasswordDetector"
+    },
+    {
+      "file_path": "./tests/detector_tests.rs",
+      "line_number": 449,
+      "finding_type": "Email Address",
+      "matched_content_hash": "ebe162e5d3cc06b42201b0bfe39fde3379a3fe97c9836631f1750f21db142808",
+      "plugin_name": "EmailDetector"
+    },
+    {
+      "file_path": "./tests/detector_tests.rs",
+      "line_number": 470,
+      "finding_type": "Base64 Encoded String",
+      "matched_content_hash": "097eda6585aa25d794c938beab55a00e7f2e7471a39375a3356535aa2e76efa6",
+      "plugin_name": "Base64Detector"
+    },
+    {
+      "file_path": "./tests/detector_tests.rs",
+      "line_number": 476,
+      "finding_type": "Generic Key/Secret",
+      "matched_content_hash": "c9bff1b4953e132d00a1d95477c81e7e73f15847b0e7d538c5a3a9c022858899",
+      "plugin_name": "GenericKeyValueDetector"
+    },
+    {
+      "file_path": "./tests/detector_tests.rs",
+      "line_number": 476,
+      "finding_type": "Base64 Encoded String",
+      "matched_content_hash": "a011d6c6ecfe7ed080c7c4bfcc08efc5a3744a7be7a117c466d2536c2ee30a9b",
+      "plugin_name": "Base64Detector"
+    },
+    {
+      "file_path": "./tests/detector_tests.rs",
+      "line_number": 476,
+      "finding_type": "Random String",
+      "matched_content_hash": "f56884431ad6ea3a4fb741ca53cae314b4d3202d6f14bf69aafe86440f799403",
+      "plugin_name": "RandomString"
+    },
+    {
+      "file_path": "./tests/fixtures/fp_corpus/README.md",
+      "line_number": 3,
+      "finding_type": "Email Address",
+      "matched_content_hash": "ebe162e5d3cc06b42201b0bfe39fde3379a3fe97c9836631f1750f21db142808",
+      "plugin_name": "EmailDetector"
+    },
+    {
+      "file_path": "./tests/fixtures/fp_corpus/package.json",
+      "line_number": 3,
+      "finding_type": "Base64 Encoded String",
+      "matched_content_hash": "097eda6585aa25d794c938beab55a00e7f2e7471a39375a3356535aa2e76efa6",
+      "plugin_name": "Base64Detector"
     }
   ]
 }

src/detector.rs

@@ -140,6 +140,28 @@ mod verhoeff_tests {
         assert!(!passes_verhoeff("446655440000"));
         assert!(passes_verhoeff("100000000004"));
     }
+
+    #[test]
+    fn verhoeff_handles_separators_and_partial_digits() {
+        // Separators are ignored: a real-world spelled number passes.
+        assert!(passes_verhoeff("2341 2341 2346"));
+        assert!(passes_verhoeff("2341-2341-2346"));
+        // Non-digit characters are filtered, so the digit suffix decides.
+        assert!(passes_verhoeff("id: 2363!"));
+        assert!(!passes_verhoeff("id: 2362!"));
+    }
+
+    #[test]
+    fn verhoeff_rejects_degenerate_inputs() {
+        assert!(!passes_verhoeff(""));
+        assert!(!passes_verhoeff("no digits here"));
+        // The p-permutation breaks the all-zeros identity chain, so the
+        // classic dummy number does NOT validate: good for a secret scanner.
+        assert!(!passes_verhoeff("000000000000"));
+        // Single digit: 0 is its own inverse, so "0" validates.
+        assert!(passes_verhoeff("0"));
+        assert!(!passes_verhoeff("1"));
+    }
 }
 
 /// Luhn checksum, ignoring embedded separators.
@@ -259,11 +281,35 @@ impl Detector {
     }
 
     /// Whether the pattern carries the dot-matches-newline flag anywhere —
-    /// `(?s)` or the grouped `(?s:...)` form — and must therefore run per
-    /// chunk instead of per line, or multiline secrets slip past it. Single
-    /// source for the production partition and the tests.
+    /// `(?s)`, a combined group like `(?is)`, or the scoped `(?s:...)` form —
+    /// and must therefore run per chunk instead of per line, or multiline
+    /// secrets slip past it. Single source for the production partition and
+    /// the tests.
     pub fn is_multiline(&self) -> bool {
-        self.regex.as_str().contains("(?s")
+        let pattern = self.regex.as_str();
+        let bytes = pattern.as_bytes();
+        let mut i = 0;
+        while i + 1 < bytes.len() {
+            if bytes[i] == b'(' && bytes[i + 1] == b'?' && (i == 0 || bytes[i - 1] != b'\\') {
+                // A flag group: scan its flag characters up to the closing
+                // paren or the group separator.
+                let mut j = i + 2;
+                let mut saw_s = false;
+                while j < bytes.len() && bytes[j] != b')' && bytes[j] != b':' {
+                    if bytes[j] == b's' {
+                        saw_s = true;
+                    }
+                    j += 1;
+                }
+                if saw_s {
+                    return true;
+                }
+                i = j;
+            } else {
+                i += 1;
+            }
+        }
+        false
     }
 
     pub fn has_sufficient_entropy(&self, matched: &str) -> bool {
@@ -459,3 +505,58 @@ fn initialize_detectors_from_config(
         .collect::<Result<Vec<_>, _>>()
         .map_err(|source| DetectorInitError::InvalidDetector { source })
 }
+
+#[cfg(test)]
+mod accept_unit_tests {
+    use super::{Detector, passes_luhn, shannon_entropy};
+
+    fn detector(pattern: &str, keywords: &[&str]) -> Detector {
+        let keywords: Vec<String> = keywords.iter().map(|k| k.to_string()).collect();
+        Detector::new("T", pattern, "T", "LOW", &[], &keywords, None).expect("valid detector")
+    }
+
+    #[test]
+    fn luhn_accepts_known_cards_and_separators() {
+        assert!(passes_luhn("4111111111111111"));
+        assert!(passes_luhn("4111-1111-1111-1111"));
+        assert!(passes_luhn("4111 1111 1111 1111"));
+        assert!(!passes_luhn("4111111111111112"));
+        assert!(!passes_luhn(""));
+        assert!(!passes_luhn("no digits"));
+        // Below the 13-digit floor.
+        assert!(!passes_luhn("41111111111"));
+    }
+
+    #[test]
+    fn shannon_entropy_matches_information_theory() {
+        assert_eq!(shannon_entropy(""), 0.0);
+        assert_eq!(shannon_entropy("aaaa"), 0.0);
+        assert!((shannon_entropy("ab") - 1.0).abs() < 1e-9);
+        assert!((shannon_entropy("abab") - 1.0).abs() < 1e-9);
+        // 64-char hex: near but under the 4.0 ceiling.
+        let hex = "8b0e7153bf7c3706d85c524e440066559a6656c90bd5482a90a29b9fa5ff5180";
+        let entropy = shannon_entropy(hex);
+        assert!(entropy > 3.5 && entropy < 4.0, "hex entropy was {entropy}");
+        // Multi-byte UTF-8 counts bytes.
+        assert!(shannon_entropy("\u{1F600}") > 0.0);
+    }
+
+    #[test]
+    fn is_multiline_matches_only_the_dotall_flag() {
+        assert!(detector(r"(?s)BEGIN.*END", &[]).is_multiline());
+        assert!(detector(r"(?s:BEGIN.*END)", &[]).is_multiline());
+        assert!(detector(r"(?is)BEGIN.*END", &[]).is_multiline());
+        assert!(!detector(r"SECRET_\w+", &[]).is_multiline());
+        assert!(!detector(r"(?i)secret", &[]).is_multiline());
+    }
+
+    #[test]
+    fn keywords_are_lowercased_for_the_prefilter() {
+        // Contract: callers lowercase content once per line; stored keywords
+        // are lowercased at construction.
+        let detector = detector(r"SECRET_\w+", &["ApiKey", "SECRET"]);
+        assert!(detector.has_keywords("set the apikey value"));
+        assert!(detector.has_keywords(&"THE SECRET VALUE".to_lowercase()));
+        assert!(!detector.has_keywords("nothing relevant"));
+    }
+}

src/report.rs

@@ -107,7 +107,7 @@ pub enum ScanStatus {
     Fail,
 }
 
-#[derive(Serialize, Clone)]
+#[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct Finding {
     pub file_path: String,
     pub line_number: usize,
@@ -118,7 +118,7 @@ pub struct Finding {
     /// detectors; the wire format keeps the historical `plugin_name` key
     /// (with an alias on deserialize) so existing report consumers are
     /// unaffected.
-    #[serde(rename = "plugin_name", alias = "plugin_name")]
+    #[serde(rename = "plugin_name", alias = "detector_name")]
     pub detector_name: String,
 }
 

tests/detector_tests.rs

@@ -377,3 +377,104 @@ fn test_high_entropy_hex_needs_credential_context() {
         "a bare hex digest is indistinguishable from a hash and must not fire"
     );
 }
+
+#[test]
+fn test_aws_example_key_is_allowlisted_but_real_keys_report() {
+    // The AWS documentation example key/secret appear in READMEs everywhere.
+    assert!(
+        !reported_by("aws_access_key_id = AKIAIOSFODNN7EXAMPLE")
+            .contains(&"AWSKeyDetector".to_string())
+    );
+    assert!(
+        !reported_by("secret = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+            .iter()
+            .any(|name| name == "Base64Detector" || name == "AWSKeyDetector")
+    );
+    assert!(
+        reported_by("aws_access_key_id = AKIA1234567890ABCDEF")
+            .contains(&"AWSKeyDetector".to_string()),
+        "any other AKIA key must still be reported"
+    );
+}
+
+#[test]
+fn test_placeholder_values_are_allowlisted_in_both_detectors() {
+    let placeholders = [
+        "API_KEY=your-api-key-here",
+        "DATABASE_PASSWORD=changeme",
+        "SECRET_KEY=replace-me-please",
+        "PASSWORD: changeme",
+        "TOKEN=xxxxxxxxxxxxxxxxxxxxxxxxx",
+    ];
+    for line in &placeholders {
+        let names = reported_by(line);
+        assert!(
+            !names.contains(&"GenericKeyValueDetector".to_string())
+                && !names.contains(&"PasswordDetector".to_string()),
+            "{line} must not report, got {names:?}"
+        );
+    }
+
+    // Mixed-case or digit-carrying values do not match the placeholder
+    // shapes and stay reported.
+    for line in [
+        "password = 'mySecretPassword'",
+        "token = YourSpecialToken123",
+        "pwd: ReplaceThisRealSecret123",
+        "api_key = \"sk_live_51abcdefghij\"",
+    ] {
+        let names = reported_by(line);
+        assert!(
+            names.contains(&"GenericKeyValueDetector".to_string())
+                || names.contains(&"PasswordDetector".to_string()),
+            "{line} must still be reported, got {names:?}"
+        );
+    }
+}
+
+#[test]
+fn test_email_allowlists_documentation_domains_but_not_real_ones() {
+    for email in [
+        "contact: support@example.com",
+        "alice@example.org",
+        "reply-to: noreply@github.com",
+        "author: 12345+user@users.noreply.github.com",
+    ] {
+        assert!(
+            !reported_by(email).contains(&"EmailDetector".to_string()),
+            "{email} must not report"
+        );
+    }
+    assert!(
+        reported_by("owner: bob.smith@company.io").contains(&"EmailDetector".to_string()),
+        "a real-looking address is still reported"
+    );
+}
+
+#[test]
+fn test_fictional_555_numbers_are_allowlisted() {
+    for phone in ["call 555-123-4567", "(212) 555-0123", "fax 555 123 4567"] {
+        assert!(
+            !reported_by(phone).contains(&"PhoneNumberDetector".to_string()),
+            "{phone} must not report"
+        );
+    }
+    assert!(
+        reported_by("call 415-123-4567").contains(&"PhoneNumberDetector".to_string()),
+        "a real-shaped number is still reported"
+    );
+}
+
+#[test]
+fn test_checksum_prefix_is_allowlisted_in_random_string() {
+    let line = r#"integrity = "sha512-abcdefghijklmnopqrstuvwxyz0123456789ABCDEFG""#;
+    assert!(
+        !reported_by(line).contains(&"RandomString".to_string()),
+        "sha-prefixed checksums must not report"
+    );
+    assert!(
+        reported_by(r#"token = "AbCdEfGhIjKlMnOpQrStUvWxYz0123456789abcd""#)
+            .contains(&"RandomString".to_string()),
+        "a quoted random string without a checksum prefix still reports"
+    );
+}

tests/fixtures/fp_corpus/.env.example

@@ -0,0 +1,6 @@
+API_KEY=your-api-key-here
+DATABASE_PASSWORD=changeme
+SECRET_KEY=replace-me-please
+JWT_SECRET=xxxxxxxxxxxxxxxxxxxxxxxxx
+AWS_SECRET_ACCESS_KEY=<your-aws-secret>
+TOKEN=$INTERPOLATED_VALUE