Merge pull request #432 from streamdal/blinktag/wasm_fix_detective

wasm: Fix plaintext detection in mixed plaintext/json data

apps/server/assets/wasm/download-wasm.sh

@@ -18,7 +18,7 @@ fi
 
 # Get the version from the version file
 version=$(cat version)
-download_url="https://github.com/streamdal/streamdal/releases/download/libs%2Fwasm%2F${version}/release.zip"
+download_url="https://github.com/streamdal/streamdal/releases/download/libs/wasm/${version}/release.zip"
 
 # Remove previous release.zip (if exists)
 rm -f release.zip

libs/wasm-detective/src/detective.rs

@@ -458,11 +458,18 @@ pub fn plaintext(request: &Request, input: &str) -> Vec<DetectiveStepResultMatch
         // split on them since they might be inside an email, ipv6, macaddr, etc.
         let no_split: HashMap<char, usize> = HashMap::from([('@', 0), (':', 0)]);
 
+        // Ignore these characters so that we can properly find PII inside JSON that is inside a plaintext string
+        let ignore_chars = HashMap::from([('"', 0), ('{', 0), ('}', 0), ('\'', 0)]);
+
         // This is our word-part accumulator and is used to
         // combine "user", "@", "somedomain.com" words into a single word
         let mut accum: Vec<Word> = Vec::new();
 
         for (word_start, word) in words {
+            if word.len() == 1 && ignore_chars.contains_key(&word.chars().next().unwrap()) {
+                continue;
+            }
+
             // We've reached the end of a word, combine our accumulator into a single Word struct
             if word == " " {
                 // Loop over accumulator and join the string value of each word

libs/wasm-detective/src/matcher_pii_tests.rs

@@ -940,6 +940,26 @@ fn test_plaintext() {
     // assert_eq!(&results[3].pii_type, "Billing");
 }
 
+#[test]
+fn test_plaintext_mixed() {
+    let sample_text = "2024-04-29T15:59:41.60221515Z stdout Exporting data {\"user\": {\"ccnum\": \"4111111111111111\"}} to billing service";
+
+    let request = &Request {
+        match_type: DetectiveType::DETECTIVE_TYPE_PII_PLAINTEXT_ANY,
+        data: &sample_text.as_bytes().to_vec(),
+        path: "".to_string(),
+        args: Vec::new(),
+        negate: false,
+        mode: DETECTIVE_TYPE_PII_KEYWORD_MODE_PERFORMANCE,
+        data_format: PIPELINE_DATA_FORMAT_JSON,
+    };
+
+    let results = crate::detective::Detective::new().matches(&request).unwrap();
+
+    assert_eq!(results.len(), 1);
+
+}
+
 #[bench]
 fn bench_plaintext(b: &mut Bencher) {
     b.iter(|| {