[feat] Add internationalization to SAP (#2210)

we can now handle:
* accents / ligature errors by the LLM (both including and excluding
them). Validated for enums, literals, and class keys.
* class keys may now allow for whitespace mismatches,
case-insensitivity, etc.

<!-- ELLIPSIS_HIDDEN -->


----

> [!IMPORTANT]
> This PR adds internationalization support by enhancing string matching
to handle accents, ligatures, and case-insensitivity, with extensive
tests to validate the changes.
> 
>   - **Behavior**:
> - Enhances `match_string` in `match_string.rs` to handle accents,
ligatures, and case-insensitivity.
> - Supports whitespace mismatches and case-insensitivity for class
keys.
>   - **Dependencies**:
> - Adds `unicode-normalization` to `Cargo.toml` and `Cargo.lock` for
accent handling.
>   - **Tests**:
> - Adds `test_international.rs` with tests for accented and unaccented
inputs across enums, literals, and class keys.
> - Updates existing tests in `test_class.rs` and `test_enum.rs` to
validate new matching logic.
> 
> <sup>This description was created by </sup>[<img alt="Ellipsis"
src="https://img.shields.io/badge/Ellipsis-blue?color=175173">](https://www.ellipsis.dev?ref=BoundaryML%2Fbaml&utm_source=github&utm_medium=referral)<sup>
for 4f3e967. You can
[customize](https://app.ellipsis.dev/BoundaryML/settings/summaries) this
summary. It will automatically update as commits are pushed.</sup>


<!-- ELLIPSIS_HIDDEN -->

Author: hellovai

engine/Cargo.lock

@@ -35,6 +35,7 @@ either = "1.10.0"
 test-log = "0.2.16"
 regex.workspace = true
 thiserror = "2.0.9"
+unicode-normalization = "0.1.24"
 
 [dev-dependencies]
 assert-json-diff = "2.0.2"

engine/baml-lib/jsonish/Cargo.toml

@@ -89,7 +89,7 @@ impl TypeCoercer for LiteralValue {
                 // second element is the list of aliases.
                 let candidates = vec![(literal_str.as_str(), vec![literal_str.clone()])];
 
-                let literal_match = match_string(ctx, target, Some(value), &candidates)?;
+                let literal_match = match_string(ctx, target, Some(value), &candidates, true)?;
 
                 Ok(BamlValueWithFlags::String(literal_match))
             }

engine/baml-lib/jsonish/src/deserializer/coercer/coerce_literal.rs

@@ -273,6 +273,7 @@ pub(super) fn coerce_bool(
                             vec!["false".into(), "False".into(), "FALSE".into()],
                         ),
                     ],
+                    true,
                 ) {
                     Ok(val) => match val.value().as_str() {
                         "true" => Ok(BamlValueWithFlags::Bool(

engine/baml-lib/jsonish/src/deserializer/coercer/coerce_primitive.rs

@@ -8,8 +8,10 @@ use internal_baml_jinja::types::{Class, Name};
 use super::ParsingContext;
 use crate::deserializer::{
     coercer::{
-        array_helper, field_type::validate_asserts, run_user_checks, DefaultValue, ParsingError,
-        TypeCoercer,
+        array_helper,
+        field_type::validate_asserts,
+        match_string::{match_string, matches_string_to_string},
+        run_user_checks, DefaultValue, ParsingError, TypeCoercer,
     },
     deserialize_flags::{DeserializerConditions, Flag},
     types::BamlValueWithFlags,
@@ -93,7 +95,7 @@ impl TypeCoercer for Class {
                     if let Some(field) = self
                         .fields
                         .iter()
-                        .find(|(name, ..)| name.rendered_name().trim() == key.trim())
+                        .find(|(name, ..)| matches_string_to_string(ctx, key, name.rendered_name()))
                     {
                         let scope = ctx.enter_scope(field.0.real_name());
                         let parsed = field.1.coerce(&scope, &field.1, Some(v));

engine/baml-lib/jsonish/src/deserializer/coercer/ir_ref/coerce_class.rs

@@ -49,7 +49,7 @@ impl TypeCoercer for Enum {
             .find_enum(self.name.real_name())
             .map_or(vec![], |class| class.constraints.clone());
 
-        let variant_match = match_string(ctx, target, value, &enum_match_candidates(self))?;
+        let variant_match = match_string(ctx, target, value, &enum_match_candidates(self), true)?;
         let enum_match = apply_constraints(
             target,
             vec![],

engine/baml-lib/jsonish/src/deserializer/coercer/ir_ref/coerce_enum.rs

@@ -17,13 +17,32 @@ use crate::{
     jsonish,
 };
 
+pub(super) fn matches_string_to_string(
+    parsing_context: &ParsingContext,
+    raw_value: &str,
+    parse_into: &str,
+) -> bool {
+    match_string(
+        parsing_context,
+        &TypeIR::string(),
+        Some(&crate::jsonish::Value::String(
+            raw_value.to_string(),
+            baml_types::CompletionState::Complete,
+        )),
+        &[(parse_into, vec![parse_into.to_string()])],
+        false,
+    )
+    .is_ok()
+}
+
 /// Heuristic match of different possible values against an input string.
 pub(super) fn match_string(
     parsing_context: &ParsingContext,
     target: &TypeIR,
     value: Option<&jsonish::Value>,
     // List of (name, [aliases]) tuples.
     candidates: &[(&str, Vec<String>)],
+    allow_substring_match: bool,
 ) -> Result<ValueWithFlags<String>, ParsingError> {
     // Get rid of nulls.
     let value = match value {
@@ -52,7 +71,9 @@ pub(super) fn match_string(
     let match_context = jsonish_string.trim();
 
     // First attempt, case sensitive match ignoring possible pucntuation.
-    if let Some(string_match) = string_match_strategy(match_context, candidates, &mut flags) {
+    if let Some(string_match) =
+        string_match_strategy(match_context, candidates, &mut flags, allow_substring_match)
+    {
         return try_match_only_once(parsing_context, target, string_match, flags);
     }
 
@@ -69,7 +90,22 @@ pub(super) fn match_string(
     }));
 
     // Second attempt, case sensitive match without punctuation.
-    if let Some(string_match) = string_match_strategy(&match_context, &candidates, &mut flags) {
+    if let Some(string_match) = string_match_strategy(
+        &match_context,
+        &candidates,
+        &mut flags,
+        allow_substring_match,
+    ) {
+        return try_match_only_once(parsing_context, target, string_match, flags);
+    }
+
+    // Third attempt, case sensitive match without punctuation.
+    if let Some(string_match) = string_match_strategy(
+        &match_context,
+        &candidates,
+        &mut flags,
+        allow_substring_match,
+    ) {
         return try_match_only_once(parsing_context, target, string_match, flags);
     }
 
@@ -84,7 +120,12 @@ pub(super) fn match_string(
     });
 
     // There goes our last hope :)
-    if let Some(string_match) = string_match_strategy(&match_context, &candidates, &mut flags) {
+    if let Some(string_match) = string_match_strategy(
+        &match_context,
+        &candidates,
+        &mut flags,
+        allow_substring_match,
+    ) {
         return try_match_only_once(parsing_context, target, string_match, flags);
     }
 
@@ -97,6 +138,26 @@ fn strip_punctuation(s: &str) -> String {
         .collect::<String>()
 }
 
+/// Remove accents from characters to enable fuzzy matching of unaccented input
+/// against accented aliases/candidates.
+fn remove_accents(s: &str) -> String {
+    use unicode_normalization::UnicodeNormalization;
+
+    // Handle ligatures separately since they're not combining marks
+    let s = s
+        .replace('ß', "ss")
+        .replace('æ', "ae")
+        .replace('Æ', "AE")
+        .replace('ø', "o")
+        .replace('Ø', "O")
+        .replace('œ', "oe")
+        .replace('Œ', "OE");
+
+    s.nfkd()
+        .filter(|c| !unicode_normalization::char::is_combining_mark(*c))
+        .collect()
+}
+
 /// Helper function to return a single string match result.
 ///
 /// Multiple results will yield an error.
@@ -132,15 +193,47 @@ fn string_match_strategy<'c>(
     value_str: &str,
     candidates: &'c [(&'c str, Vec<String>)],
     flags: &mut DeserializerConditions,
+    allow_substring_match: bool,
 ) -> Option<&'c str> {
-    // Try and look for an exact match against valid values.
+    log::info!("string_match_strategy: {value_str}");
+    log::info!(
+        "candidates:\n{}",
+        candidates
+            .iter()
+            .map(|(c, v)| format!(
+                "{c} -> {}",
+                v.iter()
+                    .map(|v| format!("\"{v}\""))
+                    .collect::<Vec<_>>()
+                    .join(", ")
+            ))
+            .collect::<Vec<_>>()
+            .join("\n")
+    );
+    // Strategy 1: Try exact case-sensitive match
     for (candidate, valid_values) in candidates {
         if valid_values.iter().any(|v| v == value_str) {
-            // We did nothing fancy, so no extra flags.
+            // No flags since we found an exact match.
             return Some(candidate);
         }
     }
 
+    // Strategy 2: Try unaccented case-sensitive match
+    let unaccented_value_str = remove_accents(value_str);
+    for (candidate, valid_values) in candidates {
+        if valid_values
+            .iter()
+            .any(|v| remove_accents(v) == unaccented_value_str)
+        {
+            // No flags since we found an exact match.
+            return Some(candidate);
+        }
+    }
+
+    if !allow_substring_match {
+        return None;
+    }
+
     // (start_index, end_index, valid_name, variant)
     // TODO: Consider using a struct with named fields instead of a 4-tuple.
     let mut all_matches: Vec<(usize, usize, &'c str, &'c str)> = Vec::new();
@@ -155,6 +248,21 @@ fn string_match_strategy<'c>(
         }
     }
 
+    // No substring match at all for any variant, early return.
+    if all_matches.is_empty() {
+        // Try to see if we can find any substring matches in the unaccented
+        // value string.
+        for (variant, valid_names) in candidates {
+            for valid_name in valid_names {
+                let unaccented_valid_name = remove_accents(valid_name);
+                for (start_idx, _) in unaccented_value_str.match_indices(&unaccented_valid_name) {
+                    let end_idx = start_idx + valid_name.len();
+                    all_matches.push((start_idx, end_idx, valid_name, variant));
+                }
+            }
+        }
+    }
+
     // No substring match at all for any variant, early return.
     if all_matches.is_empty() {
         return None;
@@ -218,3 +326,99 @@ fn string_match_strategy<'c>(
     // No match found.
     None
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_remove_accents_etude() {
+        assert_eq!(remove_accents("étude"), "etude");
+    }
+
+    #[test]
+    fn test_remove_accents_francais() {
+        assert_eq!(remove_accents("français"), "francais");
+    }
+
+    #[test]
+    fn test_remove_accents_espanol() {
+        assert_eq!(remove_accents("Español"), "Espanol");
+    }
+
+    #[test]
+    fn test_remove_accents_portugues() {
+        assert_eq!(remove_accents("português"), "portugues");
+    }
+
+    #[test]
+    fn test_remove_accents_medium() {
+        assert_eq!(remove_accents("médium"), "medium");
+    }
+
+    #[test]
+    fn test_remove_accents_grun() {
+        assert_eq!(remove_accents("Grün"), "Grun");
+    }
+
+    #[test]
+    fn test_remove_accents_uber() {
+        assert_eq!(remove_accents("Über"), "Uber");
+    }
+
+    #[test]
+    fn test_remove_accents_strasse() {
+        assert_eq!(remove_accents("Straße"), "Strasse");
+    }
+
+    #[test]
+    fn test_remove_accents_stadt() {
+        assert_eq!(remove_accents("Stadt"), "Stadt");
+    }
+
+    #[test]
+    fn test_remove_accents_ae_ligature() {
+        assert_eq!(remove_accents("æ"), "ae");
+        assert_eq!(remove_accents("Æ"), "AE");
+    }
+
+    #[test]
+    fn test_remove_accents_o_ligature() {
+        assert_eq!(remove_accents("ø"), "o");
+        assert_eq!(remove_accents("Ø"), "O");
+    }
+
+    #[test]
+    fn test_remove_accents_oe_ligature() {
+        assert_eq!(remove_accents("œ"), "oe");
+        assert_eq!(remove_accents("Œ"), "OE");
+    }
+
+    #[test]
+    fn test_remove_accents_danish_word() {
+        assert_eq!(remove_accents("København"), "Kobenhavn");
+    }
+
+    #[test]
+    fn test_remove_accents_french_word() {
+        assert_eq!(remove_accents("cœur"), "coeur");
+        assert_eq!(remove_accents("œuvre"), "oeuvre");
+    }
+
+    #[test]
+    fn test_remove_accents_mixed_ligatures() {
+        assert_eq!(
+            remove_accents("Straße ældre øl œuvre"),
+            "Strasse aeldre ol oeuvre"
+        );
+    }
+
+    #[test]
+    fn test_remove_accents_non_alphanumeric() {
+        // It correctly leaves non-alphanumeric ASCII and other scripts alone, but converts ligatures
+        assert_eq!(
+            remove_accents("ß, æ, ø, œ, こんにちは, 🦄"),
+            "ss, ae, o, oe, こんにちは, 🦄"
+        );
+    }
+}

engine/baml-lib/jsonish/src/deserializer/coercer/match_string.rs

@@ -9,6 +9,7 @@ mod test_class_2;
 mod test_code;
 mod test_constraints;
 mod test_enum;
+mod test_international;
 mod test_lists;
 mod test_literals;
 mod test_maps;