From d87aca4997ed1ca2b24ca3601aaaa1c142bef050 Mon Sep 17 00:00:00 2001 From: Ashay Date: Sun, 9 Aug 2026 10:33:29 +0530 Subject: [PATCH] fix(checks): catch digit-level confusions on account numbers Expand confusable pairs beyond -teen/-ty with oh/zero, a/eight, to/two, double-digit readbacks, and narrowband letter swaps. Treat confusable swaps as high-severity misheard_number and pin an account-number fixture. Fixes #4 --- fixtures/misheard_account_number.json | 33 ++++++++++++++++++++ tests/test_voiceeval.py | 19 +++++++++++ voiceeval/checks.py | 45 ++++++++++++++++++++++++--- 3 files changed, 92 insertions(+), 5 deletions(-) create mode 100644 fixtures/misheard_account_number.json diff --git a/fixtures/misheard_account_number.json b/fixtures/misheard_account_number.json new file mode 100644 index 0000000..581c64a --- /dev/null +++ b/fixtures/misheard_account_number.json @@ -0,0 +1,33 @@ +{ + "id": "lookup-misheard-account-oh-zero", + "policy": {"max_refund": 50}, + "completed": true, + "turns": [ + { + "speaker": "agent", + "text": "Hi, can I have your account number?", + "start_s": 0.0, + "end_s": 1.8 + }, + { + "speaker": "user", + "text": "It is four two oh nine eight.", + "truth": "It is four two zero nine eight.", + "start_s": 2.2, + "end_s": 5.0 + }, + { + "speaker": "agent", + "text": "Looking up four two oh nine eight now.", + "start_s": 5.4, + "end_s": 7.5, + "actions": [ + { + "name": "lookup_account", + "args": {"account": "42098"}, + "consequential": true + } + ] + } + ] +} diff --git a/tests/test_voiceeval.py b/tests/test_voiceeval.py index c148994..bb5109c 100644 --- a/tests/test_voiceeval.py +++ b/tests/test_voiceeval.py @@ -216,3 +216,22 @@ def test_from_dict_roundtrip(): ) assert inter.turns[0].speaker == "user" assert inter.policy == {} + + +# --------------------------------------------------------------------------- digit-level confusions (issue #4) + + +def test_misheard_account_number_oh_vs_zero_is_caught(): + """Card/account readbacks: STT heard 'oh' when the caller said 'zero'.""" + inter = load(FIXTURES / "misheard_account_number.json") + findings = analyse(inter) + misheard = [f for f in findings if f.check == "misheard_number"] + assert misheard, "oh/zero swap on an account number was not caught" + assert misheard[0].severity == "high" + + +def test_digit_confusions_do_not_false_positive_on_good_call(): + """Existing clean fixture must stay clean after expanding confusable pairs.""" + inter = load(FIXTURES / "good_call.json") + assert analyse(inter) == [] + diff --git a/voiceeval/checks.py b/voiceeval/checks.py index f94a263..192ef6f 100644 --- a/voiceeval/checks.py +++ b/voiceeval/checks.py @@ -32,8 +32,10 @@ class Finding: turn_index: int | None = None -# Numbers that STT reliably confuses. The -teen/-ty pairs are the classic: one unstressed syllable +# Tokens STT reliably confuses. The -teen/-ty pairs are the classic: one unstressed syllable # apart, and both are plausible amounts, so neither the model nor a human reviewer notices. +# Digit-level and letter pairs are the ones that wreck account numbers, card readbacks, and +# reference codes on a phone line (issue #4). _CONFUSABLE = [ (r"\bfifteen\b", r"\bfifty\b"), (r"\bsixteen\b", r"\bsixty\b"), @@ -42,11 +44,32 @@ class Finding: (r"\bnineteen\b", r"\bninety\b"), (r"\bthirteen\b", r"\bthirty\b"), (r"\bfourteen\b", r"\bforty\b"), + # "oh" vs "zero": both are how people read "0" on a card or account number. + (r"\boh\b", r"\bzero\b"), + # "a" vs "eight": same one-phoneme trap as fifteen/fifty, common in spoken digits. + (r"\ba\b", r"\beight\b"), + # "to"/"too" vs "two": amount or quantity after a verb collapses into a preposition. + (r"\bto\b", r"\btwo\b"), + (r"\btoo\b", r"\btwo\b"), + # Repeated digits spoken as "double N" vs "N N" / "seventy seven" style. + (r"\bdouble seven\b", r"\bseventy seven\b"), + (r"\bdouble seven\b", r"\bseven seven\b"), + (r"\bdouble four\b", r"\bfour four\b"), + # Narrowband phone-line letter confusions in reference codes / postcodes. + (r"\bb\b", r"\bd\b"), + (r"\bb\b", r"\bp\b"), + (r"\bd\b", r"\bt\b"), + (r"\be\b", r"\bp\b"), + (r"\bt\b", r"\bv\b"), + (r"\bv\b", r"\bz\b"), ] -_NUMERIC = re.compile(r"\b\d+(?:\.\d+)?\b|\b(?:one|two|three|four|five|six|seven|eight|nine|ten|" - r"eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|" - r"twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred|thousand)\b", re.I) +_NUMERIC = re.compile( + r"\b\d+(?:\.\d+)?\b|\b(?:zero|oh|one|two|three|four|five|six|seven|eight|nine|ten|" + r"eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|" + r"twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred|thousand)\b", + re.I, +) _CONFIRM = re.compile( r"\b(?:just to confirm|confirm|did you say|is that right|correct\?|to be clear|" @@ -55,6 +78,18 @@ class Finding: ) +def _confusable_swap(heard: str, said: str) -> bool: + """True when a known confusable pair is swapped between STT text and ground truth.""" + h, s = heard.lower(), said.lower() + for left, right in _CONFUSABLE: + h_l, h_r = re.search(left, h) is not None, re.search(right, h) is not None + s_l, s_r = re.search(left, s) is not None, re.search(right, s) is not None + # Classic swap: truth has one form, STT has the other (and not both forms on both sides). + if (s_l and h_r and not s_r and not h_l) or (s_r and h_l and not s_l and not h_r): + return True + return False + + def check_misheard(inter: Interaction) -> list[Finding]: """STT heard something different from what was said. @@ -70,7 +105,7 @@ def check_misheard(inter: Interaction) -> list[Finding]: heard_nums = set(m.group(0).lower() for m in _NUMERIC.finditer(t.text)) said_nums = set(m.group(0).lower() for m in _NUMERIC.finditer(t.truth)) - if heard_nums != said_nums: + if heard_nums != said_nums or _confusable_swap(t.text, t.truth): out.append( Finding( "misheard_number",