From 9469f63b2b98edc369429ced58a8eda442d6fdd6 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 1 Aug 2026 19:38:52 -0700 Subject: [PATCH 01/14] Make the stage's two splits siblings, not a cascade (#312) --- nameparser/_pipeline/_script_segment.py | 87 +++++++++++++++---------- 1 file changed, 51 insertions(+), 36 deletions(-) diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 55b1d29..888698e 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -325,7 +325,7 @@ def _peel_honorific_tail(state: ParseState) -> ParseState: agree under NO_COMMA except where extract_delimited has already claimed a token, which is still in the stream here with only its role set -- so "김민준씨 (Jimmy)" is the input that tells them - apart. See the note above the segment lookup in script_segment. + apart. See the note above the segment lookup in _split_surname_site. That last token is the last of the NAME PART, which under NO_COMMA reaches into a maiden clause: maiden tokens are still main-stream @@ -387,41 +387,18 @@ def _peel_honorific_tail(state: ParseState) -> ParseState: return state -def script_segment(state: ParseState) -> ParseState: - if state.original.isascii(): - # spans index the original exactly (the anti-#100 invariant), - # so an ASCII original has only ASCII tokens: nothing here is - # in any script's ranges. It also short-circuits the PEEL, - # which has no script gate of its own -- so a caller-configured - # ASCII tail never fires, and one non-ASCII character anywhere - # in the name switches it on. Correct for the CJK vocabulary - # that ships, stated because honorific_tails is public: see - # that field's own note. - return state - if not state.segments: - return state - if state.structure is Structure.FAMILY_COMMA: - return state # the comma already drew the boundary - if state.interpunct_offsets: - # #298: a 间隔号-divided name is a transcription -- its pieces - # are syllable groups, not surname+given, so neither the - # vocabulary nor the segmenter applies (codepoint-scoped: the - # nakaguro records nothing and gates nothing, spec decision 5). - # State-global like the FAMILY_COMMA gate above: a marker - # anywhere in the name reads the WHOLE name as a transcription - # listing, so even an un-dotted hangul token beside a dotted - # one stays whole. - return state - # #308: the honorific peel runs after the structural gates above - # (a FAMILY-comma or dot-divided name opts out of this stage - # whole; a suffix comma does not -- its name part still peels) - # but BEFORE the activation gate below, and independently of it -- - # 田中さん must peel under the DEFAULT policy, where HAN is in no - # activation set. The activation gate is about which - # scripts a SURNAME VOCABULARY may be trusted to divide; the peel - # asks nothing of the remainder, only whether the token ends in a - # word that can never end a name. - state = _peel_honorific_tail(state) +def _split_surname_site(state: ParseState) -> ParseState: + """The stage's other split: the first activated-script token of the + name part is matched longest-first against Lexicon.surnames, and a + hit splits it in two; where the vocabulary declines, an optional + Parser(segmenter=...) is consulted instead. + + A sibling of _peel_honorific_tail rather than a continuation of it. + The two answer different questions -- this one asks where a name + divides into surname and given, the peel asks whether a token ends + in a word that can never end a name -- which is why they carry + different gates, and why the stage entry below is preconditions + plus two calls rather than one cascade.""" scripts = state.policy.segment_scripts # an empty VOCABULARY deliberately does not bail here -- see below if not scripts: @@ -574,3 +551,41 @@ def script_segment(state: ParseState) -> ParseState: f"scoring {conf:.2f}, under the " f"{_SEGMENTER_CONFIDENCE_FLOOR} confidence floor") return _split(state, i, answer.splits, detail) + + +def script_segment(state: ParseState) -> ParseState: + if state.original.isascii(): + # spans index the original exactly (the anti-#100 invariant), + # so an ASCII original has only ASCII tokens: nothing here is + # in any script's ranges. It also short-circuits the PEEL, + # which has no script gate of its own -- so a caller-configured + # ASCII tail never fires, and one non-ASCII character anywhere + # in the name switches it on. Correct for the CJK vocabulary + # that ships, stated because honorific_tails is public: see + # that field's own note. + return state + if not state.segments: + return state + if state.structure is Structure.FAMILY_COMMA: + return state # the comma already drew the boundary + if state.interpunct_offsets: + # #298: a 间隔号-divided name is a transcription -- its pieces + # are syllable groups, not surname+given, so neither the + # vocabulary nor the segmenter applies (codepoint-scoped: the + # nakaguro records nothing and gates nothing, spec decision 5). + # State-global like the FAMILY_COMMA gate above: a marker + # anywhere in the name reads the WHOLE name as a transcription + # listing, so even an un-dotted hangul token beside a dotted + # one stays whole. + return state + # #308: the honorific peel runs after the structural gates above + # (a FAMILY-comma or dot-divided name opts out of this stage + # whole; a suffix comma does not -- its name part still peels) + # but BEFORE the activation gate below, and independently of it -- + # 田中さん must peel under the DEFAULT policy, where HAN is in no + # activation set. The activation gate is about which + # scripts a SURNAME VOCABULARY may be trusted to divide; the peel + # asks nothing of the remainder, only whether the token ends in a + # word that can never end a name. + state = _peel_honorific_tail(state) + return _split_surname_site(state) From c89536ba40117f5b3d12802bdaba762a673c26fe Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 1 Aug 2026 19:49:26 -0700 Subject: [PATCH 02/14] =?UTF-8?q?Let=20the=20peel=20cross=20a=20comma=20an?= =?UTF-8?q?d=20a=20=E9=97=B4=E9=9A=94=E5=8F=B7=20(#312)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nameparser/_pipeline/_script_segment.py | 160 ++++++++++++++--------- tests/v2/cases.py | 12 +- tests/v2/pipeline/test_script_segment.py | 73 +++++++++-- 3 files changed, 167 insertions(+), 78 deletions(-) diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 888698e..ee38a91 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -1,14 +1,15 @@ -"""Stage: script_segment (#271, #272, #308). +"""Stage: script_segment (#271, #272, #308, #312). Consumes: tokens, segments, structure, interpunct_offsets, segmenter. Produces: tokens, by two independent splits into sub-slices -- a -listed honorific peeled off the END of the name part's last -non-post-nominal token (whose tail token also carries this module's -_PEELED_TAG), and the first activated-script token of the name segment -split into n+1 -- segments (index runs remapped past the insertions), -ambiguities (indices likewise remapped, plus a SEGMENTATION report -when more than one split was vocabulary-supported, or when a -segmenter's answer scored under the confidence floor). +listed honorific peeled off the END of the name's last +non-post-nominal token, in whichever of the name's runs that falls +(the tail token also carries this module's _PEELED_TAG), and the +first activated-script token of the name segment split into n+1 -- +segments (index runs remapped past the insertions), ambiguities +(indices likewise remapped, plus a SEGMENTATION report when more than +one split was vocabulary-supported, or when a segmenter's answer +scored under the confidence floor). Reads: Policy.segment_scripts, Lexicon.surnames, Lexicon.honorific_tails, ParseState.segmenter, and Lexicon suffix vocabulary (via _vocab.is_suffix_strict) -- both the peel's scan-back @@ -24,18 +25,23 @@ invariant holds by construction. A second, independent split runs first (#308): a listed honorific -glued to the END of the name part's last non-post-nominal token is -peeled off as its own token -- 田中さん -> 田中 + さん -- so that -suffix classification can claim it and the surname match or segmenter -consult below sees the name rather than the name plus an honorific. It -is gated by the stage's own ASCII bail, by the FAMILY comma and by -间隔号, but NOT by segment_scripts: the vocabulary of tails is licensed -by the entries themselves, each of which can never end a name, so no -per-script trust question arises. The ASCII bail is the gate a caller -adding a LATIN tail meets -- it sits above everything here, so such a -tail fires only on a name carrying at least one non-ASCII character -(see the bail's own comment, and honorific_tails' field note). A -suffix comma gates nothing -- "Dr 김민준씨, Jr." peels within its name +glued to the END of the name's last non-post-nominal token is peeled +off as its own token -- 田中さん -> 田中 + さん -- so that suffix +classification can claim it and the surname match or segmenter consult +below sees the name rather than the name plus an honorific. No +STRUCTURAL gate stands over it -- the only things above it are the +stage's own two preconditions, the ASCII bail and non-empty segments. +Not segment_scripts either: the vocabulary of tails is licensed by the +entries themselves, each of which can never end a name, so no +per-script trust question arises. And since #312 not the FAMILY comma +or the 间隔号 either -- both of those answer where a name divides into +surname and given, which the peel never asks, so a comma or a dot +elsewhere in the string cannot change whether a token ends in a word +that can never end a name. The ASCII bail is the gate a caller adding +a LATIN tail meets -- it sits above everything here, so such a tail +fires only on a name carrying at least one non-ASCII character (see +the bail's own comment, and honorific_tails' field note). A suffix +comma gates nothing either -- "Dr 김민준씨, Jr." peels within its name part like any other. Where the VOCABULARY declines -- no prefix matched -- an optional @@ -83,7 +89,9 @@ structure, not the split's source, so it covers the segmenter identically: the opt-out runs before either is consulted. The post-comma side is given-name text, no surname site either, so that -structure opts out whole. +structure opts out of the SURNAME SPLIT whole -- of the stage entire +until #312 moved the peel in front of the gate, an honorific being no +part of the name whichever side of the comma it was glued to. NO_COMMA and SUFFIX_COMMA still split, within segments[0] (the name part) only. Running after segment costs the index remaps below; running BEFORE it would have made the comma structure itself depend @@ -298,8 +306,8 @@ def _is_post_nominal(state: ParseState, i: int) -> bool: def _peel_honorific_tail(state: ParseState) -> ParseState: - """#308: split a listed honorific off the END of the name part's - last NON-POST-NOMINAL token -- 田中さん -> 田中 + さん -- and let + """#308: split a listed honorific off the END of the name's last + NON-POST-NOMINAL token -- 田中さん -> 田中 + さん -- and let the existing machinery do the rest. Suffix classification claims the tail downstream (every honorific_tails entry is a suffix word too, enforced by Lexicon), and the segmentation half below then @@ -315,26 +323,29 @@ def _peel_honorific_tail(state: ParseState) -> ParseState: which the cap alone would peel to 선생 + 님) is skipped as a site and stays whole: every tail is a suffix word by the Lexicon invariant, which is what makes that guard hold, and the cap below - only keeps the offset interior for _split. And segments[0] is - reachable while empty -- a leading comma yields one -- so the scan - that returns None where the outright index would raise removes an - unpinned reliance on the FAMILY_COMMA gate landing first. - - WHICH run is scanned is a separate decision, and the one a reader - is likeliest to undo: segments[0], never state.tokens. The two - agree under NO_COMMA except where extract_delimited has already - claimed a token, which is still in the stream here with only its - role set -- so "김민준씨 (Jimmy)" is the input that tells them - apart. See the note above the segment lookup in _split_surname_site. - - That last token is the last of the NAME PART, which under NO_COMMA + only keeps the offset interior for _split. And the scan answers + None rather than indexing anything, which two reachable inputs + need: a name that is nothing but post-nominals ("씨"), and one + whose name runs are empty (", , 씨" scopes to two empty ones). + Nothing here rests on a structural gate landing first, then, which + is what let #312 move both of the stage's gates below it. + + WHICH tokens are scanned is a separate decision, and the one a + reader is likeliest to undo: the NAME's segment runs, flattened, + and never state.tokens or every segment. The alternatives agree + except where extract_delimited has already claimed a token or a + comma has closed the name, both of which this stage can still see + -- so "김민준씨 (Jimmy)" and "Dr 김민준씨, V." are the inputs that + tell the three apart. See the note at the scan itself. + + That last token is the last of the NAME, which under NO_COMMA reaches into a maiden clause: maiden tokens are still main-stream here (extract_delimited has masked only bracketed content), so "김민준 née 박씨" peels 씨 off the MAIDEN name 박씨 and hands it to the person's suffix list -- "née Ms. Park". Intended rather than incidental in that direction: the honorific is the reader's regardless of which of her names it was glued to, and a - name-part-final honorific is exactly what this peels. + name-final honorific is exactly what this peels. The other direction is a LIMIT, stated rather than fixed: a maiden clause pushes the site off the person's own name, so "김민준씨 née @@ -366,7 +377,31 @@ def _peel_honorific_tail(state: ParseState) -> ParseState: tails = state.lexicon.honorific_tails if not tails: return state - i = next((j for j in reversed(state.segments[0]) + # The NAME's runs, which is more than segments[0] but never every + # segment. A comma is how a writer says which runs are the name: a + # FAMILY comma splits the name itself across two of them ("김, + # 민준씨" is (0,) and (1,)), and the honorific is as often glued to + # the given name as to the family, so the peel has to cross that + # boundary (#312). Every OTHER structure keeps the whole name in + # segments[0] and puts post-nominals after it -- which is why the + # asymmetry is the point rather than an accident of two cases. + # Reaching past the name's own runs is a live bug, not tidiness: + # segment admits a post-comma run on is_suffix_lenient while + # _is_post_nominal asks is_suffix_strict, and the initial-shaped + # suffix words ("V.", "V", "I") fall in that gap -- as a peel site + # such a token ends in no tail and silently abandons the peel, so + # "Dr 김민준씨, V." would strand 씨 in the given name. A junk tail + # is worse: "Dr 김민준씨, Jr., 박씨" would peel the 씨 off 박씨 and + # leave the person's own glued. + # Flattening the SEGMENTS rather than state.tokens is load-bearing + # too: extracted nickname and maiden content is in tokens but in NO + # segment, and scanning tokens would put the peel site on a + # nickname ("김민준씨 (Jimmy)" -> the site becomes Jimmy and + # nothing peels). ko_honorific_glued_given_nickname pins that. + runs = (state.segments[:2] if state.structure is Structure.FAMILY_COMMA + else state.segments[:1]) + flat = [j for seg in runs for j in seg] + i = next((j for j in reversed(flat) if not _is_post_nominal(state, j)), None) if i is None: return state @@ -397,8 +432,10 @@ def _split_surname_site(state: ParseState) -> ParseState: The two answer different questions -- this one asks where a name divides into surname and given, the peel asks whether a token ends in a word that can never end a name -- which is why they carry - different gates, and why the stage entry below is preconditions - plus two calls rather than one cascade.""" + different gates: the FAMILY comma and the 间隔号 gate this half + alone (#312), and segment_scripts below gates it alone too. That + is also why the stage entry below interleaves gates with its two + calls rather than running one cascade.""" scripts = state.policy.segment_scripts # an empty VOCABULARY deliberately does not bail here -- see below if not scripts: @@ -406,15 +443,12 @@ def _split_surname_site(state: ParseState) -> ParseState: # segments[0] is the NAME part under both remaining structures # (everything, under NO_COMMA); later segments are suffixes. Its # members are main-stream token indices by construction, so - # extracted nickname/maiden content is unreachable from here. - # For the surname site that is merely tidy -- the first - # script-written token is the same either way. For the PEEL above - # it is load-bearing, and iterating state.tokens instead is a - # silent regression rather than a refactor: extracted content is - # still IN the token stream at this stage (only its role is set), - # so "김민준씨 (Jimmy)" would give the scan-back Jimmy as its site, - # which is no post-nominal to step over and ends in no tail, and - # the peel would be lost. Pinned by the case row of that name. + # extracted nickname/maiden content is unreachable from here. For + # this site that is merely tidy -- the first script-written token + # is the same either way. It is the PEEL above that reads this run + # load-bearingly, and in the two structures that reach here it + # reads exactly this one, only backwards; the argument lives at + # that scan. i = next((i for i in state.segments[0] if effective_script(state.tokens[i].text) in scripts), None) # A post-nominal in the surname's own position is not a site to @@ -562,12 +596,23 @@ def script_segment(state: ParseState) -> ParseState: # ASCII tail never fires, and one non-ASCII character anywhere # in the name switches it on. Correct for the CJK vocabulary # that ships, stated because honorific_tails is public: see - # that field's own note. + # that field's own note. Latin does not glue post-nominals to + # names anyway, and where it does glue (Mr.Smith, Lt.Gov.) + # _vocab.period_joined_vocab classifies the token instead of + # splitting it. return state if not state.segments: return state + # #312: the peel runs in front of both gates below, because both + # answer where a name divides into surname and given, and the peel + # does not ask that. It asks whether a token ends in a word that + # can never end a name, and a comma or a dot elsewhere in the + # string does not change the answer. Placing it above the block + # also keeps a future segmentation gate from silently capturing + # it: a new gate lands with its siblings, below. + state = _peel_honorific_tail(state) if state.structure is Structure.FAMILY_COMMA: - return state # the comma already drew the boundary + return state # the comma already drew the SURNAME boundary if state.interpunct_offsets: # #298: a 间隔号-divided name is a transcription -- its pieces # are syllable groups, not surname+given, so neither the @@ -576,16 +621,7 @@ def script_segment(state: ParseState) -> ParseState: # State-global like the FAMILY_COMMA gate above: a marker # anywhere in the name reads the WHOLE name as a transcription # listing, so even an un-dotted hangul token beside a dotted - # one stays whole. + # one stays whole. Scoped to the surname split since #312: an + # honorific glued to a transcription is still an honorific. return state - # #308: the honorific peel runs after the structural gates above - # (a FAMILY-comma or dot-divided name opts out of this stage - # whole; a suffix comma does not -- its name part still peels) - # but BEFORE the activation gate below, and independently of it -- - # 田中さん must peel under the DEFAULT policy, where HAN is in no - # activation set. The activation gate is about which - # scripts a SURNAME VOCABULARY may be trusted to divide; the peel - # asks nothing of the remainder, only whether the token ends in a - # word that can never end a name. - state = _peel_honorific_tail(state) return _split_surname_site(state) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 0dab9d2..e3950ae 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -946,9 +946,11 @@ def __post_init__(self) -> None: {"title": "Dr", "family": "김", "given": "민준", "suffix": "씨, Jr."}, classification="fix(#308)", - notes="the peel indexes the NAME part, not the token stream: " - "under a suffix comma segments[0] is a strict subset, " - "and the peel site is found within it. Pairs with " + notes="the peel scans the NAME's runs, not the token stream: " + "under a suffix comma that is segments[0] alone, a " + "strict subset, and the peel site is found within it " + "(a FAMILY comma is the one structure that splits the " + "name across two runs, #312). Pairs with " "ko_honorific_glued_given_trailing_suffix, whose " "comma-less spelling of the same name reaches the same " "answer by the scan-back instead"), @@ -956,9 +958,9 @@ def __post_init__(self) -> None: {"family": "김", "given": "민준", "suffix": "씨", "nickname": "Jimmy"}, classification="fix(#308)", - notes="the other half of indexing the NAME part: extracted " + notes="the other half of scanning the NAME's runs: extracted " "content is still in the token stream at this stage but " - "NOT in segments[0], so the scan-back never reaches " + "in NO segment, so the scan-back never reaches " "Jimmy. Scanning the tokens instead would take Jimmy as " "the site -- it is no post-nominal -- and lose the peel " "entirely, with 씨 back in the given name. Nothing else " diff --git a/tests/v2/pipeline/test_script_segment.py b/tests/v2/pipeline/test_script_segment.py index b12d281..be21a25 100644 --- a/tests/v2/pipeline/test_script_segment.py +++ b/tests/v2/pipeline/test_script_segment.py @@ -510,19 +510,19 @@ def test_an_unlisted_tail_never_peels() -> None: lexicon=_LEX_TAILS)) == ["김", "지양"] -def test_family_comma_skips_the_peel() -> None: - # the comma doctrine covers the peel with the rest of the stage: - # the writer already said where the family name ends - assert _texts(_run("田中さん, 太郎", lexicon=_LEX_TAILS)) == [ - "田中さん", "太郎"] - - -def test_interpunct_divided_name_never_peels() -> None: - # the transcription gate likewise: its pieces are syllable groups +def test_an_interpunct_gates_the_split_but_not_the_peel() -> None: + # The dot marks a transcription, which is a claim about where the + # name divides into syllable groups -- so it still gates the + # surname split. An honorific glued to a transcribed name is + # still an honorific, so the peel crosses it (#312). 威 is a + # surname in this lexicon precisely so the first clause bites: + # without the gate 威廉 would divide 威 + 廉. + lex = _LEX_TAILS.add(surnames={"威"}) state = segment(tokenize(ParseState( - original="威廉·莎士比亚先生", lexicon=_LEX_TAILS, policy=_HAN))) + original="威廉·莎士比亚先生", lexicon=lex, policy=_HAN))) assert state.interpunct_offsets - assert script_segment(state).tokens == state.tokens + out = script_segment(state) + assert _texts(out) == ["威廉", "莎士比亚", "先生"] def test_a_peeled_tail_is_not_a_surname_site() -> None: @@ -563,6 +563,57 @@ def test_the_peel_only_looks_at_the_last_non_post_nominal_token() -> None: lexicon=_LEX_TAILS)) == ["田中さん", "太郎"] +def test_the_peel_reaches_past_a_family_comma() -> None: + # A comma says where the FAMILY name ends. It says nothing about + # whether a trailing honorific is part of the name, so the peel + # does not inherit the gate that stops the surname split -- and + # under FAMILY_COMMA the name spans both segments, so the site + # scan has to cross the boundary to find 민준씨 at all. + assert _texts(_run("김, 민준씨", policy=_HANGUL, + lexicon=_LEX_TAILS)) == ["김", "민준", "씨"] + + +def test_the_peel_scans_the_name_runs_and_no_further() -> None: + # Crossing a FAMILY comma is not crossing every comma: past the + # name's own runs lie post-nominals, and taking one as the site + # abandons the peel. Two shapes, both silently broken by a scan + # over ALL segments. segment admits a post-comma run on + # is_suffix_lenient while the site scan asks is_suffix_strict, so + # an initial-shaped suffix word is a site rather than a token to + # step over -- "V." ends in no tail, and 씨 would stay glued. And + # a later junk segment would take the peel off the person's own + # name onto 박씨, peeling the wrong 씨 of two. + lex = _LEX_TAILS.add(suffix_words={"v"}) + assert _texts(_run("Dr 김민준씨, V.", policy=_HANGUL, + lexicon=lex)) == ["Dr", "김", "민준", "씨", "V."] + assert _texts(_run("Dr 김민준씨, Jr., 박씨", policy=_HANGUL, + lexicon=lex)) == [ + "Dr", "김", "민준", "씨", "Jr.", "박씨"] + + +def test_the_peel_survives_a_name_with_no_name_tokens() -> None: + # The name's runs are EMPTY -- a leading comma yields an empty one + # per comma -- so the scan has nothing to look at and answers None + # rather than indexing. Reachable, so taking the last token + # outright would raise here. The scan's other way to answer None, + # a name that is nothing but post-nominals, is not what this + # exercises: next() short-circuits on the empty runs and never + # asks. test_a_token_that_is_a_tail_never_peels pins that one. + assert _texts(_run(", , 씨", policy=_HANGUL, + lexicon=_LEX_TAILS)) == ["씨"] + + +def test_a_comma_does_not_by_itself_change_the_peel() -> None: + # The counterpart, and the shape #312 was originally filed about: + # the site is the last NON-POST-NOMINAL token, which is 太郎 here + # whichever spelling is used, so neither peels. Crossing the comma + # must not turn this into a peel. + for name in ("田中さん 太郎", "田中さん, 太郎"): + expected = name.replace(",", "").split() + assert _texts(_run(name, policy=_HANGUL, + lexicon=_LEX_TAILS)) == expected, name + + # -- the 间隔号 gate: a divided name is a transcription (#298) ---------- From b54f69260257f0df8cc9308af23015d0c732ac75 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 1 Aug 2026 20:22:28 -0700 Subject: [PATCH 03/14] Pin the peel's behavior across a comma and a dot (#312) --- tests/v2/cases.py | 70 ++++++++++++++++++++++------- tools/differential/corpus_cjk.jsonl | 5 +++ 2 files changed, 58 insertions(+), 17 deletions(-) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index e3950ae..75c5a93 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -967,23 +967,59 @@ def __post_init__(self) -> None: "pins that choice: under NO_COMMA the two are otherwise " "the same run"), Case("ja_honorific_glued_family_comma", "田中さん, PhD", - {"family": "田中さん", "title": "PhD"}, - classification="fix(comma-family)", - notes="the comma doctrine outranks the peel, and this is " - "where a reader meets that boundary: a one-word name " - "part before a comma is FAMILY_COMMA, which opts the " - "whole stage out, so 田中さん stays whole where the " - "comma-less 田中さん PhD gives family 田中 and suffix " - "'さん, PhD'. Deliberate -- whoever wrote the comma " - "already said where the family name ends -- and the " - "one spelling disagreement #308 does not remove, unlike " - "the 김민준씨 Jr. pair above. Classified to " - "comma-family, not #308 or #271: the peel never fires " - "on this input, and the only deviation from 1.4.0 is " - "first -> family, which the pure-Latin 'Smith, PhD' " - "makes identically (1.4.0 first Smith, here family " - "Smith) -- so no script-conditional rule can be " - "reaching it. Same move as family_comma_lone_title"), + {"title": "PhD", "family": "田中", "suffix": "さん"}, + classification="fix(#312)", + notes="the peel crosses the comma: PhD is a post-nominal, so " + "the site scan steps over it and reaches 田中さん. " + "Agrees with the spaced 田中さん PhD, which peels for " + "the same reason. Where PhD itself lands still differs " + "between the two spellings -- suffix spaced, title " + "post-comma -- and that is fix(comma-family)'s, not " + "the peel's"), + Case("ko_honorific_glued_given_after_family_comma", "김, 민준씨", + {"family": "김", "given": "민준", "suffix": "씨"}, + classification="fix(#312)", + notes="under a family comma the name spans both segments and " + "the honorific is on the GIVEN side, where the peel " + "never looked before #312. Agrees with the spaced " + "김 민준씨"), + Case("ja_honorific_glued_given_after_family_comma", "田中, 太郎さん", + {"family": "田中", "given": "太郎", "suffix": "さん"}, + classification="fix(#312)", + notes="the Han twin of the row above"), + Case("zh_interpunct_transcription_glued_honorific", "威廉·莎士比亚さん", + {"given": "威廉", "family": "莎士比亚", "suffix": "さん"}, + classification="fix(#312)", + notes="the dot still gates the surname split -- a " + "transcription's pieces are syllable groups -- but an " + "honorific glued to a transcribed name is still an " + "honorific, so the peel crosses it. Agrees with the " + "spaced 威廉·莎士比亚 さん"), + Case("ja_honorific_glued_family_comma_no_site", "田中さん, 太郎", + {"family": "田中さん", "given": "太郎"}, + notes="unchanged, and pinned because a naive fix breaks it: " + "the site is the last NON-POST-NOMINAL token, which is " + "太郎, so nothing peels -- exactly as in the spaced " + "田中さん 太郎. #312 was originally filed naming this " + "pair as a disagreement; it never was one"), + Case("ko_honorific_glued_given_suffix_comma_initial", "Dr 김민준씨, V.", + {"title": "Dr", "family": "김", "given": "민준", + "suffix": "씨, V."}, + classification="fix(#308)", + notes="fix(#308) rather than fix(#312) because the fields do " + "not move in this change -- a SUFFIX comma keeps the " + "whole name in segments[0], which is what the peel " + "already scanned. The row is here as the end-to-end " + "guard for the scoping #312 introduced: widen the scan " + "to every segment and 'V.' becomes the site, since " + "segment admits a post-comma run on is_suffix_lenient " + "while the site scan asks is_suffix_strict and an " + "initial fails it. 'V.' then ends in no tail, the peel " + "is abandoned, and 씨 is back in the given name -- the " + "original bug, and no other row in this table notices. " + "Its comma-less twin ja_honorific_glued_before_an_initial " + "shows the same veto from the other side, where 'V.' is " + "in the name's own run and so IS the site"), Case("zh_honorific_glued_surname", "王先生", {"family": "王", "suffix": "先生"}, locale="zh", diff --git a/tools/differential/corpus_cjk.jsonl b/tools/differential/corpus_cjk.jsonl index 3306642..6c9cdde 100644 --- a/tools/differential/corpus_cjk.jsonl +++ b/tools/differential/corpus_cjk.jsonl @@ -2,6 +2,7 @@ "Anderson선생님" "Dr 김민준, Jr." "Dr 김민준씨, Jr." +"Dr 김민준씨, V." "John 王" "〆木 ひろ" "〆木 太郎" @@ -17,6 +18,7 @@ "夏侯惇" "威廉·莎士比亚" "威廉·莎士比亚, PhD" +"威廉·莎士比亚さん" "威廉・莎士比亚" "山田 エミ" "山田 太郎 (マイケル・ジャクソン)" @@ -35,11 +37,13 @@ "田中 さん" "田中 太郎 様" "田中 殿" +"田中, 太郎さん" "田中『ハナ』花子" "田中さん" "田中さん II" "田中さん V." "田中さん, PhD" +"田中さん, 太郎" "田中博士" "諸葛亮" "阿明" @@ -50,6 +54,7 @@ "高橋一郎" "鵜殿" "김 민준" +"김, 민준씨" "김민준" "김민준 님" "김민준 박사" From 448f82ae674678e15f71e299abb3038608438fc4 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 1 Aug 2026 20:27:00 -0700 Subject: [PATCH 04/14] Record where the #312 diffs classify (#312) --- tools/differential/expected_changes.toml | 38 ++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tools/differential/expected_changes.toml b/tools/differential/expected_changes.toml index 01bcfc7..46fae1d 100644 --- a/tools/differential/expected_changes.toml +++ b/tools/differential/expected_changes.toml @@ -72,6 +72,33 @@ issue = "fix(comma-family) lone post-comma piece routes to suffix/title, not fir # post-comma piece in `first`; 2.0 routes it to `suffix`/`title` instead # (family/`last` is unchanged either way -- "pre-comma is definitionally # family"). +# +# #312 (the glued-honorific peel site move) sent two of its five new +# corpus rows here instead of the CJK-comma rule below, and it is worth +# recording why, since this rule's prose has nothing to do with CJK. +# '김, 민준씨' and '田中, 太郎さん' diff as {first, suffix}: the glued +# honorific peels off the post-comma given name into `suffix` +# (민준씨->민준/씨, 太郎さん->太郎/さん), and that field pair happens to +# be a subset of this rule's [first, title, suffix]. No title moved and +# the peeled token is CJK, not a Latin credential -- this rule doesn't +# know or care, because compare.py's classify() matches purely on +# regex-then-field-superset, first rule to satisfy both wins, and this +# rule sits earlier in file order (name_regex tier, written-order tie- +# break) than fix(cjk-comma-compound) below. The other two comma rows, +# 'Dr 김민준씨, V.' and '田中さん, PhD', diff with `last` in the mix +# (v1 read the whole pre-comma run as `first`; 2.0 splits family into +# `last`), which is outside this rule's fields, so they correctly fall +# through to fix(cjk-comma-compound). The fifth row, '威廉·莎士比亚さん', +# has no comma at all -- its dot is U+00B7 (间隔号), not a comma -- so +# it skips every comma-keyed rule and the honorific-suffix rule too (no +# space precedes its glued さん), landing on the fields-only +# fix(suffix-routing) tier by its plain {first, last, suffix} shape, +# same as any glued Latin honorific. Verified by the 2026-08-01 run, +# not assumed: traced through compare.py's classify(), which tries +# name_regex rules before fields-only ones and, within a tier, file +# order, returning the first rule whose regex (if any) matches the +# string and whose fields are a superset of the diff. No rule here +# inspects the honorific token itself -- the field shape alone decides. name_regex = "," fields = ["first", "title", "suffix"] @@ -80,6 +107,11 @@ issue = "fix(suffix-routing) two-token name with unambiguous trailing suffix sta # 'Johnson PhD' / 'Mr. Johnson PhD': v1 routed a lone trailing suffix # to family/first (no comma present); 2.0 keeps recognized suffixes in # `suffix`. +# #312: also claims comma-less glued-honorific names like +# '威廉·莎士比亚さん' by the same {first, last, suffix} shape as any +# glued Latin honorific -- see the note on fix(comma-family) above for +# why its four comma-bearing siblings split between that rule and +# fix(cjk-comma-compound) instead. fields = ["first", "last", "suffix"] [[change]] @@ -174,6 +206,12 @@ issue = "fix(cjk-comma-compound) comma routing compounds with the CJK order flip # The class is the same hand copy of _SCRIPT_RANGES the rules above # carry, pinned by the same sync test. The slug avoids the literal # #271/#272 substrings the canonical-rule pin selects by. +# +# #312 sends only the two glued-honorific comma rows whose diff +# includes `last` here ('Dr 김민준씨, V.', '田中さん, PhD'); the two +# whose diff is only {first, suffix} ('김, 민준씨', '田中, 太郎さん') +# match fix(comma-family) first instead, purely on file order -- see +# the note there for why. name_regex = "(?s)(?=.*,)(?=.*[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65])" fields = ["first", "middle", "last", "title", "suffix"] From 6d3f8425002b65a2f4db356ddb732f7093248387 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 1 Aug 2026 20:31:48 -0700 Subject: [PATCH 05/14] Document the peel crossing a comma and a dot (#312) --- AGENTS.md | 2 +- docs/release_log.rst | 1 + docs/usage.rst | 28 ++++++++++++++++++++-------- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c88cd40..4185af5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,7 +75,7 @@ logging.getLogger('HumanName').setLevel(logging.DEBUG) The library has two layers: `nameparser/config/` (data) and `nameparser/parser.py` (logic). -**Design philosophy — positional and language-agnostic.** The parser assigns parts by *position* plus small sets of words that join to neighbors; it never detects language. A name's language can't be reliably inferred from Latin-script transliteration ("Ali" is Arabic or Italian; "Van"/"Della"/"Bin" are first names in some cultures, particles in others), so language-specific rules belong in opt-in `Constants` config, never global defaults. Many "wrong for language X" reports (#133, #150, #130, #85, #103, #146, #83) are irreducible ambiguities — e.g. `de Mesnil` (want last name) vs `Van Johnson` (want first name) are the same `[prefix][word]` shape. Before adding a rule, confirm it doesn't break the opposite case (run the full suite — Portuguese and "Van Johnson" tests are the usual canaries). **The one scoped exception (2.1, #271/#272): script-conditional behavior is permitted exactly where the SCRIPT ITSELF — not statistics about it — determines the convention.** The never-detect-language rule above is about Latin *transliteration*, where the signal genuinely is destroyed; native script is a different question, and it is answered per behavior rather than per script. Five defaults fall out of it, plus a sixth that applies the same not-a-guess standard to specific WORDS rather than to a script (#308's honorific peel, below). Wholly-Han, wholly-Hangul and kana-licensed names read family-first (`Policy.script_orders`) — no language detection needed, because zh and ja both write family-first in native script, so order cannot be misread even though the language is unknowable. Unspaced hangul splits into surname + given name (`config/surnames.py` ships the Korean census list as DEFAULT vocabulary) — nothing but Korean is written in hangul and the surnames are a closed census set, and the vocabulary is self-selecting besides: a hangul entry can only ever match hangul text. Hiragana licenses Japanese (#272) — a name whose characters stay inside Han∪kana while carrying at least one kana cannot be Chinese (the kana rules it out) and is not a transcription (foreign names are transcribed in katakana ALONE, マイケル has no kanji), so 高橋みなみ and 山田 エミ read family-first too; mechanically they resolve to the HIRAGANA entry, the license's carrier key. PURE katakana is excluded and keeps the positional default: マイケル・ジャクソン is a transcribed foreign name in its source order. And the 间隔号 U+00B7 (#298) is the transcription marker for scripts that HAVE no transcription script: a name it divides (威廉·莎士比亚 — flanked by classified characters on both sides, so Catalan's Gal·la is untouched) keeps its source order and never segments — the orthography names the convention, exactly as pure katakana does, with the divider carrying the signal instead of the script. And a listed CJK honorific glued to the END of a name token is split off it (#308) — 田中さん is 田中 plus さん — on the same orthography-settles-it test, narrowed for the glued position: an entry peels only where it can never end a name, so 씨/님/さん/様/先生 peel while 양/군/氏/博士/殿 stay spaced-only (김지양 and 田中博士 are names, and ~90 Japanese surnames end in 殿) and 君 is in NEITHER set (王君 is a complete Chinese name), though its kana spelling くん peels. Like the nakaguro's tokenize-level separation described next, it is reached by neither policy opt-out — but for its own reason: the vocabulary carries the license itself rather than borrowing the script's, so `segment_scripts` has nothing to say about it. The nakaguro belongs to the same doctrine but is decided a layer down: U+30FB and its halfwidth twin U+FF65 separate tokens like whitespace, unconditionally and in tokenize, so neither policy opt-out (`script_orders={}`, `segment_scripts=()`) reaches it — the codepoints are CJK-only and appear in no other script's names, which is what licenses a tokenize-level rule where U+00B7 (also the Catalan punt volat, interior to Gal·la) needs the flanked-by-classified-script guard `_tokenize_region` gives it (#298). Han segmentation stays OPT-IN (`locales.ZH` for Chinese, `locales.JA` for Japanese) — a zh surname list corrupts Japanese kanji names, since 高 is a common Chinese surname and 高橋一郎 would split 高+橋一郎 where the correct reading is 高橋+一郎; no surname list divides a kanji name at all, so `locales.JA` activates the stage and a pluggable `Parser(segmenter=...)` does the dividing. Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely order-ambiguous and stays governed by `name_order` and opt-in packs. Before adding a script-conditional rule, work out which of the three it is — certain, certain for this one behavior only, or a statistical guess wearing a script's clothes. +**Design philosophy — positional and language-agnostic.** The parser assigns parts by *position* plus small sets of words that join to neighbors; it never detects language. A name's language can't be reliably inferred from Latin-script transliteration ("Ali" is Arabic or Italian; "Van"/"Della"/"Bin" are first names in some cultures, particles in others), so language-specific rules belong in opt-in `Constants` config, never global defaults. Many "wrong for language X" reports (#133, #150, #130, #85, #103, #146, #83) are irreducible ambiguities — e.g. `de Mesnil` (want last name) vs `Van Johnson` (want first name) are the same `[prefix][word]` shape. Before adding a rule, confirm it doesn't break the opposite case (run the full suite — Portuguese and "Van Johnson" tests are the usual canaries). **The one scoped exception (2.1, #271/#272): script-conditional behavior is permitted exactly where the SCRIPT ITSELF — not statistics about it — determines the convention.** The never-detect-language rule above is about Latin *transliteration*, where the signal genuinely is destroyed; native script is a different question, and it is answered per behavior rather than per script. Five defaults fall out of it, plus a sixth that applies the same not-a-guess standard to specific WORDS rather than to a script (#308's honorific peel, below). Wholly-Han, wholly-Hangul and kana-licensed names read family-first (`Policy.script_orders`) — no language detection needed, because zh and ja both write family-first in native script, so order cannot be misread even though the language is unknowable. Unspaced hangul splits into surname + given name (`config/surnames.py` ships the Korean census list as DEFAULT vocabulary) — nothing but Korean is written in hangul and the surnames are a closed census set, and the vocabulary is self-selecting besides: a hangul entry can only ever match hangul text. Hiragana licenses Japanese (#272) — a name whose characters stay inside Han∪kana while carrying at least one kana cannot be Chinese (the kana rules it out) and is not a transcription (foreign names are transcribed in katakana ALONE, マイケル has no kanji), so 高橋みなみ and 山田 エミ read family-first too; mechanically they resolve to the HIRAGANA entry, the license's carrier key. PURE katakana is excluded and keeps the positional default: マイケル・ジャクソン is a transcribed foreign name in its source order. And the 间隔号 U+00B7 (#298) is the transcription marker for scripts that HAVE no transcription script: a name it divides (威廉·莎士比亚 — flanked by classified characters on both sides, so Catalan's Gal·la is untouched) keeps its source order and never segments — the orthography names the convention, exactly as pure katakana does, with the divider carrying the signal instead of the script. And a listed CJK honorific glued to the END of a name token is split off it (#308) — 田中さん is 田中 plus さん — on the same orthography-settles-it test, narrowed for the glued position: an entry peels only where it can never end a name, so 씨/님/さん/様/先生 peel while 양/군/氏/博士/殿 stay spaced-only (김지양 and 田中博士 are names, and ~90 Japanese surnames end in 殿) and 君 is in NEITHER set (王君 is a complete Chinese name), though its kana spelling くん peels. Like the nakaguro's tokenize-level separation described next, it is reached by neither policy opt-out — but for its own reason: the vocabulary carries the license itself rather than borrowing the script's, so `segment_scripts` has nothing to say about it. Since #312 it also crosses the FAMILY comma and the 间隔号, which is the more surprising fact, since both of those still stop the surname split standing right beside it: each answers where a name DIVIDES into surname and given, and the peel never asks that question — so `김, 민준씨` reads exactly as the spaced `김 민준씨` does (family 김, given 민준, suffix 씨) while the split stands down as before. Its site is accordingly the name-bearing segment runs (`segments[:2]` under a family comma, `segments[:1]` otherwise) rather than `segments[0]`, since an honorific is as often glued to the given name as to the family; `田中さん, 太郎` is unchanged, and not because of its comma — the honorific there is not at the end of the name, 太郎 is. The nakaguro belongs to the same doctrine but is decided a layer down: U+30FB and its halfwidth twin U+FF65 separate tokens like whitespace, unconditionally and in tokenize, so neither policy opt-out (`script_orders={}`, `segment_scripts=()`) reaches it — the codepoints are CJK-only and appear in no other script's names, which is what licenses a tokenize-level rule where U+00B7 (also the Catalan punt volat, interior to Gal·la) needs the flanked-by-classified-script guard `_tokenize_region` gives it (#298). Han segmentation stays OPT-IN (`locales.ZH` for Chinese, `locales.JA` for Japanese) — a zh surname list corrupts Japanese kanji names, since 高 is a common Chinese surname and 高橋一郎 would split 高+橋一郎 where the correct reading is 高橋+一郎; no surname list divides a kanji name at all, so `locales.JA` activates the stage and a pluggable `Parser(segmenter=...)` does the dividing. Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely order-ambiguous and stays governed by `name_order` and opt-in packs. Before adding a script-conditional rule, work out which of the three it is — certain, certain for this one behavior only, or a statistical guess wearing a script's clothes. ### Configuration layer (`nameparser/config/`) diff --git a/docs/release_log.rst b/docs/release_log.rst index d997c5f..8a334d4 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -28,6 +28,7 @@ Release Log - Fix 间隔号-divided transcriptions parsing as one unsplit token: U+00B7 — the interpunct Chinese text divides a transcribed foreign name with, ``威廉·莎士比亚`` for William Shakespeare — is now a token separator between characters of a classified script, and a name it divides keeps its source order and is never segmented: the dot is the transcription marker, playing the role pure katakana plays in the kana license. It divides ONLY between classified-script characters, so the Catalan punt volat interior to names like ``Gal·la`` is untouched. The Japanese nakaguro is deliberately not a transcription marker — ``高橋・一郎`` is roster formatting, 姓・名, and keeps its family-first reading — so a transcription typed with the wrong dot (``威廉・莎士比亚``) reads by the convention of the codepoint it was typed with; like the spaced form, only the Chinese dot rescues it. Rendering stays space-joined (``str(HumanName("威廉·莎士比亚"))`` is ``"威廉 莎士比亚"``); a custom ``string_format`` such as ``"{first}·{last}"`` reinstates the dot (#298) - Fix spaced CJK postnominal honorifics parsing as name parts: 씨, 박사, 선생님, 교수님, 군, 양 (Korean — standardly written as their own token), 先生, 女士, 小姐, 博士, 教授 (Chinese, with 先生/博士/教授 shared with Japanese), and 様, 氏 (Japanese) now route to ``suffix``, so ``王小明 先生`` reads family ``王小明`` where the family-first default had confidently made 先生 the given name. Whole-token matching, which also reaches a glued surname+honorific token, since segmentation splits off the surname first (``김씨`` reads family 김, suffix 씨). The glued forms whole-token matching cannot reach are handled by the peel described below (closes #307) - Fix glued CJK honorifics parsing as part of the name: an honorific written against the name — ``田中さん``, ``山田太郎様``, ``김민준씨``, ``김민준님``, ``王小明先生`` — is now split off the end of the last name token and routed to ``suffix``, where it had been swallowed by the name (the whole of ``田中さん`` was the family name; ``김민준씨`` gave given 민준씨). The peeled name then goes through the ordinary machinery, so the Korean split still happens (``김민준씨`` → family 김, given 민준, suffix 씨) and the ``田中さん`` case stops misreading as a kana-licensed composite. Only entries that can never END a name peel — 씨, 님, 박사, 박사님, 선생님, 교수님, さん, さま, くん, ちゃん, 様, 先生, 教授, 女士, 小姐 — while 양, 군, 氏, 博士 and 殿 are recognized in their spaced form only, because 김지양 is a given name, 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka, and some ninety Japanese surnames end in 殿 (鵜殿, 真殿); 君 is recognized in neither form, since 王君 is a complete Chinese name, though its kana spelling くん peels. Seven entries are new vocabulary in this change (さん, さま, くん, ちゃん, 殿, 님, 박사님), so their spaced forms route to ``suffix`` too (``田中 さん``, ``田中 殿``, ``김민준 님``, ``김민준 박사님``). 박사님 closes a gap in the shipped set rather than opening new ground: 선생님 and 교수님 shipped in 2.1 without it, so ``김민준박사님`` stranded 박사 in the given name and the spaced ``김민준 박사님`` came back as two suffixes for one honorific. Exactly one honorific peels off a token, and every entry is a whole honorific rather than a part of one. A token that is nothing but an honorific is no longer taken apart either — ``선생님`` and ``박사`` now stay whole where hangul segmentation had split them 선 + 생님 and 박 + 사, since 선 and 박 are listed surnames — which is **default-on** in its own right, hangul segmentation being a default. A configured segmenter benefits twice over: it is handed the name without the honorific, and the honorific the peel just cut off does not then look to it like a boundary the writer drew — so ``parser_for(locales.JA, segmenter=ja_segmenter())`` reads ``山田太郎様`` as family 山田, given 太郎, suffix 様. Worth knowing before you upgrade: that exemption is what makes a GLUED honorific stop protecting a name from division, so a family name written alone with one — ``田中さん`` — now divides the way bare ``田中`` already did (family 田, given 中, suffix さん). It is exactly and only the peeled tail that is exempt. A SPACED honorific is a token boundary its writer typed, and anything standing beside a name calls the segmenter off, so the name is left as written: ``田中 さん`` and ``佐藤 氏`` keep family 田中 and 佐藤 under the pack — the division the pack gives them without this change, whichever field the honorific itself lands in. That is the conservative reading rather than a claim about intent: a spaced honorific cannot be told apart from a spaced given name by position, and counting it as one keeps four real surnames whole (``佐藤 氏``, ``田中 様``, ``鈴木 先生``, ``中村 教授``) at the price of the one division it then declines to make (``山田太郎 様``). Writing the honorific spaced is therefore an opt-out in its own right on the SEGMENTER path, alongside declining the pack or the segmenter. It is no lever where the VOCABULARY divides the name, the two spellings agreeing exactly there — ``김민준 씨`` and ``김민준씨`` both give family 김, given 민준, suffix 씨, as do ``王小明 先生`` and ``王小明先生`` under the Chinese pack — and Korean data has no pack to decline either, hangul segmentation being on by default. **Default-on: changes parse output for glued CJK honorific forms**, through ``HumanName`` as well as the 2.0 API (closes #308) + - Fix a comma or a 间隔号 stopping the glued-honorific peel: an honorific written against the name is now split off and routed to ``suffix`` whichever way the name is punctuated, so ``김, 민준씨`` gives family 김, given 민준, suffix 씨 — the same as the spaced ``김 민준씨`` — and likewise ``田中, 太郎さん``, ``田中さん, PhD`` and ``威廉·莎士比亚さん``. Previously each of those left the honorific inside the name. The peel now consults no comma structure and no dot: both of those say where a name divides into surname and given, and an honorific is not part of the name in either spelling. The surname split still stands down for both, unchanged — a comma still means the writer said where the family name ends, and the 间隔号 still marks a transcription. ``田中さん, 太郎`` is unaffected, because the honorific there is not at the end of the name: 太郎 is. **Default-on**, and it reaches ``HumanName`` too (closes #312) - Fix NFD-decomposed input missing the East Asian defaults entirely: script classification now normalizes to NFC before deciding, so a Korean or Japanese name typed on macOS — where decomposed text is routine — gets the same order rule as its composed twin, which it silently did not before. Segmentation MATCHING deliberately stays raw, so an unspaced NFD hangul name is ordered correctly but not split, rather than being split in the wrong place. One gotcha worth stating plainly: parse output preserves the encoding it was given, so for NFD input ``name.family == "김"`` is ``False`` even though it is the same name — compare NFC-normalized text when comparing across encodings (#272) - Fix the Ukrainian conjunction ``й`` not joining the pieces around it: it is the euphonic alternate of ``і``, the two chosen by the surrounding vowel and consonant rather than by meaning (``"Олесь і Олена"`` but ``"Марія й Петро"``), so real Ukrainian data carries both spellings and shipping only ``і`` recognized just one of them. ``"Олесь й Олена Коваленки"`` now gives given ``"Олесь й Олена"`` where the ``й`` previously landed in ``middle``. Same treatment as the ``и``/``і`` entries added in 2.0.0, single-letter carve-out included: the conjunction joins only once the name has enough pieces, and a punctuated initial still wins, so ``"Й. Сліпий"`` is unaffected. Raised in a comment on #267 diff --git a/docs/usage.rst b/docs/usage.rst index 3c16b05..62b90f0 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -294,19 +294,31 @@ Japanese roster formatting rather than transcription, so only the Chinese dot rescues the source order. And a comma disables the script behaviors entirely, on the reasoning ``name_order`` already follows: whoever wrote the comma has already said where the family name ends. +The honorific peel is the exception, and for the reason the rest of +the sentence gives: a comma says where the family name *ends*, which +is a claim about where the name divides — and an honorific is not part +of the name under either spelling. So ``김, 민준씨`` reads the same as +``김 민준씨``, honorific in ``suffix`` either way, while the surname +split still stands down. The 间隔号 works the same way: it marks a +transcription, so it still stops the split, but an honorific glued to +a transcribed name is still an honorific. Honorifics and degrees follow a CJK name, and both the spaced and the glued forms are recognized as suffixes: ``王小明 先生`` reads family 王小明 with 先生 in ``suffix``, and so do glued ``田中さん``, ``山田太郎様`` and ``김민준씨``. The honorific is split off the end of -the last name token before the name is split or ordered, so those -rules see the name without it — which is why ``김민준씨`` still divides -into family 김 and given 민준, and why a configured Japanese segmenter -is handed 山田太郎 rather than 山田太郎様. The comma rule above comes -first, though, and covers this too: in ``田中さん, 太郎`` the writer -has already said where the family name ends, so nothing is peeled and -the family name is the whole ``田中さん``. A 间隔号-divided -transcription opts out for its own reason, the same way. +the last token of the name before the name is split or ordered, so +those rules see the name without it — which is why ``김민준씨`` still +divides into family 김 and given 민준, and why a configured Japanese +segmenter is handed 山田太郎 rather than 山田太郎様. Neither a comma nor +a 间隔号 stops the peel. Both say where a name divides — the comma that +the writer has already given the family name, the dot that the pieces +are a transcription's syllable groups — and an honorific is not part +of the name in any of those readings. So ``김, 민준씨`` reads the same +as ``김 민준씨``. What those marks do stop is the *split*, which is a +different question and still theirs to answer. ``田中さん, 太郎`` is +unchanged, and not because of its comma: the honorific there is not at +the end of the name, 太郎 is. Where a segmenter divides the name, the two spellings part company. A spaced honorific is a token boundary the writer typed, and the From 7911f1d9bc5b7fee5b6c78c0d1bee17b9c62eaf2 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 1 Aug 2026 20:32:53 -0700 Subject: [PATCH 06/14] Say the peel is not GATED by comma structure, rather than blind to it (#312) --- docs/release_log.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index 8a334d4..f8ad300 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -28,7 +28,7 @@ Release Log - Fix 间隔号-divided transcriptions parsing as one unsplit token: U+00B7 — the interpunct Chinese text divides a transcribed foreign name with, ``威廉·莎士比亚`` for William Shakespeare — is now a token separator between characters of a classified script, and a name it divides keeps its source order and is never segmented: the dot is the transcription marker, playing the role pure katakana plays in the kana license. It divides ONLY between classified-script characters, so the Catalan punt volat interior to names like ``Gal·la`` is untouched. The Japanese nakaguro is deliberately not a transcription marker — ``高橋・一郎`` is roster formatting, 姓・名, and keeps its family-first reading — so a transcription typed with the wrong dot (``威廉・莎士比亚``) reads by the convention of the codepoint it was typed with; like the spaced form, only the Chinese dot rescues it. Rendering stays space-joined (``str(HumanName("威廉·莎士比亚"))`` is ``"威廉 莎士比亚"``); a custom ``string_format`` such as ``"{first}·{last}"`` reinstates the dot (#298) - Fix spaced CJK postnominal honorifics parsing as name parts: 씨, 박사, 선생님, 교수님, 군, 양 (Korean — standardly written as their own token), 先生, 女士, 小姐, 博士, 教授 (Chinese, with 先生/博士/教授 shared with Japanese), and 様, 氏 (Japanese) now route to ``suffix``, so ``王小明 先生`` reads family ``王小明`` where the family-first default had confidently made 先生 the given name. Whole-token matching, which also reaches a glued surname+honorific token, since segmentation splits off the surname first (``김씨`` reads family 김, suffix 씨). The glued forms whole-token matching cannot reach are handled by the peel described below (closes #307) - Fix glued CJK honorifics parsing as part of the name: an honorific written against the name — ``田中さん``, ``山田太郎様``, ``김민준씨``, ``김민준님``, ``王小明先生`` — is now split off the end of the last name token and routed to ``suffix``, where it had been swallowed by the name (the whole of ``田中さん`` was the family name; ``김민준씨`` gave given 민준씨). The peeled name then goes through the ordinary machinery, so the Korean split still happens (``김민준씨`` → family 김, given 민준, suffix 씨) and the ``田中さん`` case stops misreading as a kana-licensed composite. Only entries that can never END a name peel — 씨, 님, 박사, 박사님, 선생님, 교수님, さん, さま, くん, ちゃん, 様, 先生, 教授, 女士, 小姐 — while 양, 군, 氏, 博士 and 殿 are recognized in their spaced form only, because 김지양 is a given name, 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka, and some ninety Japanese surnames end in 殿 (鵜殿, 真殿); 君 is recognized in neither form, since 王君 is a complete Chinese name, though its kana spelling くん peels. Seven entries are new vocabulary in this change (さん, さま, くん, ちゃん, 殿, 님, 박사님), so their spaced forms route to ``suffix`` too (``田中 さん``, ``田中 殿``, ``김민준 님``, ``김민준 박사님``). 박사님 closes a gap in the shipped set rather than opening new ground: 선생님 and 교수님 shipped in 2.1 without it, so ``김민준박사님`` stranded 박사 in the given name and the spaced ``김민준 박사님`` came back as two suffixes for one honorific. Exactly one honorific peels off a token, and every entry is a whole honorific rather than a part of one. A token that is nothing but an honorific is no longer taken apart either — ``선생님`` and ``박사`` now stay whole where hangul segmentation had split them 선 + 생님 and 박 + 사, since 선 and 박 are listed surnames — which is **default-on** in its own right, hangul segmentation being a default. A configured segmenter benefits twice over: it is handed the name without the honorific, and the honorific the peel just cut off does not then look to it like a boundary the writer drew — so ``parser_for(locales.JA, segmenter=ja_segmenter())`` reads ``山田太郎様`` as family 山田, given 太郎, suffix 様. Worth knowing before you upgrade: that exemption is what makes a GLUED honorific stop protecting a name from division, so a family name written alone with one — ``田中さん`` — now divides the way bare ``田中`` already did (family 田, given 中, suffix さん). It is exactly and only the peeled tail that is exempt. A SPACED honorific is a token boundary its writer typed, and anything standing beside a name calls the segmenter off, so the name is left as written: ``田中 さん`` and ``佐藤 氏`` keep family 田中 and 佐藤 under the pack — the division the pack gives them without this change, whichever field the honorific itself lands in. That is the conservative reading rather than a claim about intent: a spaced honorific cannot be told apart from a spaced given name by position, and counting it as one keeps four real surnames whole (``佐藤 氏``, ``田中 様``, ``鈴木 先生``, ``中村 教授``) at the price of the one division it then declines to make (``山田太郎 様``). Writing the honorific spaced is therefore an opt-out in its own right on the SEGMENTER path, alongside declining the pack or the segmenter. It is no lever where the VOCABULARY divides the name, the two spellings agreeing exactly there — ``김민준 씨`` and ``김민준씨`` both give family 김, given 민준, suffix 씨, as do ``王小明 先生`` and ``王小明先生`` under the Chinese pack — and Korean data has no pack to decline either, hangul segmentation being on by default. **Default-on: changes parse output for glued CJK honorific forms**, through ``HumanName`` as well as the 2.0 API (closes #308) - - Fix a comma or a 间隔号 stopping the glued-honorific peel: an honorific written against the name is now split off and routed to ``suffix`` whichever way the name is punctuated, so ``김, 민준씨`` gives family 김, given 민준, suffix 씨 — the same as the spaced ``김 민준씨`` — and likewise ``田中, 太郎さん``, ``田中さん, PhD`` and ``威廉·莎士比亚さん``. Previously each of those left the honorific inside the name. The peel now consults no comma structure and no dot: both of those say where a name divides into surname and given, and an honorific is not part of the name in either spelling. The surname split still stands down for both, unchanged — a comma still means the writer said where the family name ends, and the 间隔号 still marks a transcription. ``田中さん, 太郎`` is unaffected, because the honorific there is not at the end of the name: 太郎 is. **Default-on**, and it reaches ``HumanName`` too (closes #312) + - Fix a comma or a 间隔号 stopping the glued-honorific peel: an honorific written against the name is now split off and routed to ``suffix`` whichever way the name is punctuated, so ``김, 민준씨`` gives family 김, given 민준, suffix 씨 — the same as the spaced ``김 민준씨`` — and likewise ``田中, 太郎さん``, ``田中さん, PhD`` and ``威廉·莎士比亚さん``. Previously each of those left the honorific inside the name. No comma structure and no dot gates the peel any more — it reads the comma structure only to find which segments the name occupies, an honorific being as often glued to the given name as to the family: both of those marks say where a name divides into surname and given, and an honorific is not part of the name in either spelling. The surname split still stands down for both, unchanged — a comma still means the writer said where the family name ends, and the 间隔号 still marks a transcription. ``田中さん, 太郎`` is unaffected, because the honorific there is not at the end of the name: 太郎 is. **Default-on**, and it reaches ``HumanName`` too (closes #312) - Fix NFD-decomposed input missing the East Asian defaults entirely: script classification now normalizes to NFC before deciding, so a Korean or Japanese name typed on macOS — where decomposed text is routine — gets the same order rule as its composed twin, which it silently did not before. Segmentation MATCHING deliberately stays raw, so an unspaced NFD hangul name is ordered correctly but not split, rather than being split in the wrong place. One gotcha worth stating plainly: parse output preserves the encoding it was given, so for NFD input ``name.family == "김"`` is ``False`` even though it is the same name — compare NFC-normalized text when comparing across encodings (#272) - Fix the Ukrainian conjunction ``й`` not joining the pieces around it: it is the euphonic alternate of ``і``, the two chosen by the surrounding vowel and consonant rather than by meaning (``"Олесь і Олена"`` but ``"Марія й Петро"``), so real Ukrainian data carries both spellings and shipping only ``і`` recognized just one of them. ``"Олесь й Олена Коваленки"`` now gives given ``"Олесь й Олена"`` where the ``й`` previously landed in ``middle``. Same treatment as the ``и``/``і`` entries added in 2.0.0, single-letter carve-out included: the conjunction joins only once the name has enough pieces, and a punctuated initial still wins, so ``"Й. Сліпий"`` is unaffected. Raised in a comment on #267 From a12b6a633426a74da6ae85bf5848da900b78ee93 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 1 Aug 2026 20:47:13 -0700 Subject: [PATCH 07/14] Scope the comma doctrine to where a name divides, and drop the duplicate passage (#312) --- AGENTS.md | 2 +- docs/release_log.rst | 2 +- docs/usage.rst | 27 +++++++++--------------- nameparser/_pipeline/_script_segment.py | 20 ++++++++++++------ tools/differential/expected_changes.toml | 12 ++++++++--- 5 files changed, 35 insertions(+), 28 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4185af5..e791068 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,7 +75,7 @@ logging.getLogger('HumanName').setLevel(logging.DEBUG) The library has two layers: `nameparser/config/` (data) and `nameparser/parser.py` (logic). -**Design philosophy — positional and language-agnostic.** The parser assigns parts by *position* plus small sets of words that join to neighbors; it never detects language. A name's language can't be reliably inferred from Latin-script transliteration ("Ali" is Arabic or Italian; "Van"/"Della"/"Bin" are first names in some cultures, particles in others), so language-specific rules belong in opt-in `Constants` config, never global defaults. Many "wrong for language X" reports (#133, #150, #130, #85, #103, #146, #83) are irreducible ambiguities — e.g. `de Mesnil` (want last name) vs `Van Johnson` (want first name) are the same `[prefix][word]` shape. Before adding a rule, confirm it doesn't break the opposite case (run the full suite — Portuguese and "Van Johnson" tests are the usual canaries). **The one scoped exception (2.1, #271/#272): script-conditional behavior is permitted exactly where the SCRIPT ITSELF — not statistics about it — determines the convention.** The never-detect-language rule above is about Latin *transliteration*, where the signal genuinely is destroyed; native script is a different question, and it is answered per behavior rather than per script. Five defaults fall out of it, plus a sixth that applies the same not-a-guess standard to specific WORDS rather than to a script (#308's honorific peel, below). Wholly-Han, wholly-Hangul and kana-licensed names read family-first (`Policy.script_orders`) — no language detection needed, because zh and ja both write family-first in native script, so order cannot be misread even though the language is unknowable. Unspaced hangul splits into surname + given name (`config/surnames.py` ships the Korean census list as DEFAULT vocabulary) — nothing but Korean is written in hangul and the surnames are a closed census set, and the vocabulary is self-selecting besides: a hangul entry can only ever match hangul text. Hiragana licenses Japanese (#272) — a name whose characters stay inside Han∪kana while carrying at least one kana cannot be Chinese (the kana rules it out) and is not a transcription (foreign names are transcribed in katakana ALONE, マイケル has no kanji), so 高橋みなみ and 山田 エミ read family-first too; mechanically they resolve to the HIRAGANA entry, the license's carrier key. PURE katakana is excluded and keeps the positional default: マイケル・ジャクソン is a transcribed foreign name in its source order. And the 间隔号 U+00B7 (#298) is the transcription marker for scripts that HAVE no transcription script: a name it divides (威廉·莎士比亚 — flanked by classified characters on both sides, so Catalan's Gal·la is untouched) keeps its source order and never segments — the orthography names the convention, exactly as pure katakana does, with the divider carrying the signal instead of the script. And a listed CJK honorific glued to the END of a name token is split off it (#308) — 田中さん is 田中 plus さん — on the same orthography-settles-it test, narrowed for the glued position: an entry peels only where it can never end a name, so 씨/님/さん/様/先生 peel while 양/군/氏/博士/殿 stay spaced-only (김지양 and 田中博士 are names, and ~90 Japanese surnames end in 殿) and 君 is in NEITHER set (王君 is a complete Chinese name), though its kana spelling くん peels. Like the nakaguro's tokenize-level separation described next, it is reached by neither policy opt-out — but for its own reason: the vocabulary carries the license itself rather than borrowing the script's, so `segment_scripts` has nothing to say about it. Since #312 it also crosses the FAMILY comma and the 间隔号, which is the more surprising fact, since both of those still stop the surname split standing right beside it: each answers where a name DIVIDES into surname and given, and the peel never asks that question — so `김, 민준씨` reads exactly as the spaced `김 민준씨` does (family 김, given 민준, suffix 씨) while the split stands down as before. Its site is accordingly the name-bearing segment runs (`segments[:2]` under a family comma, `segments[:1]` otherwise) rather than `segments[0]`, since an honorific is as often glued to the given name as to the family; `田中さん, 太郎` is unchanged, and not because of its comma — the honorific there is not at the end of the name, 太郎 is. The nakaguro belongs to the same doctrine but is decided a layer down: U+30FB and its halfwidth twin U+FF65 separate tokens like whitespace, unconditionally and in tokenize, so neither policy opt-out (`script_orders={}`, `segment_scripts=()`) reaches it — the codepoints are CJK-only and appear in no other script's names, which is what licenses a tokenize-level rule where U+00B7 (also the Catalan punt volat, interior to Gal·la) needs the flanked-by-classified-script guard `_tokenize_region` gives it (#298). Han segmentation stays OPT-IN (`locales.ZH` for Chinese, `locales.JA` for Japanese) — a zh surname list corrupts Japanese kanji names, since 高 is a common Chinese surname and 高橋一郎 would split 高+橋一郎 where the correct reading is 高橋+一郎; no surname list divides a kanji name at all, so `locales.JA` activates the stage and a pluggable `Parser(segmenter=...)` does the dividing. Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely order-ambiguous and stays governed by `name_order` and opt-in packs. Before adding a script-conditional rule, work out which of the three it is — certain, certain for this one behavior only, or a statistical guess wearing a script's clothes. +**Design philosophy — positional and language-agnostic.** The parser assigns parts by *position* plus small sets of words that join to neighbors; it never detects language. A name's language can't be reliably inferred from Latin-script transliteration ("Ali" is Arabic or Italian; "Van"/"Della"/"Bin" are first names in some cultures, particles in others), so language-specific rules belong in opt-in `Constants` config, never global defaults. Many "wrong for language X" reports (#133, #150, #130, #85, #103, #146, #83) are irreducible ambiguities — e.g. `de Mesnil` (want last name) vs `Van Johnson` (want first name) are the same `[prefix][word]` shape. Before adding a rule, confirm it doesn't break the opposite case (run the full suite — Portuguese and "Van Johnson" tests are the usual canaries). **The one scoped exception (2.1, #271/#272): script-conditional behavior is permitted exactly where the SCRIPT ITSELF — not statistics about it — determines the convention.** The never-detect-language rule above is about Latin *transliteration*, where the signal genuinely is destroyed; native script is a different question, and it is answered per behavior rather than per script. Five defaults fall out of it, plus a sixth that applies the same not-a-guess standard to specific WORDS rather than to a script (#308's honorific peel, below). Wholly-Han, wholly-Hangul and kana-licensed names read family-first (`Policy.script_orders`) — no language detection needed, because zh and ja both write family-first in native script, so order cannot be misread even though the language is unknowable. Unspaced hangul splits into surname + given name (`config/surnames.py` ships the Korean census list as DEFAULT vocabulary) — nothing but Korean is written in hangul and the surnames are a closed census set, and the vocabulary is self-selecting besides: a hangul entry can only ever match hangul text. Hiragana licenses Japanese (#272) — a name whose characters stay inside Han∪kana while carrying at least one kana cannot be Chinese (the kana rules it out) and is not a transcription (foreign names are transcribed in katakana ALONE, マイケル has no kanji), so 高橋みなみ and 山田 エミ read family-first too; mechanically they resolve to the HIRAGANA entry, the license's carrier key. PURE katakana is excluded and keeps the positional default: マイケル・ジャクソン is a transcribed foreign name in its source order. And the 间隔号 U+00B7 (#298) is the transcription marker for scripts that HAVE no transcription script: a name it divides (威廉·莎士比亚 — flanked by classified characters on both sides, so Catalan's Gal·la is untouched) keeps its source order and never segments — the orthography names the convention, exactly as pure katakana does, with the divider carrying the signal instead of the script. And a listed CJK honorific glued to the END of a name token is split off it (#308) — 田中さん is 田中 plus さん — on the same orthography-settles-it test, narrowed for the glued position: an entry peels only where it can never end a name, so 씨/님/さん/様/先生 peel while 양/군/氏/博士/殿 stay spaced-only (김지양 and 田中博士 are names, and ~90 Japanese surnames end in 殿) and 君 is in NEITHER set (王君 is a complete Chinese name), though its kana spelling くん peels. Like the nakaguro's tokenize-level separation described next, it is reached by neither policy opt-out — but for its own reason: the vocabulary carries the license itself rather than borrowing the script's, so `segment_scripts` has nothing to say about it. Since #312 it also crosses the FAMILY comma and the 间隔号, which is the more surprising fact, since both of those still stop the surname split standing right beside it: each answers where a name DIVIDES into surname and given, and the peel never asks that question — so `김, 민준씨` reads exactly as the spaced `김 민준씨` does (family 김, given 민준, suffix 씨) while the split stands down as before. Its site is accordingly the name-bearing segment runs — `segments[:2]` under a family comma, and `segments[0]` as before otherwise, the family comma being the one structure that splits the name itself across two runs, with the honorific as often glued to the given side as to the family; `田中さん, 太郎` is unchanged, and not because of its comma — the honorific there is not at the end of the name, 太郎 is. The nakaguro belongs to the same doctrine but is decided a layer down: U+30FB and its halfwidth twin U+FF65 separate tokens like whitespace, unconditionally and in tokenize, so neither policy opt-out (`script_orders={}`, `segment_scripts=()`) reaches it — the codepoints are CJK-only and appear in no other script's names, which is what licenses a tokenize-level rule where U+00B7 (also the Catalan punt volat, interior to Gal·la) needs the flanked-by-classified-script guard `_tokenize_region` gives it (#298). Han segmentation stays OPT-IN (`locales.ZH` for Chinese, `locales.JA` for Japanese) — a zh surname list corrupts Japanese kanji names, since 高 is a common Chinese surname and 高橋一郎 would split 高+橋一郎 where the correct reading is 高橋+一郎; no surname list divides a kanji name at all, so `locales.JA` activates the stage and a pluggable `Parser(segmenter=...)` does the dividing. Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely order-ambiguous and stays governed by `name_order` and opt-in packs. Before adding a script-conditional rule, work out which of the three it is — certain, certain for this one behavior only, or a statistical guess wearing a script's clothes. ### Configuration layer (`nameparser/config/`) diff --git a/docs/release_log.rst b/docs/release_log.rst index f8ad300..dd5f814 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -28,7 +28,7 @@ Release Log - Fix 间隔号-divided transcriptions parsing as one unsplit token: U+00B7 — the interpunct Chinese text divides a transcribed foreign name with, ``威廉·莎士比亚`` for William Shakespeare — is now a token separator between characters of a classified script, and a name it divides keeps its source order and is never segmented: the dot is the transcription marker, playing the role pure katakana plays in the kana license. It divides ONLY between classified-script characters, so the Catalan punt volat interior to names like ``Gal·la`` is untouched. The Japanese nakaguro is deliberately not a transcription marker — ``高橋・一郎`` is roster formatting, 姓・名, and keeps its family-first reading — so a transcription typed with the wrong dot (``威廉・莎士比亚``) reads by the convention of the codepoint it was typed with; like the spaced form, only the Chinese dot rescues it. Rendering stays space-joined (``str(HumanName("威廉·莎士比亚"))`` is ``"威廉 莎士比亚"``); a custom ``string_format`` such as ``"{first}·{last}"`` reinstates the dot (#298) - Fix spaced CJK postnominal honorifics parsing as name parts: 씨, 박사, 선생님, 교수님, 군, 양 (Korean — standardly written as their own token), 先生, 女士, 小姐, 博士, 教授 (Chinese, with 先生/博士/教授 shared with Japanese), and 様, 氏 (Japanese) now route to ``suffix``, so ``王小明 先生`` reads family ``王小明`` where the family-first default had confidently made 先生 the given name. Whole-token matching, which also reaches a glued surname+honorific token, since segmentation splits off the surname first (``김씨`` reads family 김, suffix 씨). The glued forms whole-token matching cannot reach are handled by the peel described below (closes #307) - Fix glued CJK honorifics parsing as part of the name: an honorific written against the name — ``田中さん``, ``山田太郎様``, ``김민준씨``, ``김민준님``, ``王小明先生`` — is now split off the end of the last name token and routed to ``suffix``, where it had been swallowed by the name (the whole of ``田中さん`` was the family name; ``김민준씨`` gave given 민준씨). The peeled name then goes through the ordinary machinery, so the Korean split still happens (``김민준씨`` → family 김, given 민준, suffix 씨) and the ``田中さん`` case stops misreading as a kana-licensed composite. Only entries that can never END a name peel — 씨, 님, 박사, 박사님, 선생님, 교수님, さん, さま, くん, ちゃん, 様, 先生, 教授, 女士, 小姐 — while 양, 군, 氏, 博士 and 殿 are recognized in their spaced form only, because 김지양 is a given name, 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka, and some ninety Japanese surnames end in 殿 (鵜殿, 真殿); 君 is recognized in neither form, since 王君 is a complete Chinese name, though its kana spelling くん peels. Seven entries are new vocabulary in this change (さん, さま, くん, ちゃん, 殿, 님, 박사님), so their spaced forms route to ``suffix`` too (``田中 さん``, ``田中 殿``, ``김민준 님``, ``김민준 박사님``). 박사님 closes a gap in the shipped set rather than opening new ground: 선생님 and 교수님 shipped in 2.1 without it, so ``김민준박사님`` stranded 박사 in the given name and the spaced ``김민준 박사님`` came back as two suffixes for one honorific. Exactly one honorific peels off a token, and every entry is a whole honorific rather than a part of one. A token that is nothing but an honorific is no longer taken apart either — ``선생님`` and ``박사`` now stay whole where hangul segmentation had split them 선 + 생님 and 박 + 사, since 선 and 박 are listed surnames — which is **default-on** in its own right, hangul segmentation being a default. A configured segmenter benefits twice over: it is handed the name without the honorific, and the honorific the peel just cut off does not then look to it like a boundary the writer drew — so ``parser_for(locales.JA, segmenter=ja_segmenter())`` reads ``山田太郎様`` as family 山田, given 太郎, suffix 様. Worth knowing before you upgrade: that exemption is what makes a GLUED honorific stop protecting a name from division, so a family name written alone with one — ``田中さん`` — now divides the way bare ``田中`` already did (family 田, given 中, suffix さん). It is exactly and only the peeled tail that is exempt. A SPACED honorific is a token boundary its writer typed, and anything standing beside a name calls the segmenter off, so the name is left as written: ``田中 さん`` and ``佐藤 氏`` keep family 田中 and 佐藤 under the pack — the division the pack gives them without this change, whichever field the honorific itself lands in. That is the conservative reading rather than a claim about intent: a spaced honorific cannot be told apart from a spaced given name by position, and counting it as one keeps four real surnames whole (``佐藤 氏``, ``田中 様``, ``鈴木 先生``, ``中村 教授``) at the price of the one division it then declines to make (``山田太郎 様``). Writing the honorific spaced is therefore an opt-out in its own right on the SEGMENTER path, alongside declining the pack or the segmenter. It is no lever where the VOCABULARY divides the name, the two spellings agreeing exactly there — ``김민준 씨`` and ``김민준씨`` both give family 김, given 민준, suffix 씨, as do ``王小明 先生`` and ``王小明先生`` under the Chinese pack — and Korean data has no pack to decline either, hangul segmentation being on by default. **Default-on: changes parse output for glued CJK honorific forms**, through ``HumanName`` as well as the 2.0 API (closes #308) - - Fix a comma or a 间隔号 stopping the glued-honorific peel: an honorific written against the name is now split off and routed to ``suffix`` whichever way the name is punctuated, so ``김, 민준씨`` gives family 김, given 민준, suffix 씨 — the same as the spaced ``김 민준씨`` — and likewise ``田中, 太郎さん``, ``田中さん, PhD`` and ``威廉·莎士比亚さん``. Previously each of those left the honorific inside the name. No comma structure and no dot gates the peel any more — it reads the comma structure only to find which segments the name occupies, an honorific being as often glued to the given name as to the family: both of those marks say where a name divides into surname and given, and an honorific is not part of the name in either spelling. The surname split still stands down for both, unchanged — a comma still means the writer said where the family name ends, and the 间隔号 still marks a transcription. ``田中さん, 太郎`` is unaffected, because the honorific there is not at the end of the name: 太郎 is. **Default-on**, and it reaches ``HumanName`` too (closes #312) + - Fix a comma or a 间隔号 stopping the glued-honorific peel: an honorific written against the name is now split off and routed to ``suffix``, so ``김, 민준씨`` gives family 김, given 민준, suffix 씨 — the same as the spaced ``김 민준씨`` — and likewise ``田中, 太郎さん``, ``田中さん, PhD`` and ``威廉·莎士比亚さん``. Previously each of those left the honorific inside the name. No comma structure and no dot gates the peel any more — it reads the comma structure only to find which segments the name occupies, an honorific being as often glued to the given name as to the family: both of those marks say where a name divides into surname and given, and an honorific is not part of the name in either spelling. The surname split still stands down for both, unchanged — a comma still means the writer said where the family name ends, and the 间隔号 still marks a transcription. ``田中さん, 太郎`` is unaffected, because the honorific there is not at the end of the name: 太郎 is. **Default-on**, and it reaches ``HumanName`` too (closes #312) - Fix NFD-decomposed input missing the East Asian defaults entirely: script classification now normalizes to NFC before deciding, so a Korean or Japanese name typed on macOS — where decomposed text is routine — gets the same order rule as its composed twin, which it silently did not before. Segmentation MATCHING deliberately stays raw, so an unspaced NFD hangul name is ordered correctly but not split, rather than being split in the wrong place. One gotcha worth stating plainly: parse output preserves the encoding it was given, so for NFD input ``name.family == "김"`` is ``False`` even though it is the same name — compare NFC-normalized text when comparing across encodings (#272) - Fix the Ukrainian conjunction ``й`` not joining the pieces around it: it is the euphonic alternate of ``і``, the two chosen by the surrounding vowel and consonant rather than by meaning (``"Олесь і Олена"`` but ``"Марія й Петро"``), so real Ukrainian data carries both spellings and shipping only ``і`` recognized just one of them. ``"Олесь й Олена Коваленки"`` now gives given ``"Олесь й Олена"`` where the ``й`` previously landed in ``middle``. Same treatment as the ``и``/``і`` entries added in 2.0.0, single-letter carve-out included: the conjunction joins only once the name has enough pieces, and a punctuated initial still wins, so ``"Й. Сліпий"`` is unaffected. Raised in a comment on #267 diff --git a/docs/usage.rst b/docs/usage.rst index 62b90f0..f86bf5c 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -292,16 +292,9 @@ for a transcription typed with the Japanese middle dot (威廉・莎士比亚): each dot carries its own script's convention, the nakaguro's is Japanese roster formatting rather than transcription, so only the Chinese dot rescues the source order. And a comma disables the script -behaviors entirely, on the reasoning ``name_order`` already follows: whoever -wrote the comma has already said where the family name ends. -The honorific peel is the exception, and for the reason the rest of -the sentence gives: a comma says where the family name *ends*, which -is a claim about where the name divides — and an honorific is not part -of the name under either spelling. So ``김, 민준씨`` reads the same as -``김 민준씨``, honorific in ``suffix`` either way, while the surname -split still stands down. The 间隔号 works the same way: it marks a -transcription, so it still stops the split, but an honorific glued to -a transcribed name is still an honorific. +behaviors that decide where a name divides, on the reasoning +``name_order`` already follows: whoever wrote the comma has already +said where the family name ends. Honorifics and degrees follow a CJK name, and both the spaced and the glued forms are recognized as suffixes: ``王小明 先生`` reads family @@ -310,13 +303,13 @@ glued forms are recognized as suffixes: ``王小明 先生`` reads family the last token of the name before the name is split or ordered, so those rules see the name without it — which is why ``김민준씨`` still divides into family 김 and given 민준, and why a configured Japanese -segmenter is handed 山田太郎 rather than 山田太郎様. Neither a comma nor -a 间隔号 stops the peel. Both say where a name divides — the comma that -the writer has already given the family name, the dot that the pieces -are a transcription's syllable groups — and an honorific is not part -of the name in any of those readings. So ``김, 민준씨`` reads the same -as ``김 민준씨``. What those marks do stop is the *split*, which is a -different question and still theirs to answer. ``田中さん, 太郎`` is +segmenter is handed 山田太郎 rather than 山田太郎様. A comma and a 间隔号 +both leave the peel alone. Both say where a name divides — the comma +that the writer has already given the family name, the dot that the +pieces are a transcription's syllable groups — and an honorific is not +part of the name in any of those readings. So ``김, 민준씨`` reads the +same as ``김 민준씨``. What those marks do stop is the *split*, which +is a different question and still theirs to answer. ``田中さん, 太郎`` is unchanged, and not because of its comma: the honorific there is not at the end of the name, 太郎 is. diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index ee38a91..2eb8d11 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -81,8 +81,10 @@ error, not a content error. Placed AFTER segment, on the comma doctrine that script-conditional -behavior is ignored where a comma already decides the family (the -rule script_orders follows): under FAMILY_COMMA the pre-comma text IS +behavior DECIDING WHERE A NAME DIVIDES is ignored where a comma +already decides the family (the rule script_orders follows -- and the +qualifier is load-bearing since #312, which is what the peel crosses +the comma on): under FAMILY_COMMA the pre-comma text IS the family by declaration, and splitting it would invent a boundary the writer explicitly did not draw -- "남궁민수, 지훈" must render family "남궁민수", not "남궁 민수". That doctrine is about the input's @@ -596,10 +598,16 @@ def script_segment(state: ParseState) -> ParseState: # ASCII tail never fires, and one non-ASCII character anywhere # in the name switches it on. Correct for the CJK vocabulary # that ships, stated because honorific_tails is public: see - # that field's own note. Latin does not glue post-nominals to - # names anyway, and where it does glue (Mr.Smith, Lt.Gov.) - # _vocab.period_joined_vocab classifies the token instead of - # splitting it. + # that field's own note. Latin orthography SPACES its + # post-nominals ("John Smith PhD"), so the glued position this + # peel exists for is a CJK one to begin with. What Latin does + # glue is PREnominal and period-joined (Mr.Smith, Lt.Gov.), + # which _vocab.period_joined_vocab classifies as one token + # rather than splitting -- a different mechanism, and no peel + # site either way. A glued Latin post-nominal gets neither + # treatment and simply stays in the name ("John Smith.Jr." -> + # family "Smith.Jr."), which the ASCII bail here is what makes + # moot rather than anything downstream. return state if not state.segments: return state diff --git a/tools/differential/expected_changes.toml b/tools/differential/expected_changes.toml index 46fae1d..e26c59b 100644 --- a/tools/differential/expected_changes.toml +++ b/tools/differential/expected_changes.toml @@ -73,9 +73,15 @@ issue = "fix(comma-family) lone post-comma piece routes to suffix/title, not fir # (family/`last` is unchanged either way -- "pre-comma is definitionally # family"). # -# #312 (the glued-honorific peel site move) sent two of its five new -# corpus rows here instead of the CJK-comma rule below, and it is worth -# recording why, since this rule's prose has nothing to do with CJK. +# #312 (the glued-honorific peel site move) sent two of its five +# DIFFING corpus rows here instead of the CJK-comma rule below, and it +# is worth recording why, since this rule's prose has nothing to do +# with CJK. Diffing, not new: four of the five are rows #312 added, +# while '田中さん, PhD' was already in the corpus and #312 changed only +# what it parses to. #312 added a fifth row that does NOT appear here, +# '田中さん, 太郎' -- it agrees with 1.4.0, so it never reaches +# classify() at all; it is the pinned guard for the case a naive fix +# breaks (the peel site there is 太郎, so nothing peels). # '김, 민준씨' and '田中, 太郎さん' diff as {first, suffix}: the glued # honorific peels off the post-comma given name into `suffix` # (민준씨->민준/씨, 太郎さん->太郎/さん), and that field pair happens to From 7be3c20a7edd123958412c756c174fdb26c56ae7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 1 Aug 2026 21:36:56 -0700 Subject: [PATCH 08/14] Pin how far the peel reaches under a family comma (#312) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every family-comma input in the suite has exactly two segments, so segments[:2] was only ever exercised as "more than one run", never as "and not three". Five one-line narrowings of the site scan passed the whole suite: segments[:3] '김, 민준씨, 박씨' ... and len(state.segments) == 2 '김, 민준씨, Jr.' ... and nothing was extracted '김, 민준씨 (Jimmy)' ... and not state.interpunct_offsets '威廉·莎士比亚, 太郎さん' ... and state.segments[0] ', 민준씨' Each named input now has a pin, and each mutant fails exactly the pin that names it. The first two are the hazard the site comment already argues against, unwitnessed on the side #312 opened: 씨 stranded in the given name, or the junk tail's 씨 peeled instead of the person's own. The nickname one is a case row rather than a stage test because extract_delimited runs before this stage in the pipeline and not in the stage harness; ko_honorific_glued_given_nickname pins only the NO_COMMA half, where the two runs are one and the choice is invisible. The empty-run one is separate from ", , 씨" below it, where BOTH runs are empty and the scan short-circuits before the crossing is reached. Also rename test_family_comma_is_inert: the stage is no longer inert under a family comma once the lexicon carries honorific_tails. Its assertion is sound -- it runs on _LEX, which carries none -- but the name states a general claim #312 falsified. It pins the surname split; say so. Co-Authored-By: Claude Opus 5 --- tests/v2/cases.py | 12 ++++++ tests/v2/pipeline/test_script_segment.py | 51 +++++++++++++++++++++++- tools/differential/corpus_cjk.jsonl | 1 + tools/differential/expected_changes.toml | 4 ++ 4 files changed, 66 insertions(+), 2 deletions(-) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 75c5a93..6b8bc8a 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -966,6 +966,18 @@ def __post_init__(self) -> None: "entirely, with 씨 back in the given name. Nothing else " "pins that choice: under NO_COMMA the two are otherwise " "the same run"), + Case("ko_honorific_glued_given_nickname_family_comma", + "김, 민준씨 (Jimmy)", + {"family": "김", "given": "민준", "suffix": "씨", + "nickname": "Jimmy"}, + classification="fix(#312)", + notes="the family-comma half of the same argument, which the " + "row above cannot make: there the two runs are one, so " + "'flatten the name's segments' and 'take segments[0]' " + "agree. Here the peel has to cross the comma AND still " + "not reach Jimmy, so declining to cross whenever " + "extract_delimited claimed something passes the row " + "above and fails only here"), Case("ja_honorific_glued_family_comma", "田中さん, PhD", {"title": "PhD", "family": "田中", "suffix": "さん"}, classification="fix(#312)", diff --git a/tests/v2/pipeline/test_script_segment.py b/tests/v2/pipeline/test_script_segment.py index be21a25..9bed30a 100644 --- a/tests/v2/pipeline/test_script_segment.py +++ b/tests/v2/pipeline/test_script_segment.py @@ -157,9 +157,12 @@ def test_single_possible_split_emits_no_ambiguity() -> None: assert _run("김민준", policy=_HANGUL).ambiguities == () -def test_family_comma_is_inert() -> None: +def test_a_family_comma_stands_the_surname_split_down() -> None: # the comma declared the family: splitting it would invent a - # boundary the writer did not draw ("남궁 민수" with a space) + # boundary the writer did not draw ("남궁 민수" with a space). + # The SPLIT only -- the stage as a whole is not inert under a + # family comma since #312, and would not be on a lexicon carrying + # honorific_tails (_LEX does not; see the peel's own cases below). out = _run("남궁민수, 지훈", policy=_HANGUL) assert out.structure is Structure.FAMILY_COMMA assert _texts(out) == ["남궁민수", "지훈"] @@ -591,6 +594,50 @@ def test_the_peel_scans_the_name_runs_and_no_further() -> None: "Dr", "김", "민준", "씨", "Jr.", "박씨"] +def test_the_peel_crosses_a_family_comma_and_stops_there() -> None: + # The family-comma mirror of the test above, and the only input + # that pins the SECOND half of segments[:2] -- that it is two runs + # and not three. Every other family-comma case here has exactly + # two segments, so the slice is otherwise only ever exercised as + # "more than one run": widening it to three passes them all. + # Both shapes above, re-spelled with the name split across a family + # comma. "Jr." is the strict/lenient gap again -- reached as a site + # it ends in no tail and abandons the peel, leaving 씨 in the given + # name -- and 박씨 is the junk tail, whose 씨 a wider scan peels + # instead of the person's own. + assert _texts(_run("김, 민준씨, Jr.", policy=_HANGUL, + lexicon=_LEX_TAILS)) == ["김", "민준", "씨", "Jr."] + assert _texts(_run("김, 민준씨, 박씨", policy=_HANGUL, + lexicon=_LEX_TAILS)) == ["김", "민준", "씨", "박씨"] + + +def test_an_empty_first_run_still_reaches_the_second() -> None: + # A leading comma empties segments[0] without emptying the name: + # the scan flattens both runs, so an empty first one is nothing to + # stop at. Guarding the crossing on a non-empty segments[0] passes + # the ", , 씨" test below, where BOTH runs are empty, and fails + # only here. + assert _texts(_run(", 민준씨", policy=_HANGUL, + lexicon=_LEX_TAILS)) == ["민준", "씨"] + + +def test_the_peel_crosses_a_comma_and_a_dot_at_once() -> None: + # The two gates #312 made siblings, composed: the 间隔号 gates the + # surname split (威廉 stays whole -- 威 is a listed surname here, + # so without the dot it would divide) and the family comma gates + # it too, while the peel crosses both and reaches 太郎さん on the + # far side of the comma. Neither gate alone pins this: keying the + # cross-comma reach on interpunct-free input passes every other + # case in this file. + lex = _LEX_TAILS.add(surnames={"威"}) + state = segment(tokenize(ParseState( + original="威廉·莎士比亚, 太郎さん", lexicon=lex, policy=_HAN))) + assert state.interpunct_offsets + assert state.structure is Structure.FAMILY_COMMA + assert _texts(script_segment(state)) == [ + "威廉", "莎士比亚", "太郎", "さん"] + + def test_the_peel_survives_a_name_with_no_name_tokens() -> None: # The name's runs are EMPTY -- a leading comma yields an empty one # per comma -- so the scan has nothing to look at and answers None diff --git a/tools/differential/corpus_cjk.jsonl b/tools/differential/corpus_cjk.jsonl index 6c9cdde..7746527 100644 --- a/tools/differential/corpus_cjk.jsonl +++ b/tools/differential/corpus_cjk.jsonl @@ -55,6 +55,7 @@ "鵜殿" "김 민준" "김, 민준씨" +"김, 민준씨 (Jimmy)" "김민준" "김민준 님" "김민준 박사" diff --git a/tools/differential/expected_changes.toml b/tools/differential/expected_changes.toml index e26c59b..d5887cf 100644 --- a/tools/differential/expected_changes.toml +++ b/tools/differential/expected_changes.toml @@ -105,6 +105,10 @@ issue = "fix(comma-family) lone post-comma piece routes to suffix/title, not fir # order, returning the first rule whose regex (if any) matches the # string and whose fields are a superset of the diff. No rule here # inspects the honorific token itself -- the field shape alone decides. +# The coverage pass that followed added one more row here, '김, 민준씨 +# (Jimmy)': same {first, suffix} diff as '김, 민준씨' -- the extracted +# nickname matches on both sides and so is not part of the diff -- and +# it lands on this rule for the same reason. name_regex = "," fields = ["first", "title", "suffix"] From c11932196d43f1baace18d304e5e76dad4fbf5c3 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 1 Aug 2026 21:37:45 -0700 Subject: [PATCH 09/14] Say which half of the flipped row's deviation is the peel's (#312) ja_honorific_glued_family_comma's expectation bakes in two deviations from 1.4.0 -- the peel, and the comma-family first -> family flip -- and the old note said "the only deviation is first -> family", which the flip to fix(#312) correctly removed and replaced with nothing. A reader now has to infer that the classification claims the whole diff. Name the other half and its Latin witness instead: 1.4.0 reads 'Smith, Dr.' as first Smith where family_comma_lone_title expects family Smith, so no script-conditional rule is reaching that half. Co-Authored-By: Claude Opus 5 --- tests/v2/cases.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 6b8bc8a..c47e69c 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -987,7 +987,12 @@ def __post_init__(self) -> None: "the same reason. Where PhD itself lands still differs " "between the two spellings -- suffix spaced, title " "post-comma -- and that is fix(comma-family)'s, not " - "the peel's"), + "the peel's. The expectation bakes in TWO deviations " + "from 1.4.0 and only one of them is #312's: the peel " + "is, while first -> family is comma-family's, witnessed " + "on pure Latin by family_comma_lone_title (1.4.0 first " + "Smith, here family Smith) -- so no script-conditional " + "rule is reaching that half"), Case("ko_honorific_glued_given_after_family_comma", "김, 민준씨", {"family": "김", "given": "민준", "suffix": "씨"}, classification="fix(#312)", From 754f114aeae8809963fc67950dec6e010b9edf1f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 1 Aug 2026 21:38:21 -0700 Subject: [PATCH 10/14] Record the suffixy-second-run limit under a family comma MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit segments[1] is not always name text. segment picks FAMILY_COMMA whenever the pre-comma part is a single word, including when the post-comma part is entirely suffix-shaped, and it admits that run on is_suffix_lenient where the peel's site scan asks is_suffix_strict. The initial-shaped suffix words fall in that gap, so the scan lands on a token that is no name and ends in no tail, and the peel is abandoned: 田中さん, PhD -> family 田中, suffix さん peels 田中さん, V. -> family 田中さん, given V. does not 田中さん, Ph. D. -> family 田中さん, suffix 'Ph. D.' does not Same credential, three spellings, two answers. Not a regression: both of the two that do not peel are parity with 1.4.0 (measured against 1.4.0-on-PyPI, first V. / last 田中さん), and the strict/lenient gap predates #312. Fixing it wants segment's own suffixy closure extracted into a predicate the peel can share, which is a bigger change than this branch; record the behavior with its reason instead. Co-Authored-By: Claude Opus 5 --- tests/v2/cases.py | 21 +++++++++++++++++++++ tools/differential/corpus_cjk.jsonl | 1 + tools/differential/expected_changes.toml | 4 +++- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index c47e69c..48b619f 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -993,6 +993,27 @@ def __post_init__(self) -> None: "on pure Latin by family_comma_lone_title (1.4.0 first " "Smith, here family Smith) -- so no script-conditional " "rule is reaching that half"), + Case("ja_honorific_glued_family_comma_suffixy_second_run", + "田中さん, V.", + {"family": "田中さん", "given": "V."}, + notes="a KNOWN LIMIT, recorded as it behaves rather than " + "fixed. Under a family comma the peel scans both runs " + "on the premise that segments[1] is name text, and here " + "it is not: segment picks FAMILY_COMMA when the " + "pre-comma part is a single word, even where the " + "post-comma part is entirely suffix-shaped, so the scan " + "reaches 'V.' -- which is_suffix_strict rejects as an " + "initial where segment admitted the run on " + "is_suffix_lenient. 'V.' is therefore the site, ends in " + "no tail, and the peel is abandoned with さん still in " + "the family name. Same credential, three spellings, two " + "answers: '田中さん, PhD' peels (above) while this and " + "'田中さん, Ph. D.' do not. Parity with 1.4.0 (first " + "V., last 田中さん) and unchanged by #312 -- the " + "strict/lenient gap predates it, and closing it wants " + "segment's own suffixy test extracted into a shared " + "predicate, which is why the limit is stated here " + "instead of fixed"), Case("ko_honorific_glued_given_after_family_comma", "김, 민준씨", {"family": "김", "given": "민준", "suffix": "씨"}, classification="fix(#312)", diff --git a/tools/differential/corpus_cjk.jsonl b/tools/differential/corpus_cjk.jsonl index 7746527..0d1d4d3 100644 --- a/tools/differential/corpus_cjk.jsonl +++ b/tools/differential/corpus_cjk.jsonl @@ -43,6 +43,7 @@ "田中さん II" "田中さん V." "田中さん, PhD" +"田中さん, V." "田中さん, 太郎" "田中博士" "諸葛亮" diff --git a/tools/differential/expected_changes.toml b/tools/differential/expected_changes.toml index d5887cf..9e4fef9 100644 --- a/tools/differential/expected_changes.toml +++ b/tools/differential/expected_changes.toml @@ -108,7 +108,9 @@ issue = "fix(comma-family) lone post-comma piece routes to suffix/title, not fir # The coverage pass that followed added one more row here, '김, 민준씨 # (Jimmy)': same {first, suffix} diff as '김, 민준씨' -- the extracted # nickname matches on both sides and so is not part of the diff -- and -# it lands on this rule for the same reason. +# it lands on this rule for the same reason. The limit row recorded +# next to it, '田中さん, V.', appears under no rule at all: it is +# parity with 1.4.0, so it never reaches classify(). name_regex = "," fields = ["first", "title", "suffix"] From c100d8f91c4ddb130c8e172a2ac13b1120c709a1 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 1 Aug 2026 21:39:43 -0700 Subject: [PATCH 11/14] Correct two measured-false claims in the stage's comments (#312) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both are claims about code outside this stage, and both were written from reasoning rather than from a run. The site comment said every non-family-comma structure "puts post-nominals after" segments[0]. That holds for the SUFFIX comma only. Under NO_COMMA segment returns exactly one run and a trailing post-nominal sits inside it -- '김민준씨 Jr.' is NO_COMMA with segments ((0, 1),) -- which is the fact the scan-back paragraph in the docstring above already rests on, so the two paragraphs contradicted each other on the more common structure. The ASCII bail said a glued Latin post-nominal "gets neither treatment and simply stays in the name". It gets the period-joined treatment: period_joined_vocab('Smith.Jr.') is 'title', 'jr' being title vocabulary as well as suffix vocabulary, and where position allows the treatment wins -- parse("Smith.Jr. Anderson") gives title 'Smith.Jr.', family 'Anderson'. The example the comment chose ("John Smith.Jr.") only loses to the last-name reservation. The sentence needs no theory of that function anyway; what it has to say is that the bail returns above all of it. Co-Authored-By: Claude Opus 5 --- nameparser/_pipeline/_script_segment.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 2eb8d11..3662b6c 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -385,8 +385,14 @@ def _peel_honorific_tail(state: ParseState) -> ParseState: # 민준씨" is (0,) and (1,)), and the honorific is as often glued to # the given name as to the family, so the peel has to cross that # boundary (#312). Every OTHER structure keeps the whole name in - # segments[0] and puts post-nominals after it -- which is why the - # asymmetry is the point rather than an accident of two cases. + # segments[0], so no boundary has to be crossed to find the site + # there -- which is why the asymmetry is the point rather than an + # accident of two cases. Note that "the rest is post-nominals" is + # true of the SUFFIX comma only: under NO_COMMA segment returns + # exactly one run and any trailing post-nominal is INSIDE it + # ("김민준씨 Jr." is one run of two tokens), which is why the + # scan-back above steps over such a token rather than simply never + # reaching it. # Reaching past the name's own runs is a live bug, not tidiness: # segment admits a post-comma run on is_suffix_lenient while # _is_post_nominal asks is_suffix_strict, and the initial-shaped @@ -604,10 +610,14 @@ def script_segment(state: ParseState) -> ParseState: # glue is PREnominal and period-joined (Mr.Smith, Lt.Gov.), # which _vocab.period_joined_vocab classifies as one token # rather than splitting -- a different mechanism, and no peel - # site either way. A glued Latin post-nominal gets neither - # treatment and simply stays in the name ("John Smith.Jr." -> - # family "Smith.Jr."), which the ASCII bail here is what makes - # moot rather than anything downstream. + # site either way. A glued Latin POST-nominal is spelled the + # same way, so it reaches that same mechanism rather than + # nothing: period_joined_vocab reads "Smith.Jr." as a title + # ('jr' is title vocabulary as well as suffix vocabulary), and + # where position allows, that wins -- "Smith.Jr. Anderson" + # gives title "Smith.Jr.", family "Anderson". Whatever it + # decides, this bail is what settles the question here: it + # returns above all of it, so no ASCII input reaches the peel. return state if not state.segments: return state From 8cef3168e011a011cde82b96cf13d3802f808d7e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 1 Aug 2026 21:40:49 -0700 Subject: [PATCH 12/14] Extend two stage comments that #312 outgrew MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The maiden paragraph was written entirely in NO_COMMA terms ("which under NO_COMMA reaches into a maiden clause"). #312 extended that reach to FAMILY_COMMA along with the rest of the crossing: '김, 민준 née 박씨' gives suffix 씨 / maiden 박 here where master gives maiden '박씨'. Same behavior, one structure wider. The "would strand" argument at the site scan reads as a counterfactual about a mutation nobody would ship. Policy(lenient_comma_suffixes=False) reaches it without touching the code: it drops segment's post-comma test to the strict one, which reads 'Dr 김민준씨, V.' as FAMILY_COMMA, so the scan crosses to 'V.' through the sanctioned two-run reach and strands 씨 in family 'Dr 김민준씨'. Co-Authored-By: Claude Opus 5 --- nameparser/_pipeline/_script_segment.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 3662b6c..fc73cf3 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -340,14 +340,17 @@ def _peel_honorific_tail(state: ParseState) -> ParseState: -- so "김민준씨 (Jimmy)" and "Dr 김민준씨, V." are the inputs that tell the three apart. See the note at the scan itself. - That last token is the last of the NAME, which under NO_COMMA - reaches into a maiden clause: maiden tokens are still main-stream - here (extract_delimited has masked only bracketed content), so + That last token is the last of the NAME, which reaches into a + maiden clause: maiden tokens are still main-stream here + (extract_delimited has masked only bracketed content), so "김민준 née 박씨" peels 씨 off the MAIDEN name 박씨 and hands it to the person's suffix list -- "née Ms. Park". Intended rather than incidental in that direction: the honorific is the reader's regardless of which of her names it was glued to, and a - name-final honorific is exactly what this peels. + name-final honorific is exactly what this peels. Since #312 that + reach extends to FAMILY_COMMA along with the rest of the crossing + -- "김, 민준 née 박씨" now routes 씨 to suffix where on the + unscoped segments[0] it stayed in maiden "박씨". The other direction is a LIMIT, stated rather than fixed: a maiden clause pushes the site off the person's own name, so "김민준씨 née @@ -400,7 +403,15 @@ def _peel_honorific_tail(state: ParseState) -> ParseState: # such a token ends in no tail and silently abandons the peel, so # "Dr 김민준씨, V." would strand 씨 in the given name. A junk tail # is worse: "Dr 김민준씨, Jr., 박씨" would peel the 씨 off 박씨 and - # leave the person's own glued. + # leave the person's own glued. Not only a counterfactual, either: + # Policy(lenient_comma_suffixes=False) drops segment's post-comma + # test to the strict one, which reads that same input as + # FAMILY_COMMA -- so the scan reaches "V." through the sanctioned + # two-run crossing and strands 씨 for real, in family + # "Dr 김민준씨". Same gap, reached by a documented knob rather + # than by widening this line; see the case row + # ja_honorific_glued_family_comma_suffixy_second_run, which is the + # DEFAULT-policy shape of it. # Flattening the SEGMENTS rather than state.tokens is load-bearing # too: extracted nickname and maiden content is in tokens but in NO # segment, and scanning tokens would put the peel site on a From 6c0c3bc5cc4aa6652647300c214ec148d436864c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 1 Aug 2026 21:42:48 -0700 Subject: [PATCH 13/14] Scope the peel's comma claim to what it actually does (#312) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "No comma structure and no dot gates the peel any more" and "A comma and a 间隔号 both leave the peel alone" are universal negatives, and both have counterexamples. Measured: 김, 민준 지훈씨 -> suffix 씨 peels (two runs) 김, 민준, 지훈씨 -> suffix 지훈씨 does not (third run) 김,, 민준씨 -> suffix 민준씨 does not (later run) 田中さん, V. -> family 田中さん does not (second run is not name text) One fact with four faces: the peel reaches the two runs around a family comma and nothing beyond, presuming the second of them is name text. Say that instead, in one sentence rather than as a list of caveats, in the release log, in usage.rst and in AGENTS.md's architecture paragraph (which named the slice but not its limit). Also fix the scope of "likewise" in the release log. "gives ... the same as the spaced 김 민준씨 -- and likewise 田中, 太郎さん, 田中さん, PhD and 威廉·莎士比亚さん" holds for two of the three: 田中さん PhD gives suffix 'さん, PhD' while 田中さん, PhD gives title PhD. cases.py states that difference; the release log left it inferable and wrong. Co-Authored-By: Claude Opus 5 --- AGENTS.md | 2 +- docs/release_log.rst | 2 +- docs/usage.rst | 25 ++++++++++++++++--------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e791068..9ede3b1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,7 +75,7 @@ logging.getLogger('HumanName').setLevel(logging.DEBUG) The library has two layers: `nameparser/config/` (data) and `nameparser/parser.py` (logic). -**Design philosophy — positional and language-agnostic.** The parser assigns parts by *position* plus small sets of words that join to neighbors; it never detects language. A name's language can't be reliably inferred from Latin-script transliteration ("Ali" is Arabic or Italian; "Van"/"Della"/"Bin" are first names in some cultures, particles in others), so language-specific rules belong in opt-in `Constants` config, never global defaults. Many "wrong for language X" reports (#133, #150, #130, #85, #103, #146, #83) are irreducible ambiguities — e.g. `de Mesnil` (want last name) vs `Van Johnson` (want first name) are the same `[prefix][word]` shape. Before adding a rule, confirm it doesn't break the opposite case (run the full suite — Portuguese and "Van Johnson" tests are the usual canaries). **The one scoped exception (2.1, #271/#272): script-conditional behavior is permitted exactly where the SCRIPT ITSELF — not statistics about it — determines the convention.** The never-detect-language rule above is about Latin *transliteration*, where the signal genuinely is destroyed; native script is a different question, and it is answered per behavior rather than per script. Five defaults fall out of it, plus a sixth that applies the same not-a-guess standard to specific WORDS rather than to a script (#308's honorific peel, below). Wholly-Han, wholly-Hangul and kana-licensed names read family-first (`Policy.script_orders`) — no language detection needed, because zh and ja both write family-first in native script, so order cannot be misread even though the language is unknowable. Unspaced hangul splits into surname + given name (`config/surnames.py` ships the Korean census list as DEFAULT vocabulary) — nothing but Korean is written in hangul and the surnames are a closed census set, and the vocabulary is self-selecting besides: a hangul entry can only ever match hangul text. Hiragana licenses Japanese (#272) — a name whose characters stay inside Han∪kana while carrying at least one kana cannot be Chinese (the kana rules it out) and is not a transcription (foreign names are transcribed in katakana ALONE, マイケル has no kanji), so 高橋みなみ and 山田 エミ read family-first too; mechanically they resolve to the HIRAGANA entry, the license's carrier key. PURE katakana is excluded and keeps the positional default: マイケル・ジャクソン is a transcribed foreign name in its source order. And the 间隔号 U+00B7 (#298) is the transcription marker for scripts that HAVE no transcription script: a name it divides (威廉·莎士比亚 — flanked by classified characters on both sides, so Catalan's Gal·la is untouched) keeps its source order and never segments — the orthography names the convention, exactly as pure katakana does, with the divider carrying the signal instead of the script. And a listed CJK honorific glued to the END of a name token is split off it (#308) — 田中さん is 田中 plus さん — on the same orthography-settles-it test, narrowed for the glued position: an entry peels only where it can never end a name, so 씨/님/さん/様/先生 peel while 양/군/氏/博士/殿 stay spaced-only (김지양 and 田中博士 are names, and ~90 Japanese surnames end in 殿) and 君 is in NEITHER set (王君 is a complete Chinese name), though its kana spelling くん peels. Like the nakaguro's tokenize-level separation described next, it is reached by neither policy opt-out — but for its own reason: the vocabulary carries the license itself rather than borrowing the script's, so `segment_scripts` has nothing to say about it. Since #312 it also crosses the FAMILY comma and the 间隔号, which is the more surprising fact, since both of those still stop the surname split standing right beside it: each answers where a name DIVIDES into surname and given, and the peel never asks that question — so `김, 민준씨` reads exactly as the spaced `김 민준씨` does (family 김, given 민준, suffix 씨) while the split stands down as before. Its site is accordingly the name-bearing segment runs — `segments[:2]` under a family comma, and `segments[0]` as before otherwise, the family comma being the one structure that splits the name itself across two runs, with the honorific as often glued to the given side as to the family; `田中さん, 太郎` is unchanged, and not because of its comma — the honorific there is not at the end of the name, 太郎 is. The nakaguro belongs to the same doctrine but is decided a layer down: U+30FB and its halfwidth twin U+FF65 separate tokens like whitespace, unconditionally and in tokenize, so neither policy opt-out (`script_orders={}`, `segment_scripts=()`) reaches it — the codepoints are CJK-only and appear in no other script's names, which is what licenses a tokenize-level rule where U+00B7 (also the Catalan punt volat, interior to Gal·la) needs the flanked-by-classified-script guard `_tokenize_region` gives it (#298). Han segmentation stays OPT-IN (`locales.ZH` for Chinese, `locales.JA` for Japanese) — a zh surname list corrupts Japanese kanji names, since 高 is a common Chinese surname and 高橋一郎 would split 高+橋一郎 where the correct reading is 高橋+一郎; no surname list divides a kanji name at all, so `locales.JA` activates the stage and a pluggable `Parser(segmenter=...)` does the dividing. Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely order-ambiguous and stays governed by `name_order` and opt-in packs. Before adding a script-conditional rule, work out which of the three it is — certain, certain for this one behavior only, or a statistical guess wearing a script's clothes. +**Design philosophy — positional and language-agnostic.** The parser assigns parts by *position* plus small sets of words that join to neighbors; it never detects language. A name's language can't be reliably inferred from Latin-script transliteration ("Ali" is Arabic or Italian; "Van"/"Della"/"Bin" are first names in some cultures, particles in others), so language-specific rules belong in opt-in `Constants` config, never global defaults. Many "wrong for language X" reports (#133, #150, #130, #85, #103, #146, #83) are irreducible ambiguities — e.g. `de Mesnil` (want last name) vs `Van Johnson` (want first name) are the same `[prefix][word]` shape. Before adding a rule, confirm it doesn't break the opposite case (run the full suite — Portuguese and "Van Johnson" tests are the usual canaries). **The one scoped exception (2.1, #271/#272): script-conditional behavior is permitted exactly where the SCRIPT ITSELF — not statistics about it — determines the convention.** The never-detect-language rule above is about Latin *transliteration*, where the signal genuinely is destroyed; native script is a different question, and it is answered per behavior rather than per script. Five defaults fall out of it, plus a sixth that applies the same not-a-guess standard to specific WORDS rather than to a script (#308's honorific peel, below). Wholly-Han, wholly-Hangul and kana-licensed names read family-first (`Policy.script_orders`) — no language detection needed, because zh and ja both write family-first in native script, so order cannot be misread even though the language is unknowable. Unspaced hangul splits into surname + given name (`config/surnames.py` ships the Korean census list as DEFAULT vocabulary) — nothing but Korean is written in hangul and the surnames are a closed census set, and the vocabulary is self-selecting besides: a hangul entry can only ever match hangul text. Hiragana licenses Japanese (#272) — a name whose characters stay inside Han∪kana while carrying at least one kana cannot be Chinese (the kana rules it out) and is not a transcription (foreign names are transcribed in katakana ALONE, マイケル has no kanji), so 高橋みなみ and 山田 エミ read family-first too; mechanically they resolve to the HIRAGANA entry, the license's carrier key. PURE katakana is excluded and keeps the positional default: マイケル・ジャクソン is a transcribed foreign name in its source order. And the 间隔号 U+00B7 (#298) is the transcription marker for scripts that HAVE no transcription script: a name it divides (威廉·莎士比亚 — flanked by classified characters on both sides, so Catalan's Gal·la is untouched) keeps its source order and never segments — the orthography names the convention, exactly as pure katakana does, with the divider carrying the signal instead of the script. And a listed CJK honorific glued to the END of a name token is split off it (#308) — 田中さん is 田中 plus さん — on the same orthography-settles-it test, narrowed for the glued position: an entry peels only where it can never end a name, so 씨/님/さん/様/先生 peel while 양/군/氏/博士/殿 stay spaced-only (김지양 and 田中博士 are names, and ~90 Japanese surnames end in 殿) and 君 is in NEITHER set (王君 is a complete Chinese name), though its kana spelling くん peels. Like the nakaguro's tokenize-level separation described next, it is reached by neither policy opt-out — but for its own reason: the vocabulary carries the license itself rather than borrowing the script's, so `segment_scripts` has nothing to say about it. Since #312 it also crosses the FAMILY comma and the 间隔号, which is the more surprising fact, since both of those still stop the surname split standing right beside it: each answers where a name DIVIDES into surname and given, and the peel never asks that question — so `김, 민준씨` reads exactly as the spaced `김 민준씨` does (family 김, given 민준, suffix 씨) while the split stands down as before. Its site is accordingly the name-bearing segment runs — `segments[:2]` under a family comma, and `segments[0]` as before otherwise, the family comma being the one structure that splits the name itself across two runs, with the honorific as often glued to the given side as to the family. That is the whole reach and nothing past it (`김, 민준 지훈씨` peels; `김, 민준, 지훈씨` and `김,, 민준씨` do not, both landing in a third run), and it presumes `segments[1]` is name text, which `segment` does not guarantee — a one-word part before the comma reads as FAMILY_COMMA even when the part after it is entirely suffix-shaped, so `田中さん, V.` puts the scan on `V.` and さん stays in the family name where `田中さん, PhD` gives it up (parity with 1.4.0, a strict/lenient gap that predates #312, and the reason to reach for a shared `suffixy` predicate before widening this). `田中さん, 太郎` is unchanged, and not because of its comma — the honorific there is not at the end of the name, 太郎 is. The nakaguro belongs to the same doctrine but is decided a layer down: U+30FB and its halfwidth twin U+FF65 separate tokens like whitespace, unconditionally and in tokenize, so neither policy opt-out (`script_orders={}`, `segment_scripts=()`) reaches it — the codepoints are CJK-only and appear in no other script's names, which is what licenses a tokenize-level rule where U+00B7 (also the Catalan punt volat, interior to Gal·la) needs the flanked-by-classified-script guard `_tokenize_region` gives it (#298). Han segmentation stays OPT-IN (`locales.ZH` for Chinese, `locales.JA` for Japanese) — a zh surname list corrupts Japanese kanji names, since 高 is a common Chinese surname and 高橋一郎 would split 高+橋一郎 where the correct reading is 高橋+一郎; no surname list divides a kanji name at all, so `locales.JA` activates the stage and a pluggable `Parser(segmenter=...)` does the dividing. Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely order-ambiguous and stays governed by `name_order` and opt-in packs. Before adding a script-conditional rule, work out which of the three it is — certain, certain for this one behavior only, or a statistical guess wearing a script's clothes. ### Configuration layer (`nameparser/config/`) diff --git a/docs/release_log.rst b/docs/release_log.rst index dd5f814..6eb8994 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -28,7 +28,7 @@ Release Log - Fix 间隔号-divided transcriptions parsing as one unsplit token: U+00B7 — the interpunct Chinese text divides a transcribed foreign name with, ``威廉·莎士比亚`` for William Shakespeare — is now a token separator between characters of a classified script, and a name it divides keeps its source order and is never segmented: the dot is the transcription marker, playing the role pure katakana plays in the kana license. It divides ONLY between classified-script characters, so the Catalan punt volat interior to names like ``Gal·la`` is untouched. The Japanese nakaguro is deliberately not a transcription marker — ``高橋・一郎`` is roster formatting, 姓・名, and keeps its family-first reading — so a transcription typed with the wrong dot (``威廉・莎士比亚``) reads by the convention of the codepoint it was typed with; like the spaced form, only the Chinese dot rescues it. Rendering stays space-joined (``str(HumanName("威廉·莎士比亚"))`` is ``"威廉 莎士比亚"``); a custom ``string_format`` such as ``"{first}·{last}"`` reinstates the dot (#298) - Fix spaced CJK postnominal honorifics parsing as name parts: 씨, 박사, 선생님, 교수님, 군, 양 (Korean — standardly written as their own token), 先生, 女士, 小姐, 博士, 教授 (Chinese, with 先生/博士/教授 shared with Japanese), and 様, 氏 (Japanese) now route to ``suffix``, so ``王小明 先生`` reads family ``王小明`` where the family-first default had confidently made 先生 the given name. Whole-token matching, which also reaches a glued surname+honorific token, since segmentation splits off the surname first (``김씨`` reads family 김, suffix 씨). The glued forms whole-token matching cannot reach are handled by the peel described below (closes #307) - Fix glued CJK honorifics parsing as part of the name: an honorific written against the name — ``田中さん``, ``山田太郎様``, ``김민준씨``, ``김민준님``, ``王小明先生`` — is now split off the end of the last name token and routed to ``suffix``, where it had been swallowed by the name (the whole of ``田中さん`` was the family name; ``김민준씨`` gave given 민준씨). The peeled name then goes through the ordinary machinery, so the Korean split still happens (``김민준씨`` → family 김, given 민준, suffix 씨) and the ``田中さん`` case stops misreading as a kana-licensed composite. Only entries that can never END a name peel — 씨, 님, 박사, 박사님, 선생님, 교수님, さん, さま, くん, ちゃん, 様, 先生, 教授, 女士, 小姐 — while 양, 군, 氏, 博士 and 殿 are recognized in their spaced form only, because 김지양 is a given name, 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka, and some ninety Japanese surnames end in 殿 (鵜殿, 真殿); 君 is recognized in neither form, since 王君 is a complete Chinese name, though its kana spelling くん peels. Seven entries are new vocabulary in this change (さん, さま, くん, ちゃん, 殿, 님, 박사님), so their spaced forms route to ``suffix`` too (``田中 さん``, ``田中 殿``, ``김민준 님``, ``김민준 박사님``). 박사님 closes a gap in the shipped set rather than opening new ground: 선생님 and 교수님 shipped in 2.1 without it, so ``김민준박사님`` stranded 박사 in the given name and the spaced ``김민준 박사님`` came back as two suffixes for one honorific. Exactly one honorific peels off a token, and every entry is a whole honorific rather than a part of one. A token that is nothing but an honorific is no longer taken apart either — ``선생님`` and ``박사`` now stay whole where hangul segmentation had split them 선 + 생님 and 박 + 사, since 선 and 박 are listed surnames — which is **default-on** in its own right, hangul segmentation being a default. A configured segmenter benefits twice over: it is handed the name without the honorific, and the honorific the peel just cut off does not then look to it like a boundary the writer drew — so ``parser_for(locales.JA, segmenter=ja_segmenter())`` reads ``山田太郎様`` as family 山田, given 太郎, suffix 様. Worth knowing before you upgrade: that exemption is what makes a GLUED honorific stop protecting a name from division, so a family name written alone with one — ``田中さん`` — now divides the way bare ``田中`` already did (family 田, given 中, suffix さん). It is exactly and only the peeled tail that is exempt. A SPACED honorific is a token boundary its writer typed, and anything standing beside a name calls the segmenter off, so the name is left as written: ``田中 さん`` and ``佐藤 氏`` keep family 田中 and 佐藤 under the pack — the division the pack gives them without this change, whichever field the honorific itself lands in. That is the conservative reading rather than a claim about intent: a spaced honorific cannot be told apart from a spaced given name by position, and counting it as one keeps four real surnames whole (``佐藤 氏``, ``田中 様``, ``鈴木 先生``, ``中村 教授``) at the price of the one division it then declines to make (``山田太郎 様``). Writing the honorific spaced is therefore an opt-out in its own right on the SEGMENTER path, alongside declining the pack or the segmenter. It is no lever where the VOCABULARY divides the name, the two spellings agreeing exactly there — ``김민준 씨`` and ``김민준씨`` both give family 김, given 민준, suffix 씨, as do ``王小明 先生`` and ``王小明先生`` under the Chinese pack — and Korean data has no pack to decline either, hangul segmentation being on by default. **Default-on: changes parse output for glued CJK honorific forms**, through ``HumanName`` as well as the 2.0 API (closes #308) - - Fix a comma or a 间隔号 stopping the glued-honorific peel: an honorific written against the name is now split off and routed to ``suffix``, so ``김, 민준씨`` gives family 김, given 민준, suffix 씨 — the same as the spaced ``김 민준씨`` — and likewise ``田中, 太郎さん``, ``田中さん, PhD`` and ``威廉·莎士比亚さん``. Previously each of those left the honorific inside the name. No comma structure and no dot gates the peel any more — it reads the comma structure only to find which segments the name occupies, an honorific being as often glued to the given name as to the family: both of those marks say where a name divides into surname and given, and an honorific is not part of the name in either spelling. The surname split still stands down for both, unchanged — a comma still means the writer said where the family name ends, and the 间隔号 still marks a transcription. ``田中さん, 太郎`` is unaffected, because the honorific there is not at the end of the name: 太郎 is. **Default-on**, and it reaches ``HumanName`` too (closes #312) + - Fix a comma or a 间隔号 stopping the glued-honorific peel: an honorific written against the name is now split off and routed to ``suffix``, so ``김, 민준씨`` gives family 김, given 민준, suffix 씨 — the same as the spaced ``김 민준씨`` — and likewise ``田中, 太郎さん``, which also matches its spaced form. ``田中さん, PhD`` and ``威廉·莎士比亚さん`` peel too; the first of those does *not* otherwise match its spaced form, since ``田中さん PhD`` leaves PhD in ``suffix`` beside さん while after a comma it reads as a ``title`` — where the credential lands is the comma's business, not the peel's. Previously each of these left the honorific inside the name. A comma no longer switches the peel off; what it does now is say which runs of the name to look in, and those are the two around a family comma, an honorific being as often glued to the given name as to the family. Anything past those two runs is out of reach, which is the one limit worth knowing: ``김, 민준 지훈씨`` peels, while ``김, 민준, 지훈씨`` (a third run) and ``김,, 민준씨`` (a doubled comma, which puts the name in a later run) do not. The reach also rests on the second run being name text, and a one-word part before the comma reads as a family comma even when the part after it is entirely suffix-shaped — so ``田中さん, V.`` keeps さん in the family name where ``田中さん, PhD`` gives it up. Same credential, opposite answer, and unchanged from 1.4.0 in that spelling. The 间隔号 does not switch the peel off either. Both marks say where a name divides into surname and given, and an honorific is not part of the name in either reading. The surname split still stands down for both, unchanged — a comma still means the writer said where the family name ends, and the 间隔号 still marks a transcription. ``田中さん, 太郎`` is unaffected, because the honorific there is not at the end of the name: 太郎 is. **Default-on**, and it reaches ``HumanName`` too (closes #312) - Fix NFD-decomposed input missing the East Asian defaults entirely: script classification now normalizes to NFC before deciding, so a Korean or Japanese name typed on macOS — where decomposed text is routine — gets the same order rule as its composed twin, which it silently did not before. Segmentation MATCHING deliberately stays raw, so an unspaced NFD hangul name is ordered correctly but not split, rather than being split in the wrong place. One gotcha worth stating plainly: parse output preserves the encoding it was given, so for NFD input ``name.family == "김"`` is ``False`` even though it is the same name — compare NFC-normalized text when comparing across encodings (#272) - Fix the Ukrainian conjunction ``й`` not joining the pieces around it: it is the euphonic alternate of ``і``, the two chosen by the surrounding vowel and consonant rather than by meaning (``"Олесь і Олена"`` but ``"Марія й Петро"``), so real Ukrainian data carries both spellings and shipping only ``і`` recognized just one of them. ``"Олесь й Олена Коваленки"`` now gives given ``"Олесь й Олена"`` where the ``й`` previously landed in ``middle``. Same treatment as the ``и``/``і`` entries added in 2.0.0, single-letter carve-out included: the conjunction joins only once the name has enough pieces, and a punctuated initial still wins, so ``"Й. Сліпий"`` is unaffected. Raised in a comment on #267 diff --git a/docs/usage.rst b/docs/usage.rst index f86bf5c..7e3b3d4 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -303,15 +303,22 @@ glued forms are recognized as suffixes: ``王小明 先生`` reads family the last token of the name before the name is split or ordered, so those rules see the name without it — which is why ``김민준씨`` still divides into family 김 and given 민준, and why a configured Japanese -segmenter is handed 山田太郎 rather than 山田太郎様. A comma and a 间隔号 -both leave the peel alone. Both say where a name divides — the comma -that the writer has already given the family name, the dot that the -pieces are a transcription's syllable groups — and an honorific is not -part of the name in any of those readings. So ``김, 민준씨`` reads the -same as ``김 민준씨``. What those marks do stop is the *split*, which -is a different question and still theirs to answer. ``田中さん, 太郎`` is -unchanged, and not because of its comma: the honorific there is not at -the end of the name, 太郎 is. +segmenter is handed 山田太郎 rather than 山田太郎様. Neither a comma nor +a 间隔号 switches the peel off. Both say where a name divides — the +comma that the writer has already given the family name, the dot that +the pieces are a transcription's syllable groups — and an honorific is +not part of the name in either reading. So ``김, 민준씨`` reads the +same as ``김 민준씨``. What a comma does instead is say which runs are +the name: the two around a family comma, an honorific being as often +glued to the given name as to the family. Nothing past those two is in +reach — ``김, 민준 지훈씨`` peels, ``김, 민준, 지훈씨`` does not — and +the reach rests on the second run being name text, so where it is not +the peel is abandoned: in ``田中さん, V.`` the scan lands on ``V.``, +which is neither a name nor an honorific, and さん stays in the family +name where ``田中さん, PhD`` gives it up. What those marks do stop is +the *split*, which is a different question and still theirs to answer. +``田中さん, 太郎`` is unchanged, and not because of its comma: the +honorific there is not at the end of the name, 太郎 is. Where a segmenter divides the name, the two spellings part company. A spaced honorific is a token boundary the writer typed, and the From 3c27dadd11993a1b8e5775cc397810d2774b7fb7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 1 Aug 2026 21:49:22 -0700 Subject: [PATCH 14/14] Correct two comments this pass got wrong itself MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both describe the branch's own mechanism, and both were checked against origin/master afterwards rather than before. The maiden clause said 씨 "stayed in maiden 박씨 on the unscoped segments[0]". It did stay there, but not for that reason: before #312 the family-comma gate sat ABOVE the peel and the stage returned without running it at all. The comma-and-dot test comment said the 间隔号 gates the surname split and the family comma "gates it too". Both stand over the split, but the family comma returns first, so the dot's gate is never reached on this input -- which is the only reason the test is a composition and not two facts. Co-Authored-By: Claude Opus 5 --- nameparser/_pipeline/_script_segment.py | 5 +++-- tests/v2/pipeline/test_script_segment.py | 14 +++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index fc73cf3..f9ae7a0 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -349,8 +349,9 @@ def _peel_honorific_tail(state: ParseState) -> ParseState: regardless of which of her names it was glued to, and a name-final honorific is exactly what this peels. Since #312 that reach extends to FAMILY_COMMA along with the rest of the crossing - -- "김, 민준 née 박씨" now routes 씨 to suffix where on the - unscoped segments[0] it stayed in maiden "박씨". + -- "김, 민준 née 박씨" now routes 씨 to suffix, where before #312 + the stage returned at the family-comma gate above the peel and + left it in maiden "박씨". The other direction is a LIMIT, stated rather than fixed: a maiden clause pushes the site off the person's own name, so "김민준씨 née diff --git a/tests/v2/pipeline/test_script_segment.py b/tests/v2/pipeline/test_script_segment.py index 9bed30a..5cab446 100644 --- a/tests/v2/pipeline/test_script_segment.py +++ b/tests/v2/pipeline/test_script_segment.py @@ -622,13 +622,13 @@ def test_an_empty_first_run_still_reaches_the_second() -> None: def test_the_peel_crosses_a_comma_and_a_dot_at_once() -> None: - # The two gates #312 made siblings, composed: the 间隔号 gates the - # surname split (威廉 stays whole -- 威 is a listed surname here, - # so without the dot it would divide) and the family comma gates - # it too, while the peel crosses both and reaches 太郎さん on the - # far side of the comma. Neither gate alone pins this: keying the - # cross-comma reach on interpunct-free input passes every other - # case in this file. + # The two gates #312 made siblings, composed. Both stand over the + # surname split here -- 威 is a listed surname in this lexicon, so + # bare 威廉 would divide 威 + 廉, and either gate alone stops that + # (the family comma reaches its return first) -- while the peel + # crosses both and reaches 太郎さん on the far side of the comma. + # Neither gate alone pins this: keying the cross-comma reach on + # interpunct-free input passes every other case in this file. lex = _LEX_TAILS.add(surnames={"威"}) state = segment(tokenize(ParseState( original="威廉·莎士比亚, 太郎さん", lexicon=lex, policy=_HAN)))